No OneTemporary
Actions

Size

393 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/cib/remote.c b/cib/remote.c
	index 060101769b..c739f82c92 100644
	--- a/cib/remote.c
	+++ b/cib/remote.c
	@@ -1,701 +1,701 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/socket.h>
	#include <arpa/inet.h>

	#include <netinet/ip.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <glib.h>

	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipcs.h>
	#include <crm/common/xml.h>
	#include <crm/cib/internal.h>

	#include "callbacks.h"
	/* #undef HAVE_PAM_PAM_APPL_H */
	/* #undef HAVE_GNUTLS_GNUTLS_H */

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# undef KEYFILE
	# include <gnutls/gnutls.h>
	#endif

	#include <pwd.h>
	#include <grp.h>
	#if HAVE_SECURITY_PAM_APPL_H
	# include <security/pam_appl.h>
	# define HAVE_PAM 1
	#else
	# if HAVE_PAM_PAM_APPL_H
	# include <pam/pam_appl.h>
	# define HAVE_PAM 1
	# endif
	#endif

	extern int remote_tls_fd;
	extern gboolean cib_shutdown_flag;

	int init_remote_listener(int port, gboolean encrypted);
	void cib_remote_connection_destroy(gpointer user_data);

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# define DH_BITS 1024
	gnutls_dh_params_t dh_params;
	gnutls_anon_server_credentials_t anon_cred_s;
	static void
	debug_log(int level, const char *str)
	{
	fputs(str, stderr);
	}
	#endif

	#define REMOTE_AUTH_TIMEOUT 10000

	int num_clients;
	int authenticate_user(const char user, const char passwd);
	int cib_remote_listen(gpointer data);
	int cib_remote_msg(gpointer data);

	static void
	remote_connection_destroy(gpointer user_data)
	{
	return;
	}

	#define ERROR_SUFFIX " Shutting down remote listener"
	int
	init_remote_listener(int port, gboolean encrypted)
	{
	int rc;
	int *ssock = NULL;
	struct sockaddr_in saddr;
	int optval;

	static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
	.dispatch = cib_remote_listen,
	.destroy = remote_connection_destroy,
	};

	if (port <= 0) {
	/* dont start it */
	return 0;
	}

	if (encrypted) {
	#ifndef HAVE_GNUTLS_GNUTLS_H
	crm_warn("TLS support is not available");
	return 0;
	#else
	crm_notice("Starting a tls listener on port %d.", port);
	- gnutls_global_init();
	+ crm_gnutls_global_init();
	/* gnutls_global_set_log_level (10); */
	gnutls_global_set_log_function(debug_log);
	gnutls_dh_params_init(&dh_params);
	gnutls_dh_params_generate2(dh_params, DH_BITS);
	gnutls_anon_allocate_server_credentials(&anon_cred_s);
	gnutls_anon_set_server_dh_params(anon_cred_s, dh_params);
	#endif
	} else {
	crm_warn("Starting a plain_text listener on port %d.", port);
	}
	#ifndef HAVE_PAM
	crm_warn("PAM is _not_ enabled!");
	#endif

	/* create server socket */
	ssock = malloc(sizeof(int));
	if(ssock == NULL) {
	crm_perror(LOG_ERR, "Can not create server socket." ERROR_SUFFIX);
	return -1;
	}

	*ssock = socket(AF_INET, SOCK_STREAM, 0);
	if (*ssock == -1) {
	crm_perror(LOG_ERR, "Can not create server socket." ERROR_SUFFIX);
	free(ssock);
	return -1;
	}

	/* reuse address */
	optval = 1;
	rc = setsockopt(*ssock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
	if (rc < 0) {
	crm_perror(LOG_INFO, "Couldn't allow the reuse of local addresses by our remote listener");
	}

	/* bind server socket */
	memset(&saddr, '\0', sizeof(saddr));
	saddr.sin_family = AF_INET;
	saddr.sin_addr.s_addr = INADDR_ANY;
	saddr.sin_port = htons(port);
	if (bind(ssock, (struct sockaddr )&saddr, sizeof(saddr)) == -1) {
	crm_perror(LOG_ERR, "Can not bind server socket." ERROR_SUFFIX);
	close(*ssock);
	free(ssock);
	return -2;
	}
	if (listen(*ssock, 10) == -1) {
	crm_perror(LOG_ERR, "Can not start listen." ERROR_SUFFIX);
	close(*ssock);
	free(ssock);
	return -3;
	}

	mainloop_add_fd("cib-remote", G_PRIORITY_DEFAULT, *ssock, ssock, &remote_listen_fd_callbacks);

	return *ssock;
	}

	static int
	check_group_membership(const char usr, const char grp)
	{
	int index = 0;
	struct passwd *pwd = NULL;
	struct group *group = NULL;

	CRM_CHECK(usr != NULL, return FALSE);
	CRM_CHECK(grp != NULL, return FALSE);

	pwd = getpwnam(usr);
	if (pwd == NULL) {
	crm_err("No user named '%s' exists!", usr);
	return FALSE;
	}

	group = getgrgid(pwd->pw_gid);
	if (group != NULL && crm_str_eq(grp, group->gr_name, TRUE)) {
	return TRUE;
	}

	group = getgrnam(grp);
	if (group == NULL) {
	crm_err("No group named '%s' exists!", grp);
	return FALSE;
	}

	while (TRUE) {
	char *member = group->gr_mem[index++];

	if (member == NULL) {
	break;

	} else if (crm_str_eq(usr, member, TRUE)) {
	return TRUE;
	}
	};

	return FALSE;
	}

	static gboolean
	cib_remote_auth(xmlNode * login)
	{
	const char *user = NULL;
	const char *pass = NULL;
	const char *tmp = NULL;

	crm_log_xml_info(login, "Login: ");
	if (login == NULL) {
	return FALSE;
	}

	tmp = crm_element_name(login);
	if (safe_str_neq(tmp, "cib_command")) {
	crm_err("Wrong tag: %s", tmp);
	return FALSE;
	}

	tmp = crm_element_value(login, "op");
	if (safe_str_neq(tmp, "authenticate")) {
	crm_err("Wrong operation: %s", tmp);
	return FALSE;
	}

	user = crm_element_value(login, "user");
	pass = crm_element_value(login, "password");

	if (!user \|\| !pass) {
	crm_err("missing auth credentials");
	return FALSE;
	}

	/* Non-root daemons can only validate the password of the
	* user they're running as
	*/
	if (check_group_membership(user, CRM_DAEMON_GROUP) == FALSE) {
	crm_err("User is not a member of the required group");
	return FALSE;

	} else if (authenticate_user(user, pass) == FALSE) {
	crm_err("PAM auth failed");
	return FALSE;
	}

	return TRUE;
	}

	static gboolean
	remote_auth_timeout_cb(gpointer data)
	{
	crm_client_t *client = data;

	client->remote->auth_timeout = 0;

	if (client->remote->authenticated == TRUE) {
	return FALSE;
	}

	mainloop_del_fd(client->remote->source);
	crm_err("Remote client authentication timed out");

	return FALSE;
	}

	int
	cib_remote_listen(gpointer data)
	{
	int csock = 0;
	unsigned laddr;
	struct sockaddr_in addr;
	int ssock = (int )data;
	int flag;

	crm_client_t *new_client = NULL;

	static struct mainloop_fd_callbacks remote_client_fd_callbacks = {
	.dispatch = cib_remote_msg,
	.destroy = cib_remote_connection_destroy,
	};

	/* accept the connection */
	laddr = sizeof(addr);
	memset(&addr, 0, sizeof(addr));
	csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
	crm_debug("New %s connection from %s",
	ssock == remote_tls_fd ? "secure" : "clear-text", inet_ntoa(addr.sin_addr));

	if (csock == -1) {
	crm_err("accept socket failed");
	return TRUE;
	}

	if ((flag = fcntl(csock, F_GETFL)) >= 0) {
	if (fcntl(csock, F_SETFL, flag \| O_NONBLOCK) < 0) {
	crm_err("fcntl() write failed");
	close(csock);
	return TRUE;
	}
	} else {
	crm_err("fcntl() read failed");
	close(csock);
	return TRUE;
	}

	num_clients++;

	crm_client_init();
	new_client = calloc(1, sizeof(crm_client_t));
	new_client->remote = calloc(1, sizeof(crm_remote_t));

	new_client->id = crm_generate_uuid();

	g_hash_table_insert(client_connections, new_client->id /* Should work */ , new_client);

	if (ssock == remote_tls_fd) {
	#ifdef HAVE_GNUTLS_GNUTLS_H
	new_client->kind = CRM_CLIENT_TLS;

	/* create gnutls session for the server socket */
	new_client->remote->tls_session =
	crm_create_anon_tls_session(csock, GNUTLS_SERVER, anon_cred_s);

	if (new_client->remote->tls_session == NULL) {
	crm_err("TLS session creation failed");
	close(csock);
	return TRUE;
	}
	#endif
	} else {
	new_client->kind = CRM_CLIENT_TCP;
	new_client->remote->tcp_socket = csock;
	}

	/* clients have a few seconds to perform handshake. */
	new_client->remote->auth_timeout =
	g_timeout_add(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, new_client);

	new_client->remote->source =
	mainloop_add_fd("cib-remote-client", G_PRIORITY_DEFAULT, csock, new_client,
	&remote_client_fd_callbacks);

	return TRUE;
	}

	void
	cib_remote_connection_destroy(gpointer user_data)
	{
	crm_client_t *client = user_data;
	int csock = 0;

	if (client == NULL) {
	return;
	}

	crm_trace("Cleaning up after client disconnect: %s/%s", crm_str(client->name), client->id);

	num_clients--;
	crm_trace("Num unfree'd clients: %d", num_clients);

	switch (client->kind) {
	case CRM_CLIENT_TCP:
	csock = client->remote->tcp_socket;
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	if (client->remote->tls_session) {
	void sock_ptr = gnutls_transport_get_ptr(client->remote->tls_session);

	csock = GPOINTER_TO_INT(sock_ptr);
	if (client->remote->tls_handshake_complete) {
	gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_WR);
	}
	gnutls_deinit(*client->remote->tls_session);
	gnutls_free(client->remote->tls_session);
	client->remote->tls_session = NULL;
	}
	break;
	#endif
	default:
	crm_warn("Unexpected client type %d", client->kind);
	}

	if (csock > 0) {
	close(csock);
	}

	crm_client_destroy(client);

	crm_trace("Freed the cib client");

	if (cib_shutdown_flag) {
	cib_shutdown(0);
	}
	return;
	}

	static void
	cib_handle_remote_msg(crm_client_t * client, xmlNode * command)
	{
	const char *value = NULL;

	value = crm_element_name(command);
	if (safe_str_neq(value, "cib_command")) {
	crm_log_xml_trace(command, "Bad command: ");
	return;
	}

	if (client->name == NULL) {
	value = crm_element_value(command, F_CLIENTNAME);
	if (value == NULL) {
	client->name = strdup(client->id);
	} else {
	client->name = strdup(value);
	}
	}

	if (client->userdata == NULL) {
	value = crm_element_value(command, F_CIB_CALLBACK_TOKEN);
	if (value != NULL) {
	client->userdata = strdup(value);
	crm_trace("Callback channel for %s is %s", client->id, client->userdata);

	} else {
	client->userdata = strdup(client->id);
	}
	}

	/* unset dangerous options */
	xml_remove_prop(command, F_ORIG);
	xml_remove_prop(command, F_CIB_HOST);
	xml_remove_prop(command, F_CIB_GLOBAL_UPDATE);

	crm_xml_add(command, F_TYPE, T_CIB);
	crm_xml_add(command, F_CIB_CLIENTID, client->id);
	crm_xml_add(command, F_CIB_CLIENTNAME, client->name);
	#if ENABLE_ACL
	crm_xml_add(command, F_CIB_USER, client->user);
	#endif

	if (crm_element_value(command, F_CIB_CALLID) == NULL) {
	char *call_uuid = crm_generate_uuid();

	/* fix the command */
	crm_xml_add(command, F_CIB_CALLID, call_uuid);
	free(call_uuid);
	}

	if (crm_element_value(command, F_CIB_CALLOPTS) == NULL) {
	crm_xml_add_int(command, F_CIB_CALLOPTS, 0);
	}

	crm_log_xml_trace(command, "Remote command: ");
	cib_common_callback_worker(0, 0, command, client, TRUE);
	}

	int
	cib_remote_msg(gpointer data)
	{
	xmlNode *command = NULL;
	crm_client_t *client = data;
	int disconnected = 0;
	int timeout = client->remote->authenticated ? -1 : 1000;

	crm_trace("%s callback", client->kind != CRM_CLIENT_TCP ? "secure" : "clear-text");

	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (client->kind == CRM_CLIENT_TLS && (client->remote->tls_handshake_complete == FALSE)) {
	int rc = 0;

	/* Muliple calls to handshake will be required, this callback
	* will be invoked once the client sends more handshake data. */
	do {
	rc = gnutls_handshake(*client->remote->tls_session);

	if (rc < 0 && rc != GNUTLS_E_AGAIN) {
	crm_err("Remote cib tls handshake failed");
	return -1;
	}
	} while (rc == GNUTLS_E_INTERRUPTED);

	if (rc == 0) {
	crm_debug("Remote cib tls handshake completed");
	client->remote->tls_handshake_complete = TRUE;
	if (client->remote->auth_timeout) {
	g_source_remove(client->remote->auth_timeout);
	}
	/* after handshake, clients must send auth in a few seconds */
	client->remote->auth_timeout =
	g_timeout_add(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, client);
	}
	return 0;
	}
	#endif

	crm_remote_recv(client->remote, timeout, &disconnected);

	/* must pass auth before we will process anything else */
	if (client->remote->authenticated == FALSE) {
	xmlNode *reg;

	#if ENABLE_ACL
	const char *user = NULL;
	#endif
	command = crm_remote_parse_buffer(client->remote);
	if (cib_remote_auth(command) == FALSE) {
	free_xml(command);
	return -1;
	}

	crm_debug("remote connection authenticated successfully");
	client->remote->authenticated = TRUE;
	g_source_remove(client->remote->auth_timeout);
	client->remote->auth_timeout = 0;
	client->name = crm_element_value_copy(command, "name");

	#if ENABLE_ACL
	user = crm_element_value(command, "user");
	if (user) {
	client->user = strdup(user);
	}
	#endif

	/* send ACK */
	reg = create_xml_node(NULL, "cib_result");
	crm_xml_add(reg, F_CIB_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(reg, F_CIB_CLIENTID, client->id);
	crm_remote_send(client->remote, reg);
	free_xml(reg);
	free_xml(command);
	}

	command = crm_remote_parse_buffer(client->remote);
	while (command) {
	crm_trace("command received");
	cib_handle_remote_msg(client, command);
	free_xml(command);
	command = crm_remote_parse_buffer(client->remote);
	}

	if (disconnected) {
	crm_trace("disconnected while receiving remote cib msg.");
	return -1;
	}

	return 0;
	}

	#ifdef HAVE_PAM
	/*
	* Useful Examples:
	* http://www.kernel.org/pub/linux/libs/pam/Linux-PAM-html
	* http://developer.apple.com/samplecode/CryptNoMore/index.html
	*/
	static int
	construct_pam_passwd(int num_msg, const struct pam_message **msg,
	struct pam_response *response, void data)
	{
	int count = 0;
	struct pam_response *reply;
	char string = (char )data;

	CRM_CHECK(data, return PAM_CONV_ERR);
	CRM_CHECK(num_msg == 1, return PAM_CONV_ERR); /* We only want to handle one message */

	reply = calloc(1, sizeof(struct pam_response));
	CRM_ASSERT(reply != NULL);

	for (count = 0; count < num_msg; ++count) {
	switch (msg[count]->msg_style) {
	case PAM_TEXT_INFO:
	crm_info("PAM: %s\n", msg[count]->msg);
	break;
	case PAM_PROMPT_ECHO_OFF:
	case PAM_PROMPT_ECHO_ON:
	reply[count].resp_retcode = 0;
	reply[count].resp = string; /* We already made a copy */
	case PAM_ERROR_MSG:
	/* In theory we'd want to print this, but then
	* we see the password prompt in the logs
	*/
	/* crm_err("PAM error: %s\n", msg[count]->msg); */
	break;
	default:
	crm_err("Unhandled conversation type: %d", msg[count]->msg_style);
	goto bail;
	}
	}

	*response = reply;
	reply = NULL;

	return PAM_SUCCESS;

	bail:
	for (count = 0; count < num_msg; ++count) {
	if (reply[count].resp != NULL) {
	switch (msg[count]->msg_style) {
	case PAM_PROMPT_ECHO_ON:
	case PAM_PROMPT_ECHO_OFF:
	/* Erase the data - it contained a password */
	while (*(reply[count].resp)) {
	*(reply[count].resp)++ = '\0';
	}
	free(reply[count].resp);
	break;
	}
	reply[count].resp = NULL;
	}
	}
	free(reply);
	reply = NULL;

	return PAM_CONV_ERR;
	}
	#endif

	int
	authenticate_user(const char user, const char passwd)
	{
	#ifndef HAVE_PAM
	gboolean pass = TRUE;
	#else
	int rc = 0;
	gboolean pass = FALSE;
	const void *p_user = NULL;

	struct pam_conv p_conv;
	struct pam_handle *pam_h = NULL;
	static const char *pam_name = NULL;

	if (pam_name == NULL) {
	pam_name = getenv("CIB_pam_service");
	}
	if (pam_name == NULL) {
	pam_name = "login";
	}

	p_conv.conv = construct_pam_passwd;
	p_conv.appdata_ptr = strdup(passwd);

	rc = pam_start(pam_name, user, &p_conv, &pam_h);
	if (rc != PAM_SUCCESS) {
	crm_err("Could not initialize PAM: %s (%d)", pam_strerror(pam_h, rc), rc);
	goto bail;
	}

	rc = pam_authenticate(pam_h, 0);
	if (rc != PAM_SUCCESS) {
	crm_err("Authentication failed for %s: %s (%d)", user, pam_strerror(pam_h, rc), rc);
	goto bail;
	}

	/* Make sure we authenticated the user we wanted to authenticate.
	* Since we also run as non-root, it might be worth pre-checking
	* the user has the same EID as us, since that the only user we
	* can authenticate.
	*/
	rc = pam_get_item(pam_h, PAM_USER, &p_user);
	if (rc != PAM_SUCCESS) {
	crm_err("Internal PAM error: %s (%d)", pam_strerror(pam_h, rc), rc);
	goto bail;

	} else if (p_user == NULL) {
	crm_err("Unknown user authenticated.");
	goto bail;

	} else if (safe_str_neq(p_user, user)) {
	crm_err("User mismatch: %s vs. %s.", (const char )p_user, (const char )user);
	goto bail;
	}

	rc = pam_acct_mgmt(pam_h, 0);
	if (rc != PAM_SUCCESS) {
	crm_err("Access denied: %s (%d)", pam_strerror(pam_h, rc), rc);
	goto bail;
	}
	pass = TRUE;

	bail:
	pam_end(pam_h, rc);
	#endif
	return pass;
	}
	diff --git a/crmd/lrm_state.c b/crmd/lrm_state.c
	index f56f74a89d..72f66c7c38 100644
	--- a/crmd/lrm_state.c
	+++ b/crmd/lrm_state.c
	@@ -1,761 +1,774 @@
	/*
	* Copyright (C) 2012 David Vossel <dvossel@redhat.com>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>

	#include <crmd.h>
	#include <crmd_fsa.h>
	#include <crmd_messages.h>
	#include <crmd_callbacks.h>
	#include <crmd_lrm.h>

	GHashTable *lrm_state_table = NULL;
	GHashTable *proxy_table = NULL;
	int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
	void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void userdata, void (callback)(lrmd_t lrmd, void userdata, xmlNode *msg));

	typedef struct remote_proxy_s {
	char *node_name;
	char *session_id;

	gboolean is_local;

	crm_ipc_t *ipc;
	mainloop_io_t *source;
	uint32_t last_request_id;

	} remote_proxy_t;

	static void
	history_cache_destroy(gpointer data)
	{
	rsc_history_t *entry = data;

	if (entry->stop_params) {
	g_hash_table_destroy(entry->stop_params);
	}

	free(entry->rsc.type);
	free(entry->rsc.class);
	free(entry->rsc.provider);

	lrmd_free_event(entry->failed);
	lrmd_free_event(entry->last);
	free(entry->id);
	free(entry);
	}
	static void
	free_rsc_info(gpointer value)
	{
	lrmd_rsc_info_t *rsc_info = value;

	lrmd_free_rsc_info(rsc_info);
	}

	static void
	free_deletion_op(gpointer value)
	{
	struct pending_deletion_op_s *op = value;

	free(op->rsc);
	delete_ha_msg_input(op->input);
	free(op);
	}

	static void
	free_recurring_op(gpointer value)
	{
	struct recurring_op_s op = (struct recurring_op_s )value;

	free(op->rsc_id);
	free(op->op_type);
	free(op->op_key);
	free(op);
	}

	lrm_state_t *
	lrm_state_create(const char *node_name)
	{
	lrm_state_t *state = NULL;

	if (!node_name) {
	crm_err("No node name given for lrm state object");
	return NULL;
	}

	state = calloc(1, sizeof(lrm_state_t));
	if (!state) {
	return NULL;
	}

	state->node_name = strdup(node_name);

	state->rsc_info_cache = g_hash_table_new_full(crm_str_hash,
	g_str_equal, NULL, free_rsc_info);

	state->deletion_ops = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str, free_deletion_op);

	state->pending_ops = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str, free_recurring_op);

	state->resource_history = g_hash_table_new_full(crm_str_hash,
	g_str_equal, NULL, history_cache_destroy);

	g_hash_table_insert(lrm_state_table, (char *)state->node_name, state);
	return state;

	}

	void
	lrm_state_destroy(const char *node_name)
	{
	g_hash_table_remove(lrm_state_table, node_name);
	}

	static gboolean
	remote_proxy_remove_by_node(gpointer key, gpointer value, gpointer user_data)
	{
	remote_proxy_t *proxy = value;
	const char *node_name = user_data;

	if (safe_str_eq(node_name, proxy->node_name)) {
	return TRUE;
	}

	return FALSE;
	}

	static void
	internal_lrm_state_destroy(gpointer data)
	{
	lrm_state_t *lrm_state = data;

	if (!lrm_state) {
	return;
	}

	crm_trace("Destroying proxy table with %d members", g_hash_table_size(proxy_table));
	g_hash_table_foreach_remove(proxy_table, remote_proxy_remove_by_node, (char *) lrm_state->node_name);
	remote_ra_cleanup(lrm_state);
	lrmd_api_delete(lrm_state->conn);

	if (lrm_state->rsc_info_cache) {
	crm_trace("Destroying rsc info cache with %d members", g_hash_table_size(lrm_state->rsc_info_cache));
	g_hash_table_destroy(lrm_state->rsc_info_cache);
	}
	if (lrm_state->resource_history) {
	crm_trace("Destroying history op cache with %d members", g_hash_table_size(lrm_state->resource_history));
	g_hash_table_destroy(lrm_state->resource_history);
	}
	if (lrm_state->deletion_ops) {
	crm_trace("Destroying deletion op cache with %d members", g_hash_table_size(lrm_state->deletion_ops));
	g_hash_table_destroy(lrm_state->deletion_ops);
	}
	if (lrm_state->pending_ops) {
	crm_trace("Destroying pending op cache with %d members", g_hash_table_size(lrm_state->pending_ops));
	g_hash_table_destroy(lrm_state->pending_ops);
	}

	free((char *)lrm_state->node_name);
	free(lrm_state);
	}

	void
	lrm_state_reset_tables(lrm_state_t * lrm_state)
	{
	if (lrm_state->resource_history) {
	crm_trace("Re-setting history op cache with %d members",
	g_hash_table_size(lrm_state->resource_history));
	g_hash_table_remove_all(lrm_state->resource_history);
	}
	if (lrm_state->deletion_ops) {
	crm_trace("Re-setting deletion op cache with %d members",
	g_hash_table_size(lrm_state->deletion_ops));
	g_hash_table_remove_all(lrm_state->deletion_ops);
	}
	if (lrm_state->pending_ops) {
	crm_trace("Re-setting pending op cache with %d members",
	g_hash_table_size(lrm_state->pending_ops));
	g_hash_table_remove_all(lrm_state->pending_ops);
	}
	if (lrm_state->rsc_info_cache) {
	crm_trace("Re-setting rsc info cache with %d members",
	g_hash_table_size(lrm_state->rsc_info_cache));
	g_hash_table_remove_all(lrm_state->rsc_info_cache);
	}
	}

	static void
	-remote_proxy_free(gpointer data)
	+remote_proxy_end_session(const char *session)
	{
	- remote_proxy_t *proxy = data;
	- crm_debug("Signing out of the IPC Service");
	+ remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);
	+
	+ if (proxy == NULL) {
	+ return;
	+ }
	+ crm_trace("ending session ID %s", proxy->session_id);

	- if (proxy->source != NULL) {
	+ if (proxy->source) {
	mainloop_del_ipc_client(proxy->source);
	}
	+}
	+
	+static void
	+remote_proxy_free(gpointer data)
	+{
	+ remote_proxy_t *proxy = data;

	+ crm_trace("freed proxy session ID %s", proxy->session_id);
	free(proxy->node_name);
	free(proxy->session_id);
	+ free(proxy);
	}

	gboolean
	lrm_state_init_local(void)
	{
	if (lrm_state_table) {
	return TRUE;
	}

	lrm_state_table =
	g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, internal_lrm_state_destroy);
	if (!lrm_state_table) {
	return FALSE;
	}

	proxy_table =
	g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, remote_proxy_free);
	if (!proxy_table) {
	g_hash_table_destroy(lrm_state_table);
	return FALSE;
	}

	return TRUE;
	}

	void
	lrm_state_destroy_all(void)
	{
	if (lrm_state_table) {
	crm_trace("Destroying state table with %d members", g_hash_table_size(lrm_state_table));
	g_hash_table_destroy(lrm_state_table); lrm_state_table = NULL;
	}
	if(proxy_table) {
	crm_trace("Destroying proxy table with %d members", g_hash_table_size(proxy_table));
	g_hash_table_destroy(proxy_table); proxy_table = NULL;
	}
	}

	lrm_state_t *
	lrm_state_find(const char *node_name)
	{
	if (!node_name) {
	return NULL;
	}
	return g_hash_table_lookup(lrm_state_table, node_name);
	}

	lrm_state_t *
	lrm_state_find_or_create(const char *node_name)
	{
	lrm_state_t *lrm_state;

	lrm_state = g_hash_table_lookup(lrm_state_table, node_name);
	if (!lrm_state) {
	lrm_state = lrm_state_create(node_name);
	}

	return lrm_state;
	}

	GList *
	lrm_state_get_list(void)
	{
	return g_hash_table_get_values(lrm_state_table);
	}

	void
	lrm_state_disconnect(lrm_state_t * lrm_state)
	{
	if (!lrm_state->conn) {
	return;
	}
	((lrmd_t *) lrm_state->conn)->cmds->disconnect(lrm_state->conn);
	lrmd_api_delete(lrm_state->conn);
	lrm_state->conn = NULL;
	}

	int
	lrm_state_is_connected(lrm_state_t * lrm_state)
	{
	if (!lrm_state->conn) {
	return FALSE;
	}
	return ((lrmd_t *) lrm_state->conn)->cmds->is_connected(lrm_state->conn);
	}

	int
	lrm_state_poke_connection(lrm_state_t * lrm_state)
	{

	if (!lrm_state->conn) {
	return -1;
	}
	return ((lrmd_t *) lrm_state->conn)->cmds->poke_connection(lrm_state->conn);
	}

	int
	lrm_state_ipc_connect(lrm_state_t * lrm_state)
	{
	int ret;

	if (!lrm_state->conn) {
	lrm_state->conn = lrmd_api_new();
	((lrmd_t *) lrm_state->conn)->cmds->set_callback(lrm_state->conn, lrm_op_callback);
	}

	ret = ((lrmd_t *) lrm_state->conn)->cmds->connect(lrm_state->conn, CRM_SYSTEM_CRMD, NULL);

	if (ret != pcmk_ok) {
	lrm_state->num_lrm_register_fails++;
	} else {
	lrm_state->num_lrm_register_fails = 0;
	}

	return ret;
	}

	static void
	remote_proxy_notify_destroy(lrmd_t lrmd, const char session_id)
	{
	/* sending to the remote node that an ipc connection has been destroyed */
	xmlNode *msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
	crm_xml_add(msg, F_LRMD_IPC_OP, "destroy");
	crm_xml_add(msg, F_LRMD_IPC_SESSION, session_id);
	lrmd_internal_proxy_send(lrmd, msg);
	free_xml(msg);
	}

	static void
	remote_proxy_relay_event(lrmd_t lrmd, const char session_id, xmlNode *msg)
	{
	/* sending to the remote node an event msg. */
	xmlNode *event = create_xml_node(NULL, T_LRMD_IPC_PROXY);
	crm_xml_add(event, F_LRMD_IPC_OP, "event");
	crm_xml_add(event, F_LRMD_IPC_SESSION, session_id);
	add_message_xml(event, F_LRMD_IPC_MSG, msg);
	crm_log_xml_explicit(event, "EventForProxy");
	lrmd_internal_proxy_send(lrmd, event);
	free_xml(event);
	}

	static void
	remote_proxy_relay_response(lrmd_t lrmd, const char session_id, xmlNode *msg, int msg_id)
	{
	/* sending to the remote node a response msg. */
	xmlNode *response = create_xml_node(NULL, T_LRMD_IPC_PROXY);
	crm_xml_add(response, F_LRMD_IPC_OP, "response");
	crm_xml_add(response, F_LRMD_IPC_SESSION, session_id);
	crm_xml_add_int(response, F_LRMD_IPC_MSG_ID, msg_id);
	add_message_xml(response, F_LRMD_IPC_MSG, msg);
	lrmd_internal_proxy_send(lrmd, response);
	free_xml(response);
	}

	static int
	remote_proxy_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata)
	{
	/* Async responses from cib and friends back to clients via pacemaker_remoted */
	xmlNode *xml = NULL;
	remote_proxy_t *proxy = userdata;
	lrm_state_t *lrm_state = lrm_state_find(proxy->node_name);
	uint32_t flags;

	if (lrm_state == NULL) {
	return 0;
	}

	xml = string2xml(buffer);
	if (xml == NULL) {
	crm_warn("Received a NULL msg from IPC service.");
	return 1;
	}

	flags = crm_ipc_buffer_flags(proxy->ipc);
	if (flags & crm_ipc_proxied_relay_response) {
	crm_trace("Passing response back to %.8s on %s: %.200s - request id: %d", proxy->session_id, proxy->node_name, buffer, proxy->last_request_id);
	remote_proxy_relay_response(lrm_state->conn, proxy->session_id, xml, proxy->last_request_id);
	proxy->last_request_id = 0;

	} else {
	crm_trace("Passing event back to %.8s on %s: %.200s", proxy->session_id, proxy->node_name, buffer);
	remote_proxy_relay_event(lrm_state->conn, proxy->session_id, xml);
	}
	free_xml(xml);
	return 1;
	}

	static void
	remote_proxy_disconnected(void *userdata)
	{
	remote_proxy_t *proxy = userdata;
	lrm_state_t *lrm_state = lrm_state_find(proxy->node_name);

	crm_trace("destroying %p", userdata);

	proxy->source = NULL;
	proxy->ipc = NULL;

	if (lrm_state && lrm_state->conn) {
	remote_proxy_notify_destroy(lrm_state->conn, proxy->session_id);
	}
	g_hash_table_remove(proxy_table, proxy->session_id);
	}

	static remote_proxy_t *
	remote_proxy_new(const char node_name, const char session_id, const char *channel)
	{
	static struct ipc_client_callbacks proxy_callbacks = {
	.dispatch = remote_proxy_dispatch_internal,
	.destroy = remote_proxy_disconnected
	};
	remote_proxy_t *proxy = calloc(1, sizeof(remote_proxy_t));

	proxy->node_name = strdup(node_name);
	proxy->session_id = strdup(session_id);

	if (safe_str_eq(channel, CRM_SYSTEM_CRMD)) {
	proxy->is_local = TRUE;
	} else {
	proxy->source = mainloop_add_ipc_client(channel, G_PRIORITY_LOW, 0, proxy, &proxy_callbacks);
	proxy->ipc = mainloop_get_ipc_client(proxy->source);

	if (proxy->source == NULL) {
	remote_proxy_free(proxy);
	return NULL;
	}
	}

	+ crm_trace("created proxy session ID %s", proxy->session_id);
	g_hash_table_insert(proxy_table, proxy->session_id, proxy);

	return proxy;
	}

	gboolean
	crmd_is_proxy_session(const char *session)
	{
	return g_hash_table_lookup(proxy_table, session) ? TRUE : FALSE;
	}

	void
	crmd_proxy_send(const char session, xmlNode msg)
	{
	remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);
	lrm_state_t *lrm_state = NULL;

	if (!proxy) {
	return;
	}
	crm_log_xml_trace(msg, "to-proxy");
	lrm_state = lrm_state_find(proxy->node_name);
	if (lrm_state) {
	crm_trace("Sending event to %.8s on %s", proxy->session_id, proxy->node_name);
	remote_proxy_relay_event(lrm_state->conn, session, msg);
	}
	}

	static void
	crmd_proxy_dispatch(const char session, xmlNode msg)
	{

	crm_log_xml_trace(msg, "CRMd-PROXY[inbound]");

	crm_xml_add(msg, F_CRM_SYS_FROM, session);
	if (crmd_authorize_message(msg, NULL, session)) {
	route_message(C_IPC_MESSAGE, msg);
	}

	trigger_fsa(fsa_source);
	}

	static void
	remote_proxy_cb(lrmd_t lrmd, void userdata, xmlNode *msg)
	{
	lrm_state_t *lrm_state = userdata;
	const char *op = crm_element_value(msg, F_LRMD_IPC_OP);
	const char *session = crm_element_value(msg, F_LRMD_IPC_SESSION);
	int msg_id = 0;

	/* sessions are raw ipc connections to IPC,
	* all we do is proxy requests/responses exactly
	* like they are given to us at the ipc level. */

	CRM_CHECK(op != NULL, return);
	CRM_CHECK(session != NULL, return);

	crm_element_value_int(msg, F_LRMD_IPC_MSG_ID, &msg_id);

	/* This is msg from remote ipc client going to real ipc server */
	if (safe_str_eq(op, "new")) {
	const char *channel = crm_element_value(msg, F_LRMD_IPC_IPC_SERVER);

	CRM_CHECK(channel != NULL, return);

	if (remote_proxy_new(lrm_state->node_name, session, channel) == NULL) {
	remote_proxy_notify_destroy(lrmd, session);
	}
	crm_info("new remote proxy client established to %s, session id %s", channel, session);
	} else if (safe_str_eq(op, "destroy")) {
	- g_hash_table_remove(proxy_table, session);
	+ remote_proxy_end_session(session);

	} else if (safe_str_eq(op, "request")) {
	int flags = 0;
	xmlNode *request = get_message_xml(msg, F_LRMD_IPC_MSG);
	const char *name = crm_element_value(msg, F_LRMD_IPC_CLIENT);
	remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);

	CRM_CHECK(request != NULL, return);

	if (proxy == NULL) {
	/* proxy connection no longer exists */
	remote_proxy_notify_destroy(lrmd, session);
	return;
	} else if ((proxy->is_local == FALSE) && (crm_ipc_connected(proxy->ipc) == FALSE)) {
	- g_hash_table_remove(proxy_table, session);
	+ remote_proxy_end_session(session);
	return;
	}
	proxy->last_request_id = 0;
	crm_element_value_int(msg, F_LRMD_IPC_MSG_FLAGS, &flags);
	crm_xml_add(request, XML_ACL_TAG_ROLE, "pacemaker-remote");

	#if ENABLE_ACL
	CRM_ASSERT(lrm_state->node_name);
	crm_acl_get_set_user(request, F_LRMD_IPC_USER, lrm_state->node_name);
	#endif

	if (proxy->is_local) {
	/* this is for the crmd, which we are, so don't try
	* and connect/send to ourselves over ipc. instead
	* do it directly. */
	crmd_proxy_dispatch(session, request);
	if (flags & crm_ipc_client_response) {
	xmlNode *op_reply = create_xml_node(NULL, "ack");

	crm_xml_add(op_reply, "function", __FUNCTION__);
	crm_xml_add_int(op_reply, "line", __LINE__);
	remote_proxy_relay_response(lrmd, session, op_reply, msg_id);
	free_xml(op_reply);
	}

	} else if(is_set(flags, crm_ipc_proxied)) {
	int rc = crm_ipc_send(proxy->ipc, request, flags, 5000, NULL);

	if(rc < 0) {
	xmlNode *op_reply = create_xml_node(NULL, "nack");

	crm_err("Could not relay %s request %d from %s to %s for %s: %s (%d)",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name, pcmk_strerror(rc), rc);

	/* Send a n'ack so the caller doesn't block */
	crm_xml_add(op_reply, "function", __FUNCTION__);
	crm_xml_add_int(op_reply, "line", __LINE__);
	crm_xml_add_int(op_reply, "rc", rc);
	remote_proxy_relay_response(lrmd, session, op_reply, msg_id);
	free_xml(op_reply);

	} else {
	crm_trace("Relayed %s request %d from %s to %s for %s",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name);
	proxy->last_request_id = msg_id;
	}

	} else {
	int rc = pcmk_ok;
	xmlNode *op_reply = NULL;
	/* For backwards compatibility with pacemaker_remoted <= 1.1.10 */

	crm_trace("Relaying %s request %d from %s to %s for %s",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name);

	rc = crm_ipc_send(proxy->ipc, request, flags, 10000, &op_reply);
	if(rc < 0) {
	crm_err("Could not relay %s request %d from %s to %s for %s: %s (%d)",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name, pcmk_strerror(rc), rc);
	} else {
	crm_trace("Relayed %s request %d from %s to %s for %s",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name);
	}

	if(op_reply) {
	remote_proxy_relay_response(lrmd, session, op_reply, msg_id);
	free_xml(op_reply);
	}
	}
	} else {
	crm_err("Unknown proxy operation: %s", op);
	}
	}

	int
	lrm_state_remote_connect_async(lrm_state_t * lrm_state, const char *server, int port,
	int timeout_ms)
	{
	int ret;

	if (!lrm_state->conn) {
	lrm_state->conn = lrmd_remote_api_new(lrm_state->node_name, server, port);
	if (!lrm_state->conn) {
	return -1;
	}
	((lrmd_t *) lrm_state->conn)->cmds->set_callback(lrm_state->conn, remote_lrm_op_callback);
	lrmd_internal_set_proxy_callback(lrm_state->conn, lrm_state, remote_proxy_cb);
	}

	crm_trace("initiating remote connection to %s at %d with timeout %d", server, port, timeout_ms);
	ret =
	((lrmd_t *) lrm_state->conn)->cmds->connect_async(lrm_state->conn, lrm_state->node_name,
	timeout_ms);

	if (ret != pcmk_ok) {
	lrm_state->num_lrm_register_fails++;
	} else {
	lrm_state->num_lrm_register_fails = 0;
	}

	return ret;
	}

	int
	lrm_state_get_metadata(lrm_state_t * lrm_state,
	const char *class,
	const char *provider,
	const char agent, char *output, enum lrmd_call_options options)
	{
	if (!lrm_state->conn) {
	return -ENOTCONN;
	}

	/* Optimize this... only retrieve metadata from local lrmd connection. Perhaps consider
	* caching result. */
	return ((lrmd_t *) lrm_state->conn)->cmds->get_metadata(lrm_state->conn, class, provider, agent,
	output, options);
	}

	int
	lrm_state_cancel(lrm_state_t * lrm_state, const char rsc_id, const char action, int interval)
	{
	if (!lrm_state->conn) {
	return -ENOTCONN;
	}

	/* Optimize this, cancel requires a synced request/response to the server.
	* Figure out a way to make this async. */
	if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
	return remote_ra_cancel(lrm_state, rsc_id, action, interval);
	}
	return ((lrmd_t *) lrm_state->conn)->cmds->cancel(lrm_state->conn, rsc_id, action, interval);
	}

	lrmd_rsc_info_t *
	lrm_state_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id, enum lrmd_call_options options)
	{
	lrmd_rsc_info_t *rsc = NULL;

	if (!lrm_state->conn) {
	return NULL;
	}
	if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
	return remote_ra_get_rsc_info(lrm_state, rsc_id);
	}

	rsc = g_hash_table_lookup(lrm_state->rsc_info_cache, rsc_id);
	if (rsc == NULL) {
	/* only contact the lrmd if we don't already have a cached rsc info */
	rsc = ((lrmd_t *) lrm_state->conn)->cmds->get_rsc_info(lrm_state->conn, rsc_id, options);
	if (rsc == NULL) {
	return NULL;
	}
	/* cache the result */
	g_hash_table_insert(lrm_state->rsc_info_cache, rsc->id, rsc);
	}

	return lrmd_copy_rsc_info(rsc);

	}

	int
	lrm_state_exec(lrm_state_t * lrm_state, const char rsc_id, const char action, const char userdata, int interval, / ms */
	int timeout, /* ms */
	int start_delay, /* ms */
	lrmd_key_value_t * params)
	{

	if (!lrm_state->conn) {
	lrmd_key_value_freeall(params);
	return -ENOTCONN;
	}

	if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
	return remote_ra_exec(lrm_state,
	rsc_id, action, userdata, interval, timeout, start_delay, params);
	}

	return ((lrmd_t *) lrm_state->conn)->cmds->exec(lrm_state->conn,
	rsc_id,
	action,
	userdata,
	interval,
	timeout,
	start_delay,
	lrmd_opt_notify_changes_only, params);
	}

	int
	lrm_state_register_rsc(lrm_state_t * lrm_state,
	const char *rsc_id,
	const char *class,
	const char provider, const char agent, enum lrmd_call_options options)
	{
	if (!lrm_state->conn) {
	return -ENOTCONN;
	}

	/* optimize this... this function is a synced round trip from client to daemon.
	* The crmd/lrm.c code path should be re-factored to allow the register of resources
	* to be performed async. The lrmd client api needs to make an async version
	* of register available. */
	if (is_remote_lrmd_ra(agent, provider, NULL)) {
	return lrm_state_find_or_create(rsc_id) ? pcmk_ok : -1;
	}

	return ((lrmd_t *) lrm_state->conn)->cmds->register_rsc(lrm_state->conn, rsc_id, class,
	provider, agent, options);
	}

	int
	lrm_state_unregister_rsc(lrm_state_t * lrm_state,
	const char *rsc_id, enum lrmd_call_options options)
	{
	if (!lrm_state->conn) {
	return -ENOTCONN;
	}

	/* optimize this... this function is a synced round trip from client to daemon.
	* The crmd/lrm.c code path that uses this function should always treat it as an
	* async operation. The lrmd client api needs to make an async version unreg available. */
	if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
	lrm_state_destroy(rsc_id);
	return pcmk_ok;
	}

	g_hash_table_remove(lrm_state->rsc_info_cache, rsc_id);

	return ((lrmd_t *) lrm_state->conn)->cmds->unregister_rsc(lrm_state->conn, rsc_id, options);
	}
	diff --git a/crmd/membership.c b/crmd/membership.c
	index 3c17913a5c..c15ed808f7 100644
	--- a/crmd/membership.c
	+++ b/crmd/membership.c
	@@ -1,322 +1,418 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	/* put these first so that uuid_t is defined without conflicts */
	#include <crm_internal.h>

	#include <string.h>

	#include <crm/crm.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cluster/internal.h>
	#include <crmd_messages.h>
	#include <crmd_fsa.h>
	#include <crmd_lrm.h>
	#include <fsa_proto.h>
	#include <crmd_callbacks.h>
	#include <tengine.h>
	#include <membership.h>

	gboolean membership_flux_hack = FALSE;
	void post_cache_update(int instance);

	int last_peer_update = 0;
	guint highest_born_on = -1;

	extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);

	static void
	reap_dead_nodes(gpointer key, gpointer value, gpointer user_data)
	{
	crm_node_t *node = value;

	if (crm_is_peer_active(node) == FALSE) {
	crm_update_peer_join(__FUNCTION__, node, crm_join_none);

	if(node && node->uname) {
	election_remove(fsa_election, node->uname);

	if (safe_str_eq(fsa_our_uname, node->uname)) {
	crm_err("We're not part of the cluster anymore");
	register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);

	} else if (AM_I_DC == FALSE && safe_str_eq(node->uname, fsa_our_dc)) {
	crm_warn("Our DC node (%s) left the cluster", node->uname);
	register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
	}
	}

	if (fsa_state == S_INTEGRATION \|\| fsa_state == S_FINALIZE_JOIN) {
	check_join_state(fsa_state, __FUNCTION__);
	}
	if(node && node->uuid) {
	fail_incompletable_actions(transition_graph, node->uuid);
	}
	}
	}

	gboolean ever_had_quorum = FALSE;

	void
	post_cache_update(int instance)
	{
	xmlNode *no_op = NULL;

	crm_peer_seq = instance;
	crm_debug("Updated cache after membership event %d.", instance);

	g_hash_table_foreach(crm_peer_cache, reap_dead_nodes, NULL);
	set_bit(fsa_input_register, R_MEMBERSHIP);

	if (AM_I_DC) {
	populate_cib_nodes(node_update_quick \| node_update_cluster \| node_update_peer \|
	node_update_expected, __FUNCTION__);
	}

	/*
	* If we lost nodes, we should re-check the election status
	* Safe to call outside of an election
	*/
	register_fsa_action(A_ELECTION_CHECK);

	/* Membership changed, remind everyone we're here.
	* This will aid detection of duplicate DCs
	*/
	no_op = create_request(CRM_OP_NOOP, NULL, NULL, CRM_SYSTEM_CRMD,
	AM_I_DC ? CRM_SYSTEM_DC : CRM_SYSTEM_CRMD, NULL);
	send_cluster_message(NULL, crm_msg_crmd, no_op, FALSE);
	free_xml(no_op);
	}

	static void
	crmd_node_update_complete(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	fsa_data_t *msg_data = NULL;

	last_peer_update = 0;

	if (rc == pcmk_ok) {
	crm_trace("Node update %d complete", call_id);

	} else if(call_id < pcmk_ok) {
	crm_err("Node update failed: %s (%d)", pcmk_strerror(call_id), call_id);
	crm_log_xml_debug(msg, "failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);

	} else {
	crm_err("Node update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
	crm_log_xml_debug(msg, "failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	xmlNode *
	do_update_node_cib(crm_node_t * node, int flags, xmlNode * parent, const char *source)
	{
	const char *value = NULL;
	xmlNode *node_state;

	if (is_set(node->flags, crm_remote_node)) {
	return simple_remote_node_status(node->uname, parent, source);
	}

	if (!node->state) {
	crm_info("Node update for %s cancelled: no state, not seen yet", node->uname);
	return NULL;
	}

	node_state = create_xml_node(parent, XML_CIB_TAG_STATE);
	set_uuid(node_state, XML_ATTR_UUID, node);

	if (crm_element_value(node_state, XML_ATTR_UUID) == NULL) {
	crm_info("Node update for %s cancelled: no id", node->uname);
	free_xml(node_state);
	return NULL;
	}

	crm_xml_add(node_state, XML_ATTR_UNAME, node->uname);

	if (flags & node_update_cluster) {
	if (safe_str_eq(node->state, CRM_NODE_ACTIVE)) {
	value = XML_BOOLEAN_YES;
	} else if (node->state) {
	value = XML_BOOLEAN_NO;
	} else {
	value = NULL;
	}
	crm_xml_add(node_state, XML_NODE_IN_CLUSTER, value);
	}

	if (flags & node_update_peer) {
	value = OFFLINESTATUS;
	if (node->processes & proc_flags) {
	value = ONLINESTATUS;
	}
	crm_xml_add(node_state, XML_NODE_IS_PEER, value);
	}

	if (flags & node_update_join) {
	if(node->join <= crm_join_none) {
	value = CRMD_JOINSTATE_DOWN;
	} else {
	value = CRMD_JOINSTATE_MEMBER;
	}
	crm_xml_add(node_state, XML_NODE_JOIN_STATE, value);
	}

	if (flags & node_update_expected) {
	crm_xml_add(node_state, XML_NODE_EXPECTED, node->expected);
	}

	crm_xml_add(node_state, XML_ATTR_ORIGIN, source);

	return node_state;
	}

	+static void
	+remove_conflicting_node_callback(xmlNode * msg, int call_id, int rc,
	+ xmlNode * output, void *user_data)
	+{
	+ char *node_uuid = user_data;
	+
	+ do_crm_log_unlikely(rc == 0 ? LOG_DEBUG : LOG_NOTICE,
	+ "Deletion of the unknown conflicting node \"%s\": %s (rc=%d)",
	+ node_uuid, pcmk_strerror(rc), rc);
	+ free(node_uuid);
	+}
	+
	+static void
	+search_conflicting_node_callback(xmlNode * msg, int call_id, int rc,
	+ xmlNode * output, void *user_data)
	+{
	+ char *new_node_uuid = user_data;
	+ xmlNode *node_xml = NULL;
	+
	+ if (rc != pcmk_ok) {
	+ if (rc != -ENXIO) {
	+ crm_notice("Searching conflicting nodes for %s failed: %s (%d)",
	+ new_node_uuid, pcmk_strerror(rc), rc);
	+ }
	+ return;
	+
	+ } else if (output == NULL) {
	+ return;
	+ }
	+
	+ if (safe_str_eq(crm_element_name(output), XML_CIB_TAG_NODE)) {
	+ node_xml = output;
	+
	+ } else {
	+ node_xml = __xml_first_child(output);
	+ }
	+
	+ for (; node_xml != NULL; node_xml = __xml_next(node_xml)) {
	+ const char *node_uuid = NULL;
	+ const char *node_uname = NULL;
	+ GHashTableIter iter;
	+ crm_node_t *node = NULL;
	+ gboolean known = FALSE;
	+
	+ if (safe_str_neq(crm_element_name(node_xml), XML_CIB_TAG_NODE)) {
	+ continue;
	+ }
	+
	+ node_uuid = crm_element_value(node_xml, XML_ATTR_ID);
	+ node_uname = crm_element_value(node_xml, XML_ATTR_UNAME);
	+
	+ if (node_uuid == NULL \|\| node_uname == NULL) {
	+ continue;
	+ }
	+
	+ g_hash_table_iter_init(&iter, crm_peer_cache);
	+ while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	+ if (node->uuid
	+ && safe_str_eq(node->uuid, node_uuid)
	+ && node->uname
	+ && safe_str_eq(node->uname, node_uname)) {
	+
	+ known = TRUE;
	+ break;
	+ }
	+ }
	+
	+ if (known == FALSE) {
	+ int delete_call_id = 0;
	+
	+ crm_notice("Deleting unknown node %s/%s which has conflicting uname with %s",
	+ node_uuid, node_uname, new_node_uuid);
	+ delete_call_id = fsa_cib_conn->cmds->delete(fsa_cib_conn, XML_CIB_TAG_NODES, node_xml,
	+ cib_scope_local \| cib_quorum_override);
	+ fsa_register_cib_callback(delete_call_id, FALSE, strdup(node_uuid),
	+ remove_conflicting_node_callback);
	+ }
	+ }
	+
	+ free(new_node_uuid);
	+}

	static void
	node_list_update_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	fsa_data_t *msg_data = NULL;

	if(call_id < pcmk_ok) {
	crm_err("Node list update failed: %s (%d)", pcmk_strerror(call_id), call_id);
	crm_log_xml_debug(msg, "update:failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);

	} else if(rc < pcmk_ok) {
	crm_err("Node update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
	crm_log_xml_debug(msg, "update:failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	+#define NODE_PATH_MAX 512
	+
	void
	populate_cib_nodes(enum node_update_flags flags, const char *source)
	{
	int call_id = 0;
	gboolean from_hashtable = TRUE;
	int call_options = cib_scope_local \| cib_quorum_override;
	xmlNode *node_list = create_xml_node(NULL, XML_CIB_TAG_NODES);

	#if SUPPORT_HEARTBEAT
	if (is_not_set(flags, node_update_quick) && is_heartbeat_cluster()) {
	from_hashtable = heartbeat_initialize_nodelist(fsa_cluster_conn, FALSE, node_list);
	}
	#endif

	#if SUPPORT_COROSYNC
	# if !SUPPORT_PLUGIN
	if (is_not_set(flags, node_update_quick) && is_corosync_cluster()) {
	from_hashtable = corosync_initialize_nodelist(NULL, FALSE, node_list);
	}
	# endif
	#endif

	if (from_hashtable) {
	GHashTableIter iter;
	crm_node_t *node = NULL;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	xmlNode *new_node = NULL;

	crm_trace("Creating node entry for %s/%s", node->uname, node->uuid);
	if(node->uuid && node->uname) {
	+ char xpath[NODE_PATH_MAX];
	+
	/* We need both to be valid */
	new_node = create_xml_node(node_list, XML_CIB_TAG_NODE);
	crm_xml_add(new_node, XML_ATTR_ID, node->uuid);
	crm_xml_add(new_node, XML_ATTR_UNAME, node->uname);
	+
	+ /* Search and remove unknown nodes with the conflicting uname from CIB */
	+ snprintf(xpath, NODE_PATH_MAX,
	+ "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_NODES
	+ "/" XML_CIB_TAG_NODE "[@uname='%s'][@id!='%s']",
	+ node->uname, node->uuid);
	+
	+ call_id = fsa_cib_conn->cmds->query(fsa_cib_conn, xpath, NULL,
	+ cib_scope_local \| cib_xpath);
	+ fsa_register_cib_callback(call_id, FALSE, strdup(node->uuid),
	+ search_conflicting_node_callback);
	}
	}
	}

	crm_trace("Populating <nodes> section from %s", from_hashtable ? "hashtable" : "cluster");

	fsa_cib_update(XML_CIB_TAG_NODES, node_list, call_options, call_id, NULL);
	fsa_register_cib_callback(call_id, FALSE, NULL, node_list_update_callback);

	free_xml(node_list);

	if (call_id >= pcmk_ok && crm_peer_cache != NULL && AM_I_DC) {
	/*
	* There is no need to update the local CIB with our values if
	* we've not seen valid membership data
	*/
	GHashTableIter iter;
	crm_node_t *node = NULL;

	node_list = create_xml_node(NULL, XML_CIB_TAG_STATUS);

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	do_update_node_cib(node, flags, node_list, source);
	}

	if (crm_remote_peer_cache) {
	g_hash_table_iter_init(&iter, crm_remote_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	do_update_node_cib(node, flags, node_list, source);
	}
	}

	fsa_cib_update(XML_CIB_TAG_STATUS, node_list, call_options, call_id, NULL);
	fsa_register_cib_callback(call_id, FALSE, NULL, crmd_node_update_complete);
	last_peer_update = call_id;

	free_xml(node_list);
	}
	}

	static void
	cib_quorum_update_complete(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	fsa_data_t *msg_data = NULL;

	if (rc == pcmk_ok) {
	crm_trace("Quorum update %d complete", call_id);

	} else {
	crm_err("Quorum update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
	crm_log_xml_debug(msg, "failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	void
	crm_update_quorum(gboolean quorum, gboolean force_update)
	{
	ever_had_quorum \|= quorum;
	if (AM_I_DC && (force_update \|\| fsa_has_quorum != quorum)) {
	int call_id = 0;
	xmlNode *update = NULL;
	int call_options = cib_scope_local \| cib_quorum_override;

	update = create_xml_node(NULL, XML_TAG_CIB);
	crm_xml_add_int(update, XML_ATTR_HAVE_QUORUM, quorum);
	crm_xml_add(update, XML_ATTR_DC_UUID, fsa_our_uuid);

	fsa_cib_update(XML_TAG_CIB, update, call_options, call_id, NULL);
	crm_debug("Updating quorum status to %s (call=%d)", quorum ? "true" : "false", call_id);
	fsa_register_cib_callback(call_id, FALSE, NULL, cib_quorum_update_complete);
	free_xml(update);
	}
	fsa_has_quorum = quorum;
	}
	diff --git a/cts/CM_ais.py b/cts/CM_ais.py
	index 0db5f6a75d..9fce591094 100644
	--- a/cts/CM_ais.py
	+++ b/cts/CM_ais.py
	@@ -1,154 +1,154 @@
	'''CTS: Cluster Testing System: AIS dependent modules...
	'''

	__copyright__ = '''
	Copyright (C) 2007 Andrew Beekhof <andrew@suse.de>

	'''

	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public License
	# as published by the Free Software Foundation; either version 2
	# of the License, or (at your option) any later version.
	#
	# This program is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	# GNU General Public License for more details.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write to the Free Software
	# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

	import os, sys, warnings
	from cts.CTSvars import *
	from cts.CM_lha import crm_lha
	from cts.CTS import Process
	from cts.patterns import PatternSelector

	#######################################################################
	#
	# LinuxHA v2 dependent modules
	#
	#######################################################################


	class crm_ais(crm_lha):
	'''
	The crm version 3 cluster manager class.
	It implements the things we need to talk to and manipulate
	crm clusters running on top of openais
	'''
	def __init__(self, Environment, randseed=None, name=None):
	if not name: name="crm-ais"
	crm_lha.__init__(self, Environment, randseed=randseed, name=name)

	self.fullcomplist = {}
	self.templates = PatternSelector(self.name)

	def NodeUUID(self, node):
	return node

	def ais_components(self):

	complist = []
	if not len(self.fullcomplist.keys()):
	for c in ["cib", "lrmd", "crmd", "attrd" ]:
	self.fullcomplist[c] = Process(
	self, c,
	pats = self.templates.get_component(self.name, c),
	badnews_ignore = self.templates.get_component(self.name, "%s-ignore"%c),
	common_ignore = self.templates.get_component(self.name, "common-ignore"))

	self.fullcomplist["pengine"] = Process(
	self, "pengine",
	dc_pats = self.templates.get_component(self.name, "pengine"),
	badnews_ignore = self.templates.get_component(self.name, "pengine-ignore"),
	common_ignore = self.templates.get_component(self.name, "common-ignore"))

	self.fullcomplist["stonith-ng"] = Process(
	self, "stonith-ng", process="stonithd",
	pats = self.templates.get_component(self.name, "stonith"),
	badnews_ignore = self.templates.get_component(self.name, "stonith-ignore"),
	common_ignore = self.templates.get_component(self.name, "common-ignore"))

	vgrind = self.Env["valgrind-procs"].split()
	for key in self.fullcomplist.keys():
	if self.Env["valgrind-tests"]:
	if key in vgrind:
	# Processes running under valgrind can't be shot with "killall -9 processname"
	self.log("Filtering %s from the component list as it is being profiled by valgrind" % key)
	continue
	if key == "stonith-ng" and not self.Env["DoFencing"]:
	continue

	complist.append(self.fullcomplist[key])

	#self.complist = [ fullcomplist["pengine"] ]
	return complist


	class crm_cs_v0(crm_ais):
	'''
	The crm version 3 cluster manager class.
	It implements the things we need to talk to and manipulate

	crm clusters running against version 0 of our plugin
	'''
	def __init__(self, Environment, randseed=None, name=None):
	if not name: name="crm-plugin-v0"
	crm_ais.__init__(self, Environment, randseed=randseed, name=name)

	def Components(self):
	self.ais_components()
	c = "corosync"

	self.fullcomplist[c] = Process(
	self, c,
	pats = self.templates.get_component(self.name, c),
	badnews_ignore = self.templates.get_component(self.name, "%s-ignore"%c),
	common_ignore = self.templates.get_component(self.name, "common-ignore")
	)

	return self.ais_components()


	class crm_cs_v1(crm_cs_v0):
	'''
	The crm version 3 cluster manager class.
	It implements the things we need to talk to and manipulate

	crm clusters running on top of version 1 of our plugin
	'''
	def __init__(self, Environment, randseed=None, name=None):
	if not name: name="crm-plugin-v1"
	crm_cs_v0.__init__(self, Environment, randseed=randseed, name=name)


	class crm_mcp(crm_cs_v0):
	'''
	The crm version 4 cluster manager class.
	It implements the things we need to talk to and manipulate
	crm clusters running on top of native corosync (no plugins)
	'''
	def __init__(self, Environment, randseed=None, name=None):
	if not name: name="crm-mcp"
	crm_cs_v0.__init__(self, Environment, randseed=randseed, name=name)

	if self.Env["have_systemd"]:
	self.update({
	# When systemd is in use, we can look for this instead
	- "Pat:We_stopped" : "%s.*Stopped Corosync Cluster Engine",
	+ "Pat:We_stopped" : "%s.*Corosync Cluster Engine exiting normally",
	})


	class crm_cman(crm_cs_v0):
	'''
	The crm version 3 cluster manager class.
	It implements the things we need to talk to and manipulate
	crm clusters running on top of openais
	'''
	def __init__(self, Environment, randseed=None, name=None):
	if not name: name="crm-cman"
	crm_cs_v0.__init__(self, Environment, randseed=randseed, name=name)
	diff --git a/extra/resources/controld b/extra/resources/controld
	index 2cd85dcbc2..28bffd664c 100644
	--- a/extra/resources/controld
	+++ b/extra/resources/controld
	@@ -1,301 +1,301 @@
	#!/bin/sh
	#
	# Resource Agent for managing the DLM controld process.
	#
	# Copyright (c) 2009 Novell, Inc
	# All Rights Reserved.
	#
	# This program is free software; you can redistribute it and/or modify
	# it under the terms of version 2 of the GNU General Public License as
	# published by the Free Software Foundation.
	#
	# This program is distributed in the hope that it would be useful, but
	# WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
	#
	# Further, this software is distributed without any warranty that it is
	# free of the rightful claim of any third person regarding infringement
	# or the like. Any license provided herein, whether implied or
	# otherwise, applies only to this software file. Patent licenses, if
	# any, provided herein do not apply to combinations of this program with
	# other software, or any other product whatsoever.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write the Free Software Foundation,
	# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
	#

	#######################################################################
	# Initialization:

	: ${OCF_FUNCTIONS=${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs}
	. ${OCF_FUNCTIONS}
	: ${__OCF_ACTION=$1}

	#######################################################################

	if [ -e "$OCF_ROOT/resource.d/heartbeat/controld" ]; then
	ocf_log info "Using heartbeat controld agent"
	$OCF_ROOT/resource.d/heartbeat/controld $1
	exit $?
	fi

	meta_data() {
	cat <<END
	<?xml version="1.0"?>
	<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
	<resource-agent name="controld" version="0.9">
	<version>1.0</version>

	<longdesc lang="en">
	This Resource Agent can control the dlm_controld services needed by cluster-aware file systems.
	It assumes that dlm_controld is in your default PATH.
	In most cases, it should be run as an anonymous clone.
	</longdesc>
	<shortdesc lang="en">DLM Agent for cluster file systems</shortdesc>

	<parameters>

	<parameter name="args" unique="1">
	<longdesc lang="en">
	Any additional options to start the dlm_controld service with
	</longdesc>
	<shortdesc lang="en">DLM Options</shortdesc>
	<content type="string" default="-s 0" />
	</parameter>

	<parameter name="configdir" unique="1">
	<longdesc lang="en">
	The location where configfs is or should be mounted
	</longdesc>
	<shortdesc lang="en">Location of configfs</shortdesc>
	<content type="string" default="/sys/kernel/config" />
	</parameter>

	<parameter name="daemon" unique="1">
	<longdesc lang="en">
	The daemon to start - supports gfs_controld(.pcmk) and dlm_controld(.pcmk)
	</longdesc>
	<shortdesc lang="en">The daemon to start</shortdesc>
	<content type="string" default="dlm_controld.pcmk" />
	</parameter>

	<parameter name="allow_stonith_disabled">
	<longdesc lang="en">
	Allow DLM start-up even if STONITH/fencing is disabled in the cluster.

	Setting this option to true will cause cluster malfunction and hangs on
	fail-over for DLM clients that require fencing (such as GFS2, OCFS2, and
	cLVM2).

	This option is advanced use only.
	</longdesc>
	<shortdesc lang="en">Allow start-up even without STONITH/fencing</shortdesc>
	<content type="string" default="false" />
	</parameter>

	</parameters>

	<actions>
	<action name="start" timeout="90" />
	<action name="stop" timeout="100" />
	<action name="monitor" timeout="20" interval="10" depth="0" start-delay="0" />
	<action name="meta-data" timeout="5" />
	<action name="validate-all" timeout="30" />
	</actions>
	</resource-agent>
	END
	}

	#######################################################################

	DLM_SYSFS_DIR="/sys/kernel/dlm"

	controld_usage() {
	cat <<END
	usage: $0 {start\|stop\|monitor\|validate-all\|meta-data}

	Expects to have a fully populated OCF RA-compliant environment set.
	END
	}

	check_uncontrolled_locks()
	{
	local tmp
	tmp=$(ls $DLM_SYSFS_DIR 2>&1)
	if [ $? -eq 0 ]; then
	if [ -n "$tmp" ]; then

	ocf_log err "Uncontrolled lockspace exists, system must reboot. Executing suicide fencing"
	stonith_admin --reboot=$(crm_node -n) --tag controld

	exit $OCF_ERR_GENERIC
	fi
	fi
	}

	controld_start() {
	controld_monitor; rc=$?

	case $rc in
	$OCF_SUCCESS) return $OCF_SUCCESS;;
	$OCF_NOT_RUNNING) ;;
	*) return $OCF_ERR_GENERIC;;
	esac

	if [ ! -e $OCF_RESKEY_configdir ]; then
	modprobe configfs
	if [ ! -e $OCF_RESKEY_configdir ]; then
	ocf_log err "$OCF_RESKEY_configdir not available"
	- return $OCF_NOT_INSTALLED
	+ return $OCF_ERR_INSTALLED
	fi
	fi

	mount \| grep "type configfs" > /dev/null
	if [ $? != 0 ]; then
	mount -t configfs none $OCF_RESKEY_configdir
	fi

	if [ ! -e $OCF_RESKEY_configdir/dlm ]; then
	modprobe dlm
	if [ ! -e $OCF_RESKEY_configdir/dlm ]; then
	ocf_log err "$OCF_RESKEY_configdir/dlm not available"
	- return $OCF_NOT_INSTALLED
	+ return $OCF_ERR_INSTALLED
	fi
	fi

	if ! ocf_is_true "$OCF_RESKEY_allow_stonith_disabled" && \
	! ocf_is_true "`crm_attribute --type=crm_config --name=stonith-enabled --query --quiet --default=true`"; then
	ocf_log err "The cluster property stonith-enabled may not be deactivated to use the DLM"
	return $OCF_ERR_CONFIGURED
	fi

	${OCF_RESKEY_daemon} $OCF_RESKEY_args

	while true
	do
	sleep 1

	controld_monitor; rc=$?
	case $rc in
	$OCF_SUCCESS)
	local addr_list=$(cat /sys/kernel/config/dlm/cluster/comms/*/addr_list 2>/dev/null)
	if [ $? -eq 0 ] && [ -n "$addr_list" ]; then
	return $OCF_SUCCESS
	fi
	;;
	$OCF_NOT_RUNNING)
	return $OCF_NOT_RUNNING
	;;
	*)
	return $OCF_ERR_GENERIC
	;;
	esac

	ocf_log debug "Waiting for ${OCF_RESKEY_daemon} to be ready"
	done
	}

	controld_stop() {
	controld_monitor; rc=$?

	if [ $rc = $OCF_NOT_RUNNING ]; then
	return $OCF_SUCCESS
	fi

	killall -TERM ${OCF_RESKEY_daemon}; rc=$?

	if [ $rc != 0 ]; then
	return $OCF_ERR_GENERIC
	fi

	rc=$OCF_SUCCESS
	while [ $rc = $OCF_SUCCESS ]; do
	controld_monitor; rc=$?
	sleep 1
	done

	if [ $rc = $OCF_NOT_RUNNING ]; then
	rc=$OCF_SUCCESS
	fi

	return $rc
	}

	controld_monitor() {
	local rc
	killall -0 ${OCF_RESKEY_daemon} >/dev/null 2>&1 ; rc=$?

	case $rc in
	0) rc=$OCF_SUCCESS;;
	1) rc=$OCF_NOT_RUNNING;;
	*) rc=$OCF_ERR_GENERIC;;
	esac

	# if the dlm is not successfully running, but
	# dlm lockspace bits are left over, we self must fence.
	if [ $rc -ne $OCF_SUCCESS ]; then
	check_uncontrolled_locks
	fi

	return $rc
	}

	controld_validate() {
	check_binary killall
	check_binary ${OCF_RESKEY_daemon}

	case ${OCF_RESKEY_CRM_meta_globally_unique} in
	yes\|Yes\|true\|True\|1)
	ocf_log err "$OCF_RESOURCE_INSTANCE must be configured with the globally_unique=false meta attribute"
	exit $OCF_ERR_CONFIGURED
	;;
	esac

	[ -d /var/run/cluster ] \|\| mkdir /var/run/cluster

	return $OCF_SUCCESS
	}

	: ${OCF_RESKEY_sctp=false}
	: ${OCF_RESKEY_configdir=/sys/kernel/config}
	: ${OCF_RESKEY_CRM_meta_globally_unique:="false"}

	case "$HA_quorum_type" in
	pcmk) daemon_ext=".pcmk";;
	*) daemon_ext="";;
	esac

	case "$OCF_RESOURCE_INSTANCE" in
	[gG][fF][sS])
	: ${OCF_RESKEY_args=-g 0}
	: ${OCF_RESKEY_daemon=gfs_controld${daemon_ext}}
	;;
	[dD][lL][mM])
	: ${OCF_RESKEY_args=-s 0}
	: ${OCF_RESKEY_daemon=dlm_controld${daemon_ext}}
	;;
	*)
	: ${OCF_RESKEY_args=-s 0}
	: ${OCF_RESKEY_daemon=dlm_controld${daemon_ext}}
	esac

	case $__OCF_ACTION in
	meta-data) meta_data
	exit $OCF_SUCCESS
	;;
	start) controld_validate; controld_start;;
	stop) controld_stop;;
	monitor) controld_validate; controld_monitor;;
	validate-all) controld_validate;;
	usage\|help) controld_usage
	exit $OCF_SUCCESS
	;;
	*) controld_usage
	exit $OCF_ERR_UNIMPLEMENTED
	;;
	esac
	rc=$?

	exit $rc

	diff --git a/include/crm/common/util.h b/include/crm/common/util.h
	index 47f37bbf75..71821b5808 100644
	--- a/include/crm/common/util.h
	+++ b/include/crm/common/util.h
	@@ -1,136 +1,140 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/
	#ifndef CRM_COMMON_UTIL__H
	# define CRM_COMMON_UTIL__H

	/**
	* \file
	* \brief Utility functions
	* \ingroup core
	*/

	# include <sys/types.h>
	# include <stdlib.h>
	# include <limits.h>
	# include <signal.h>
	# include <sysexits.h>

	# include <crm/lrmd.h>

	# if SUPPORT_HEARTBEAT
	# include <heartbeat.h>
	# else
	# define NORMALNODE "normal"
	# define ACTIVESTATUS "active"/* fully functional, and all links are up */
	# define DEADSTATUS "dead"
	/* Status of non-working link or machine */
	# define PINGSTATUS "ping"
	/* Status of a working ping node */
	# define JOINSTATUS "join"
	/* Status when an api client joins */
	# define LEAVESTATUS "leave"
	/* Status when an api client leaves */
	# define ONLINESTATUS "online"/* Status of an online client */
	# define OFFLINESTATUS "offline"
	/* Status of an offline client */
	# endif

	char crm_itoa_stack(int an_int, char buf, size_t len);
	char *crm_itoa(int an_int);
	gboolean crm_is_true(const char *s);
	int crm_str_to_boolean(const char s, int ret);
	int crm_parse_int(const char text, const char default_text);
	long long crm_get_msec(const char *input);
	unsigned long long crm_get_interval(const char *input);
	int char2score(const char *score);
	char *score2char(int score);
	char score2char_stack(int score, char buf, size_t len);

	int compare_version(const char version1, const char version2);

	gboolean parse_op_key(const char key, char rsc_id, char op_type, int interval);
	gboolean decode_transition_key(const char key, char uuid, int action, int *transition_id,
	int *target_rc);
	gboolean decode_transition_magic(const char magic, char uuid, int transition_id, int *action_id,
	int op_status, int op_rc, int *target_rc);

	char * crm_strip_trailing_newline(char *str);

	# define safe_str_eq(a, b) crm_str_eq(a, b, FALSE)

	gboolean crm_str_eq(const char a, const char b, gboolean use_case);

	/* used with hash tables where case does not matter */
	static inline gboolean
	crm_strcase_equal(gconstpointer a, gconstpointer b)
	{
	return crm_str_eq((const char ) a, (const char ) b, FALSE);
	}

	gboolean safe_str_neq(const char a, const char b);

	# define crm_atoi(text, default_text) crm_parse_int(text, default_text)

	/* coverity[+kill] */
	void crm_abort(const char file, const char function, int line,
	const char *condition, gboolean do_core, gboolean do_fork);

	static inline gboolean
	is_not_set(long long word, long long bit)
	{
	return ((word & bit) == 0);
	}

	static inline gboolean
	is_set(long long word, long long bit)
	{
	return ((word & bit) == bit);
	}

	static inline gboolean
	is_set_any(long long word, long long bit)
	{
	return ((word & bit) != 0);
	}

	static inline guint
	crm_hash_table_size(GHashTable * hashtable)
	{
	if (hashtable == NULL) {
	return 0;
	}
	return g_hash_table_size(hashtable);
	}

	char crm_meta_name(const char field);
	const char crm_meta_value(GHashTable hash, const char *field);

	int rsc_op_expected_rc(lrmd_event_data_t * event);
	gboolean did_rsc_op_fail(lrmd_event_data_t * event, int target_rc);

	char crm_md5sum(const char buffer);

	char *crm_generate_uuid(void);

	void crm_build_path(const char *path_c, mode_t mode);
	int crm_user_lookup(const char name, uid_t uid, gid_t * gid);

	+#ifdef HAVE_GNUTLS_GNUTLS_H
	+void crm_gnutls_global_init(void);
	+#endif
	+
	int crm_exit(int rc);
	bool pcmk_acl_required(const char *user);

	#endif
	diff --git a/lib/cib/cib_remote.c b/lib/cib/cib_remote.c
	index 74e74c5502..d9dde9b185 100644
	--- a/lib/cib/cib_remote.c
	+++ b/lib/cib/cib_remote.c
	@@ -1,634 +1,634 @@
	/*
	* Copyright (c) 2008 Andrew Beekhof
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/
	#include <crm_internal.h>

	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <netdb.h>
	#include <termios.h>
	#include <sys/socket.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipcs.h>
	#include <crm/common/mainloop.h>

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# undef KEYFILE
	# include <gnutls/gnutls.h>
	gnutls_anon_client_credentials_t anon_cred_c;

	# define DEFAULT_CLIENT_HANDSHAKE_TIMEOUT 5000 /* 5 seconds */

	const int kx_prio[] = {
	GNUTLS_KX_ANON_DH,
	0
	};

	static gboolean remote_gnutls_credentials_init = FALSE;
	#else
	typedef void gnutls_session_t;
	#endif

	#include <arpa/inet.h>
	#ifndef ON_BSD
	# include <sgtty.h>
	#endif

	#define DH_BITS 1024

	typedef struct cib_remote_opaque_s {
	int flags;
	int socket;
	int port;
	char *server;
	char *user;
	char *passwd;
	gboolean encrypted;
	crm_remote_t command;
	crm_remote_t callback;

	} cib_remote_opaque_t;

	void cib_remote_connection_destroy(gpointer user_data);
	int cib_remote_callback_dispatch(gpointer user_data);
	int cib_remote_command_dispatch(gpointer user_data);
	int cib_remote_signon(cib_t * cib, const char *name, enum cib_conn_type type);
	int cib_remote_signoff(cib_t * cib);
	int cib_remote_free(cib_t * cib);

	int cib_remote_perform_op(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options,
	const char *name);

	static int
	cib_remote_inputfd(cib_t * cib)
	{
	cib_remote_opaque_t *private = cib->variant_opaque;

	return private->callback.tcp_socket;
	}

	static int
	cib_remote_set_connection_dnotify(cib_t * cib, void (*dnotify) (gpointer user_data))
	{
	return -EPROTONOSUPPORT;
	}

	static int
	cib_remote_register_notification(cib_t * cib, const char *callback, int enabled)
	{
	xmlNode *notify_msg = create_xml_node(NULL, "cib_command");
	cib_remote_opaque_t *private = cib->variant_opaque;

	crm_xml_add(notify_msg, F_CIB_OPERATION, T_CIB_NOTIFY);
	crm_xml_add(notify_msg, F_CIB_NOTIFY_TYPE, callback);
	crm_xml_add_int(notify_msg, F_CIB_NOTIFY_ACTIVATE, enabled);
	crm_remote_send(&private->callback, notify_msg);
	free_xml(notify_msg);
	return pcmk_ok;
	}

	cib_t *
	cib_remote_new(const char server, const char user, const char *passwd, int port,
	gboolean encrypted)
	{
	cib_remote_opaque_t *private = NULL;
	cib_t *cib = cib_new_variant();

	private = calloc(1, sizeof(cib_remote_opaque_t));

	cib->variant = cib_remote;
	cib->variant_opaque = private;

	if (server) {
	private->server = strdup(server);
	}

	if (user) {
	private->user = strdup(user);
	}

	if (passwd) {
	private->passwd = strdup(passwd);
	}

	private->port = port;
	private->encrypted = encrypted;

	/* assign variant specific ops */
	cib->delegate_fn = cib_remote_perform_op;
	cib->cmds->signon = cib_remote_signon;
	cib->cmds->signoff = cib_remote_signoff;
	cib->cmds->free = cib_remote_free;
	cib->cmds->inputfd = cib_remote_inputfd;

	cib->cmds->register_notification = cib_remote_register_notification;
	cib->cmds->set_connection_dnotify = cib_remote_set_connection_dnotify;

	return cib;
	}

	static int
	cib_tls_close(cib_t * cib)
	{
	cib_remote_opaque_t *private = cib->variant_opaque;

	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (private->encrypted) {
	if (private->command.tls_session) {
	gnutls_bye(*(private->command.tls_session), GNUTLS_SHUT_RDWR);
	gnutls_deinit(*(private->command.tls_session));
	gnutls_free(private->command.tls_session);
	}

	if (private->callback.tls_session) {
	gnutls_bye(*(private->callback.tls_session), GNUTLS_SHUT_RDWR);
	gnutls_deinit(*(private->callback.tls_session));
	gnutls_free(private->callback.tls_session);
	}
	private->command.tls_session = NULL;
	private->callback.tls_session = NULL;
	if (remote_gnutls_credentials_init) {
	gnutls_anon_free_client_credentials(anon_cred_c);
	gnutls_global_deinit();
	remote_gnutls_credentials_init = FALSE;
	}
	}
	#endif

	if (private->command.tcp_socket) {
	shutdown(private->command.tcp_socket, SHUT_RDWR); /* no more receptions */
	close(private->command.tcp_socket);
	}
	if (private->callback.tcp_socket) {
	shutdown(private->callback.tcp_socket, SHUT_RDWR); /* no more receptions */
	close(private->callback.tcp_socket);
	}
	private->command.tcp_socket = 0;
	private->callback.tcp_socket = 0;

	free(private->command.buffer);
	free(private->callback.buffer);
	private->command.buffer = NULL;
	private->callback.buffer = NULL;

	return 0;
	}

	static int
	cib_tls_signon(cib_t * cib, crm_remote_t * connection, gboolean event_channel)
	{
	int sock;
	cib_remote_opaque_t *private = cib->variant_opaque;
	int rc = 0;
	int disconnected = 0;

	xmlNode *answer = NULL;
	xmlNode *login = NULL;

	static struct mainloop_fd_callbacks cib_fd_callbacks = { 0, };

	cib_fd_callbacks.dispatch =
	event_channel ? cib_remote_callback_dispatch : cib_remote_command_dispatch;
	cib_fd_callbacks.destroy = cib_remote_connection_destroy;

	connection->tcp_socket = 0;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	connection->tls_session = NULL;
	#endif
	sock = crm_remote_tcp_connect(private->server, private->port);
	if (sock < 0) {
	crm_perror(LOG_ERR, "remote tcp connection to %s:%d failed", private->server,
	private->port);
	return -ENOTCONN;
	}

	if (private->encrypted) {
	/* initialize GnuTls lib */
	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (remote_gnutls_credentials_init == FALSE) {
	- gnutls_global_init();
	+ crm_gnutls_global_init();
	gnutls_anon_allocate_client_credentials(&anon_cred_c);
	remote_gnutls_credentials_init = TRUE;
	}

	/* bind the socket to GnuTls lib */
	connection->tls_session = crm_create_anon_tls_session(sock, GNUTLS_CLIENT, anon_cred_c);

	if (crm_initiate_client_tls_handshake(connection, DEFAULT_CLIENT_HANDSHAKE_TIMEOUT) != 0) {
	crm_err("Session creation for %s:%d failed", private->server, private->port);

	gnutls_deinit(*connection->tls_session);
	gnutls_free(connection->tls_session);
	connection->tls_session = NULL;
	cib_tls_close(cib);
	return -1;
	}
	#else
	return -EPROTONOSUPPORT;
	#endif
	} else {
	connection->tcp_socket = sock;
	}

	/* login to server */
	login = create_xml_node(NULL, "cib_command");
	crm_xml_add(login, "op", "authenticate");
	crm_xml_add(login, "user", private->user);
	crm_xml_add(login, "password", private->passwd);
	crm_xml_add(login, "hidden", "password");

	crm_remote_send(connection, login);
	free_xml(login);

	crm_remote_recv(connection, -1, &disconnected);

	if (disconnected) {
	rc = -ENOTCONN;
	}

	answer = crm_remote_parse_buffer(connection);

	crm_log_xml_trace(answer, "Reply");
	if (answer == NULL) {
	rc = -EPROTO;

	} else {
	/* grab the token */
	const char *msg_type = crm_element_value(answer, F_CIB_OPERATION);
	const char *tmp_ticket = crm_element_value(answer, F_CIB_CLIENTID);

	if (safe_str_neq(msg_type, CRM_OP_REGISTER)) {
	crm_err("Invalid registration message: %s", msg_type);
	rc = -EPROTO;

	} else if (tmp_ticket == NULL) {
	rc = -EPROTO;

	} else {
	connection->token = strdup(tmp_ticket);
	}
	}
	free_xml(answer);
	answer = NULL;

	if (rc != 0) {
	cib_tls_close(cib);
	return rc;
	}

	crm_trace("remote client connection established");
	connection->source =
	mainloop_add_fd("cib-remote", G_PRIORITY_HIGH, sock, cib,
	&cib_fd_callbacks);
	return rc;
	}

	void
	cib_remote_connection_destroy(gpointer user_data)
	{
	crm_err("Connection destroyed");
	#ifdef HAVE_GNUTLS_GNUTLS_H
	cib_tls_close(user_data);
	#endif
	return;
	}

	int
	cib_remote_command_dispatch(gpointer user_data)
	{
	int disconnected = 0;
	cib_t *cib = user_data;
	cib_remote_opaque_t *private = cib->variant_opaque;

	crm_remote_recv(&private->command, -1, &disconnected);

	free(private->command.buffer);
	private->command.buffer = NULL;
	crm_err("received late reply for remote cib connection, discarding");

	if (disconnected) {
	return -1;
	}
	return 0;
	}

	int
	cib_remote_callback_dispatch(gpointer user_data)
	{
	cib_t *cib = user_data;
	cib_remote_opaque_t *private = cib->variant_opaque;

	xmlNode *msg = NULL;
	int disconnected = 0;

	crm_info("Message on callback channel");

	crm_remote_recv(&private->callback, -1, &disconnected);

	msg = crm_remote_parse_buffer(&private->callback);
	while (msg) {
	const char *type = crm_element_value(msg, F_TYPE);

	crm_trace("Activating %s callbacks...", type);

	if (safe_str_eq(type, T_CIB)) {
	cib_native_callback(cib, msg, 0, 0);

	} else if (safe_str_eq(type, T_CIB_NOTIFY)) {
	g_list_foreach(cib->notify_list, cib_native_notify, msg);

	} else {
	crm_err("Unknown message type: %s", type);
	}

	free_xml(msg);
	msg = crm_remote_parse_buffer(&private->callback);
	}

	if (disconnected) {
	return -1;
	}

	return 0;
	}

	int
	cib_remote_signon(cib_t * cib, const char *name, enum cib_conn_type type)
	{
	int rc = pcmk_ok;
	cib_remote_opaque_t *private = cib->variant_opaque;

	if (private->passwd == NULL) {
	struct termios settings;

	rc = tcgetattr(0, &settings);
	if(rc == 0) {
	settings.c_lflag &= ~ECHO;
	rc = tcsetattr(0, TCSANOW, &settings);
	}

	if(rc == 0) {
	fprintf(stderr, "Password: ");
	private->passwd = calloc(1, 1024);
	rc = scanf("%s", private->passwd);
	fprintf(stdout, "\n");
	}

	/* fprintf(stderr, "entered: '%s'\n", buffer); */
	if (rc < 1) {
	private->passwd = NULL;
	}

	settings.c_lflag \|= ECHO;
	rc = tcsetattr(0, TCSANOW, &settings);
	}

	if (private->server == NULL \|\| private->user == NULL) {
	rc = -EINVAL;
	}

	if (rc == pcmk_ok) {
	rc = cib_tls_signon(cib, &(private->command), FALSE);
	}

	if (rc == pcmk_ok) {
	rc = cib_tls_signon(cib, &(private->callback), TRUE);
	}

	if (rc == pcmk_ok) {
	xmlNode *hello =
	cib_create_op(0, private->callback.token, CRM_OP_REGISTER, NULL, NULL, NULL, 0, NULL);
	crm_xml_add(hello, F_CIB_CLIENTNAME, name);
	crm_remote_send(&private->command, hello);
	free_xml(hello);
	}

	if (rc == pcmk_ok) {
	crm_notice("%s: Opened connection to %s:%d\n", name, private->server, private->port);
	cib->state = cib_connected_command;
	cib->type = cib_command;

	} else {
	fprintf(stderr, "%s: Connection to %s:%d failed: %s\n",
	name, private->server, private->port, pcmk_strerror(rc));
	}

	return rc;
	}

	int
	cib_remote_signoff(cib_t * cib)
	{
	int rc = pcmk_ok;

	/* cib_remote_opaque_t private = cib->variant_opaque; /

	crm_debug("Signing out of the CIB Service");
	#ifdef HAVE_GNUTLS_GNUTLS_H
	cib_tls_close(cib);
	#endif

	cib->state = cib_disconnected;
	cib->type = cib_no_connection;

	return rc;
	}

	int
	cib_remote_free(cib_t * cib)
	{
	int rc = pcmk_ok;

	crm_warn("Freeing CIB");
	if (cib->state != cib_disconnected) {
	rc = cib_remote_signoff(cib);
	if (rc == pcmk_ok) {
	cib_remote_opaque_t *private = cib->variant_opaque;

	free(private->server);
	free(private->user);
	free(private->passwd);
	free(cib->cmds);
	free(private);
	free(cib);
	}
	}

	return rc;
	}

	int
	cib_remote_perform_op(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options, const char *name)
	{
	int rc = pcmk_ok;
	int disconnected = 0;
	int remaining_time = 0;
	time_t start_time;

	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	cib_remote_opaque_t *private = cib->variant_opaque;

	if (cib->state == cib_disconnected) {
	return -ENOTCONN;
	}

	if (output_data != NULL) {
	*output_data = NULL;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	cib->call_id++;
	/* prevent call_id from being negative (or zero) and conflicting
	* with the cib_errors enum
	* use 2 because we use it as (cib->call_id - 1) below
	*/
	if (cib->call_id < 1) {
	cib->call_id = 1;
	}

	op_msg =
	cib_create_op(cib->call_id, private->callback.token, op, host, section, data, call_options,
	NULL);
	if (op_msg == NULL) {
	return -EPROTO;
	}

	crm_trace("Sending %s message to CIB service", op);
	if (!(call_options & cib_sync_call)) {
	crm_remote_send(&private->callback, op_msg);
	} else {
	crm_remote_send(&private->command, op_msg);
	}
	free_xml(op_msg);

	if ((call_options & cib_discard_reply)) {
	crm_trace("Discarding reply");
	return pcmk_ok;

	} else if (!(call_options & cib_sync_call)) {
	return cib->call_id;
	}

	crm_trace("Waiting for a syncronous reply");

	start_time = time(NULL);
	remaining_time = cib->call_timeout ? cib->call_timeout : 60;

	while (remaining_time > 0 && !disconnected) {
	int reply_id = -1;
	int msg_id = cib->call_id;

	crm_remote_recv(&private->command, remaining_time * 1000, &disconnected);
	op_reply = crm_remote_parse_buffer(&private->command);

	if (!op_reply) {
	break;
	}

	crm_element_value_int(op_reply, F_CIB_CALLID, &reply_id);

	if (reply_id == msg_id) {
	break;

	} else if (reply_id < msg_id) {
	crm_debug("Received old reply: %d (wanted %d)", reply_id, msg_id);
	crm_log_xml_trace(op_reply, "Old reply");

	} else if ((reply_id - 10000) > msg_id) {
	/* wrap-around case */
	crm_debug("Received old reply: %d (wanted %d)", reply_id, msg_id);
	crm_log_xml_trace(op_reply, "Old reply");
	} else {
	crm_err("Received a __future__ reply:" " %d (wanted %d)", reply_id, msg_id);
	}

	free_xml(op_reply);
	op_reply = NULL;

	/* wasn't the right reply, try and read some more */
	remaining_time = time(NULL) - start_time;
	}

	/* if(IPC_ISRCONN(native->command_channel) == FALSE) { */
	/* crm_err("CIB disconnected: %d", */
	/* native->command_channel->ch_status); */
	/* cib->state = cib_disconnected; */
	/* } */

	if (disconnected) {
	crm_err("Disconnected while waiting for reply.");
	return -ENOTCONN;
	} else if (op_reply == NULL) {
	crm_err("No reply message - empty");
	return -ENOMSG;
	}

	crm_trace("Syncronous reply received");

	/* Start processing the reply... */
	if (crm_element_value_int(op_reply, F_CIB_RC, &rc) != 0) {
	rc = -EPROTO;
	}

	if (rc == -pcmk_err_diff_resync) {
	/* This is an internal value that clients do not and should not care about */
	rc = pcmk_ok;
	}

	if (rc == pcmk_ok \|\| rc == -EPERM) {
	crm_log_xml_debug(op_reply, "passed");

	} else {
	/* } else if(rc == -ETIME) { */
	crm_err("Call failed: %s", pcmk_strerror(rc));
	crm_log_xml_warn(op_reply, "failed");
	}

	if (output_data == NULL) {
	/* do nothing more */

	} else if (!(call_options & cib_discard_reply)) {
	xmlNode *tmp = get_message_xml(op_reply, F_CIB_CALLDATA);

	if (tmp == NULL) {
	crm_trace("No output in reply to \"%s\" command %d", op, cib->call_id - 1);
	} else {
	*output_data = copy_xml(tmp);
	}
	}

	free_xml(op_reply);

	return rc;
	}
	diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c
	index 5bad02fc30..022a8ab2fc 100644
	--- a/lib/cib/cib_utils.c
	+++ b/lib/cib/cib_utils.c
	@@ -1,836 +1,836 @@
	/*
	* Copyright (c) 2004 International Business Machines
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/
	#include <crm_internal.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <sys/utsname.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/pengine/rules.h>

	struct config_root_s {
	const char *name;
	const char *parent;
	const char *path;
	};

	/*
	* "//crm_config" will also work in place of "/cib/configuration/crm_config"
	* The / prefix means find starting from the root, whereas the // prefix means
	* find anywhere and risks multiple matches
	*/
	/* INDENT-OFF */
	struct config_root_s known_paths[] = {
	{ NULL, NULL, "//cib" },
	{ XML_TAG_CIB, NULL, "//cib" },
	{ XML_CIB_TAG_STATUS, "/cib", "//cib/status" },
	{ XML_CIB_TAG_CONFIGURATION,"/cib", "//cib/configuration" },
	{ XML_CIB_TAG_CRMCONFIG, "/cib/configuration", "//cib/configuration/crm_config" },
	{ XML_CIB_TAG_NODES, "/cib/configuration", "//cib/configuration/nodes" },
	{ XML_CIB_TAG_DOMAINS, "/cib/configuration", "//cib/configuration/domains" },
	{ XML_CIB_TAG_RESOURCES, "/cib/configuration", "//cib/configuration/resources" },
	{ XML_CIB_TAG_CONSTRAINTS, "/cib/configuration", "//cib/configuration/constraints" },
	{ XML_CIB_TAG_OPCONFIG, "/cib/configuration", "//cib/configuration/op_defaults" },
	{ XML_CIB_TAG_RSCCONFIG, "/cib/configuration", "//cib/configuration/rsc_defaults" },
	{ XML_CIB_TAG_ACLS, "/cib/configuration", "//cib/configuration/acls" },
	{ XML_TAG_FENCING_TOPOLOGY, "/cib/configuration", "//cib/configuration/fencing-topology" },
	{ XML_CIB_TAG_SECTION_ALL, NULL, "//cib" },
	};
	/* INDENT-ON */

	int
	cib_compare_generation(xmlNode * left, xmlNode * right)
	{
	int lpc = 0;

	const char *attributes[] = {
	XML_ATTR_GENERATION_ADMIN,
	XML_ATTR_GENERATION,
	XML_ATTR_NUMUPDATES,
	};

	crm_log_xml_trace(left, "left");
	crm_log_xml_trace(right, "right");

	for (lpc = 0; lpc < DIMOF(attributes); lpc++) {
	int int_elem_l = -1;
	int int_elem_r = -1;
	const char *elem_r = NULL;
	const char *elem_l = crm_element_value(left, attributes[lpc]);

	if (right != NULL) {
	elem_r = crm_element_value(right, attributes[lpc]);
	}

	if (elem_l != NULL) {
	int_elem_l = crm_parse_int(elem_l, NULL);
	}
	if (elem_r != NULL) {
	int_elem_r = crm_parse_int(elem_r, NULL);
	}

	if (int_elem_l < int_elem_r) {
	crm_trace("%s (%s < %s)", attributes[lpc], crm_str(elem_l), crm_str(elem_r));
	return -1;

	} else if (int_elem_l > int_elem_r) {
	crm_trace("%s (%s > %s)", attributes[lpc], crm_str(elem_l), crm_str(elem_r));
	return 1;
	}
	}

	return 0;
	}

	/* Deprecated - doesn't expose -EACCES */
	xmlNode *
	get_cib_copy(cib_t * cib)
	{
	xmlNode *xml_cib;
	int options = cib_scope_local \| cib_sync_call;
	int rc = cib->cmds->query(cib, NULL, &xml_cib, options);

	if (rc == -EACCES) {
	return NULL;

	} else if (rc != pcmk_ok) {
	crm_err("Couldnt retrieve the CIB");
	free_xml(xml_cib);
	return NULL;

	} else if (xml_cib == NULL) {
	crm_err("The CIB result was empty");
	free_xml(xml_cib);
	return NULL;
	}

	if (safe_str_eq(crm_element_name(xml_cib), XML_TAG_CIB)) {
	return xml_cib;
	}
	free_xml(xml_cib);
	return NULL;
	}

	xmlNode *
	cib_get_generation(cib_t * cib)
	{
	xmlNode *the_cib = NULL;
	xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);

	cib->cmds->query(cib, NULL, &the_cib, cib_scope_local \| cib_sync_call);
	if (the_cib != NULL) {
	copy_in_properties(generation, the_cib);
	free_xml(the_cib);
	}

	return generation;
	}

	gboolean
	cib_version_details(xmlNode * cib, int admin_epoch, int epoch, int *updates)
	{
	*epoch = -1;
	*updates = -1;
	*admin_epoch = -1;

	if (cib == NULL) {
	return FALSE;

	} else {
	crm_element_value_int(cib, XML_ATTR_GENERATION, epoch);
	crm_element_value_int(cib, XML_ATTR_NUMUPDATES, updates);
	crm_element_value_int(cib, XML_ATTR_GENERATION_ADMIN, admin_epoch);
	}
	return TRUE;
	}

	gboolean
	cib_diff_version_details(xmlNode * diff, int admin_epoch, int epoch, int *updates,
	int _admin_epoch, int _epoch, int *_updates)
	{
	int add[] = { 0, 0, 0 };
	int del[] = { 0, 0, 0 };

	xml_patch_versions(diff, add, del);

	*admin_epoch = add[0];
	*epoch = add[1];
	*updates = add[2];

	*_admin_epoch = del[0];
	*_epoch = del[1];
	*_updates = del[2];

	return TRUE;
	}

	/*
	* The caller should never free the return value
	*/

	const char *
	get_object_path(const char *object_type)
	{
	int lpc = 0;
	int max = DIMOF(known_paths);

	for (; lpc < max; lpc++) {
	if ((object_type == NULL && known_paths[lpc].name == NULL)
	\|\| safe_str_eq(object_type, known_paths[lpc].name)) {
	return known_paths[lpc].path;
	}
	}
	return NULL;
	}

	const char *
	get_object_parent(const char *object_type)
	{
	int lpc = 0;
	int max = DIMOF(known_paths);

	for (; lpc < max; lpc++) {
	if (safe_str_eq(object_type, known_paths[lpc].name)) {
	return known_paths[lpc].parent;
	}
	}
	return NULL;
	}

	xmlNode *
	get_object_root(const char object_type, xmlNode the_root)
	{
	const char *xpath = get_object_path(object_type);

	if (xpath == NULL) {
	return the_root; /* or return NULL? */
	}

	return get_xpath_object(xpath, the_root, LOG_DEBUG_4);
	}

	/*
	* It is the callers responsibility to free both the new CIB (output)
	* and the new CIB (input)
	*/
	xmlNode *
	createEmptyCib(int admin_epoch)
	{
	xmlNode cib_root = NULL, config = NULL;

	cib_root = create_xml_node(NULL, XML_TAG_CIB);
	crm_xml_add(cib_root, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	crm_xml_add(cib_root, XML_ATTR_VALIDATION, xml_latest_schema());

	crm_xml_add_int(cib_root, XML_ATTR_GENERATION, admin_epoch);
	crm_xml_add_int(cib_root, XML_ATTR_NUMUPDATES, 0);
	crm_xml_add_int(cib_root, XML_ATTR_GENERATION_ADMIN, 0);

	config = create_xml_node(cib_root, XML_CIB_TAG_CONFIGURATION);
	create_xml_node(cib_root, XML_CIB_TAG_STATUS);

	create_xml_node(config, XML_CIB_TAG_CRMCONFIG);
	create_xml_node(config, XML_CIB_TAG_NODES);
	create_xml_node(config, XML_CIB_TAG_RESOURCES);
	create_xml_node(config, XML_CIB_TAG_CONSTRAINTS);

	return cib_root;
	}

	static bool
	cib_acl_enabled(xmlNode xml, const char user)
	{
	bool rc = FALSE;

	#if ENABLE_ACL
	if(pcmk_acl_required(user)) {
	const char *value = NULL;
	GHashTable *options = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	cib_read_config(options, xml);
	value = cib_pref(options, "enable-acl");
	rc = crm_is_true(value);
	g_hash_table_destroy(options);
	}

	crm_debug("CIB ACL is %s", rc ? "enabled" : "disabled");
	#endif
	return rc;
	}

	int
	cib_perform_op(const char op, int call_options, cib_op_t fn, gboolean is_query,
	const char section, xmlNode req, xmlNode * input,
	gboolean manage_counters, gboolean * config_changed,
	xmlNode * current_cib, xmlNode result_cib, xmlNode diff, xmlNode ** output)
	{
	int rc = pcmk_ok;
	gboolean check_dtd = TRUE;
	xmlNode *top = NULL;
	xmlNode *scratch = NULL;
	xmlNode *local_diff = NULL;

	const char *new_version = NULL;
	static struct qb_log_callsite *diff_cs = NULL;
	const char *user = crm_element_value(req, F_CIB_USER);

	crm_trace("Begin %s%s op", is_query ? "read-only " : "", op);

	CRM_CHECK(output != NULL, return -ENOMSG);
	CRM_CHECK(result_cib != NULL, return -ENOMSG);
	CRM_CHECK(config_changed != NULL, return -ENOMSG);

	if(output) {
	*output = NULL;
	}

	*result_cib = NULL;
	*config_changed = FALSE;

	if (fn == NULL) {
	return -EINVAL;
	}

	if (is_query) {
	xmlNode *cib_ro = current_cib;
	xmlNode *cib_filtered = NULL;

	if(cib_acl_enabled(cib_ro, user)) {
	if(xml_acl_filtered_copy(user, current_cib, current_cib, &cib_filtered)) {
	if (cib_filtered == NULL) {
	crm_debug("Pre-filtered the entire cib");
	return -EACCES;
	}
	cib_ro = cib_filtered;
	crm_log_xml_trace(cib_ro, "filtered");
	}
	}

	rc = (*fn) (op, call_options, section, req, input, cib_ro, result_cib, output);

	if(output == NULL \|\| *output == NULL) {
	/* nothing */

	} else if(cib_filtered == *output) {
	cib_filtered = NULL; /* Let them have this copy */

	} else if(*output == current_cib) {
	/* They already know not to free it */

	} else if(cib_filtered && (*output)->doc == cib_filtered->doc) {
	/* We're about to free the document of which output is a part /
	output = copy_xml(output);

	} else if((*output)->doc == current_cib->doc) {
	/* Give them a copy they can free */
	output = copy_xml(output);
	}

	free_xml(cib_filtered);
	return rc;
	}


	if (is_set(call_options, cib_zero_copy)) {
	/* Conditional on v2 patch style */

	scratch = current_cib;

	/* Create a shallow copy of current_cib for the version details */
	current_cib = create_xml_node(NULL, (const char *)scratch->name);
	copy_in_properties(current_cib, scratch);
	top = current_cib;

	xml_track_changes(scratch, user, NULL, cib_acl_enabled(scratch, user));
	rc = (*fn) (op, call_options, section, req, input, scratch, &scratch, output);

	} else {
	scratch = copy_xml(current_cib);
	xml_track_changes(scratch, user, NULL, cib_acl_enabled(scratch, user));
	rc = (*fn) (op, call_options, section, req, input, current_cib, &scratch, output);

	- if(xml_tracking_changes(scratch) == FALSE) {
	+ if(scratch && xml_tracking_changes(scratch) == FALSE) {
	crm_trace("Inferring changes after %s op", op);
	xml_track_changes(scratch, user, current_cib, cib_acl_enabled(current_cib, user));
	xml_calculate_changes(current_cib, scratch);
	}
	CRM_CHECK(current_cib != scratch, return -EINVAL);
	}

	xml_acl_disable(scratch); /* Allow the system to make any additional changes */

	if (rc == pcmk_ok && scratch == NULL) {
	rc = -EINVAL;
	goto done;

	} else if(rc == pcmk_ok && xml_acl_denied(scratch)) {
	crm_trace("ACL rejected part or all of the proposed changes");
	rc = -EACCES;
	goto done;

	} else if (rc != pcmk_ok) {
	goto done;
	}

	if (scratch) {
	new_version = crm_element_value(scratch, XML_ATTR_CRM_VERSION);

	if (new_version && compare_version(new_version, CRM_FEATURE_SET) > 0) {
	crm_err("Discarding update with feature set '%s' greater than our own '%s'",
	new_version, CRM_FEATURE_SET);
	rc = -EPROTONOSUPPORT;
	goto done;
	}
	}

	if (current_cib) {
	int old = 0;
	int new = 0;

	crm_element_value_int(scratch, XML_ATTR_GENERATION_ADMIN, &new);
	crm_element_value_int(current_cib, XML_ATTR_GENERATION_ADMIN, &old);

	if (old > new) {
	crm_err("%s went backwards: %d -> %d (Opts: 0x%x)",
	XML_ATTR_GENERATION_ADMIN, old, new, call_options);
	crm_log_xml_warn(req, "Bad Op");
	crm_log_xml_warn(input, "Bad Data");
	rc = -pcmk_err_old_data;

	} else if (old == new) {
	crm_element_value_int(scratch, XML_ATTR_GENERATION, &new);
	crm_element_value_int(current_cib, XML_ATTR_GENERATION, &old);
	if (old > new) {
	crm_err("%s went backwards: %d -> %d (Opts: 0x%x)",
	XML_ATTR_GENERATION, old, new, call_options);
	crm_log_xml_warn(req, "Bad Op");
	crm_log_xml_warn(input, "Bad Data");
	rc = -pcmk_err_old_data;
	}
	}
	}

	crm_trace("Massaging CIB contents");
	strip_text_nodes(scratch);
	fix_plus_plus_recursive(scratch);

	if (is_set(call_options, cib_zero_copy)) {
	/* At this point, current_cib is just the 'cib' tag and its properties,
	*
	* The v1 format would barf on this, but we know the v2 patch
	* format only needs it for the top-level version fields
	*/
	local_diff = xml_create_patchset(2, current_cib, scratch, (bool*)config_changed, manage_counters, FALSE);

	} else {
	static time_t expires = 0;
	time_t tm_now = time(NULL);
	bool with_digest = FALSE;

	if (expires < tm_now) {
	expires = tm_now + 60; /* Validate clients are correctly applying v2-style diffs at most once a minute */
	with_digest = TRUE;
	}

	local_diff = xml_create_patchset(0, current_cib, scratch, (bool*)config_changed, manage_counters, with_digest);
	}

	if (diff_cs == NULL) {
	diff_cs = qb_log_callsite_get(__PRETTY_FUNCTION__, __FILE__, "diff-validation", LOG_DEBUG, __LINE__, crm_trace_nonlog);
	}

	if(local_diff) {
	xml_log_patchset(LOG_INFO, __FUNCTION__, local_diff);
	crm_log_xml_trace(local_diff, "raw patch");
	}

	if (is_not_set(call_options, cib_zero_copy) /* The original to compare against doesn't exist */
	&& local_diff
	&& crm_is_callsite_active(diff_cs, LOG_TRACE, 0)) {

	/* Validate the calculated patch set */
	int test_rc, format = 1;
	xmlNode * c = copy_xml(current_cib);

	crm_element_value_int(local_diff, "format", &format);
	test_rc = xml_apply_patchset(c, local_diff, manage_counters);

	if(test_rc != pcmk_ok) {
	save_xml_to_file(c, "PatchApply:calculated", NULL);
	save_xml_to_file(current_cib, "PatchApply:input", NULL);
	save_xml_to_file(scratch, "PatchApply:actual", NULL);
	save_xml_to_file(local_diff, "PatchApply:diff", NULL);
	crm_err("v%d patchset error, patch failed to apply: %s (%d)", format, pcmk_strerror(test_rc), test_rc);
	}
	free_xml(c);
	}

	xml_log_changes(LOG_TRACE, __FUNCTION__, scratch);
	xml_accept_changes(scratch);

	if (safe_str_eq(section, XML_CIB_TAG_STATUS)) {
	/* Throttle the amount of costly validation we perform due to status updates
	* a) we don't really care whats in the status section
	* b) we don't validate any of it's contents at the moment anyway
	*/
	check_dtd = FALSE;
	}

	/* === scratch must not be modified after this point ===
	* Exceptions, anything in:

	static filter_t filter[] = {
	{ 0, XML_ATTR_ORIGIN },
	{ 0, XML_CIB_ATTR_WRITTEN },
	{ 0, XML_ATTR_UPDATE_ORIG },
	{ 0, XML_ATTR_UPDATE_CLIENT },
	{ 0, XML_ATTR_UPDATE_USER },
	};
	*/

	if (*config_changed && is_not_set(call_options, cib_no_mtime)) {
	char *now_str = NULL;
	time_t now = time(NULL);
	const char *schema = crm_element_value(scratch, XML_ATTR_VALIDATION);

	now_str = ctime(&now);
	now_str[24] = EOS; /* replace the newline */
	crm_xml_replace(scratch, XML_CIB_ATTR_WRITTEN, now_str);

	if (schema) {
	static int minimum_schema = 0;
	int current_schema = get_schema_version(schema);

	if (minimum_schema == 0) {
	minimum_schema = get_schema_version("pacemaker-1.1");
	}

	/* Does the CIB support the "update-" attributes... /
	if (current_schema >= minimum_schema) {
	const char *origin = crm_element_value(req, F_ORIG);

	CRM_LOG_ASSERT(origin != NULL);
	crm_xml_replace(scratch, XML_ATTR_UPDATE_ORIG, origin);
	crm_xml_replace(scratch, XML_ATTR_UPDATE_CLIENT,
	crm_element_value(req, F_CIB_CLIENTNAME));
	#if ENABLE_ACL
	crm_xml_replace(scratch, XML_ATTR_UPDATE_USER, crm_element_value(req, F_CIB_USER));
	#endif
	}
	}
	}

	crm_trace("Perform validation: %s", check_dtd ? "true" : "false");
	if (rc == pcmk_ok && check_dtd && validate_xml(scratch, NULL, TRUE) == FALSE) {
	const char *current_dtd = crm_element_value(scratch, XML_ATTR_VALIDATION);

	crm_warn("Updated CIB does not validate against %s schema/dtd", crm_str(current_dtd));
	rc = -pcmk_err_schema_validation;
	}

	done:

	*result_cib = scratch;
	#if ENABLE_ACL
	if(rc != pcmk_ok && cib_acl_enabled(current_cib, user)) {
	if(xml_acl_filtered_copy(user, current_cib, scratch, result_cib)) {
	if (*result_cib == NULL) {
	crm_debug("Pre-filtered the entire cib result");
	}
	free_xml(scratch);
	}
	}
	#endif

	if(diff) {
	*diff = local_diff;
	} else {
	free_xml(local_diff);
	}

	free_xml(top);
	crm_trace("Done");
	return rc;
	}

	xmlNode *
	cib_create_op(int call_id, const char token, const char op, const char host, const char section,
	xmlNode * data, int call_options, const char *user_name)
	{
	xmlNode *op_msg = create_xml_node(NULL, "cib_command");

	CRM_CHECK(op_msg != NULL, return NULL);
	CRM_CHECK(token != NULL, return NULL);

	crm_xml_add(op_msg, F_XML_TAGNAME, "cib_command");

	crm_xml_add(op_msg, F_TYPE, T_CIB);
	crm_xml_add(op_msg, F_CIB_CALLBACK_TOKEN, token);
	crm_xml_add(op_msg, F_CIB_OPERATION, op);
	crm_xml_add(op_msg, F_CIB_HOST, host);
	crm_xml_add(op_msg, F_CIB_SECTION, section);
	crm_xml_add_int(op_msg, F_CIB_CALLID, call_id);
	#if ENABLE_ACL
	if (user_name) {
	crm_xml_add(op_msg, F_CIB_USER, user_name);
	}
	#endif
	crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options);
	crm_xml_add_int(op_msg, F_CIB_CALLOPTS, call_options);

	if (data != NULL) {
	add_message_xml(op_msg, F_CIB_CALLDATA, data);
	}

	if (call_options & cib_inhibit_bcast) {
	CRM_CHECK((call_options & cib_scope_local), return NULL);
	}
	return op_msg;
	}

	void
	cib_native_callback(cib_t * cib, xmlNode * msg, int call_id, int rc)
	{
	xmlNode *output = NULL;
	cib_callback_client_t *blob = NULL;
	cib_callback_client_t local_blob;

	local_blob.id = NULL;
	local_blob.callback = NULL;
	local_blob.user_data = NULL;
	local_blob.only_success = FALSE;

	if (msg != NULL) {
	crm_element_value_int(msg, F_CIB_RC, &rc);
	crm_element_value_int(msg, F_CIB_CALLID, &call_id);
	output = get_message_xml(msg, F_CIB_CALLDATA);
	}

	blob = g_hash_table_lookup(cib_op_callback_table, GINT_TO_POINTER(call_id));

	if (blob != NULL) {
	local_blob = *blob;
	blob = NULL;

	remove_cib_op_callback(call_id, FALSE);

	} else {
	crm_trace("No callback found for call %d", call_id);
	local_blob.callback = NULL;
	}

	if (cib == NULL) {
	crm_debug("No cib object supplied");
	}

	if (rc == -pcmk_err_diff_resync) {
	/* This is an internal value that clients do not and should not care about */
	rc = pcmk_ok;
	}

	if (local_blob.callback != NULL && (rc == pcmk_ok \|\| local_blob.only_success == FALSE)) {
	crm_trace("Invoking callback %s for call %d", crm_str(local_blob.id), call_id);
	local_blob.callback(msg, call_id, rc, output, local_blob.user_data);

	} else if (cib && cib->op_callback == NULL && rc != pcmk_ok) {
	crm_warn("CIB command failed: %s", pcmk_strerror(rc));
	crm_log_xml_debug(msg, "Failed CIB Update");
	}

	if (cib && cib->op_callback != NULL) {
	crm_trace("Invoking global callback for call %d", call_id);
	cib->op_callback(msg, call_id, rc, output);
	}
	crm_trace("OP callback activated.");
	}

	void
	cib_native_notify(gpointer data, gpointer user_data)
	{
	xmlNode *msg = user_data;
	cib_notify_client_t *entry = data;
	const char *event = NULL;

	if (msg == NULL) {
	crm_warn("Skipping callback - NULL message");
	return;
	}

	event = crm_element_value(msg, F_SUBTYPE);

	if (entry == NULL) {
	crm_warn("Skipping callback - NULL callback client");
	return;

	} else if (entry->callback == NULL) {
	crm_warn("Skipping callback - NULL callback");
	return;

	} else if (safe_str_neq(entry->event, event)) {
	crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event);
	return;
	}

	crm_trace("Invoking callback for %p/%s event...", entry, event);
	entry->callback(event, msg);
	crm_trace("Callback invoked...");
	}

	pe_cluster_option cib_opts[] = {
	/* name, old-name, validate, default, description */
	{"enable-acl", NULL, "boolean", NULL, "false", &check_boolean,
	"Enable CIB ACL", NULL}
	,
	};

	void
	cib_metadata(void)
	{
	config_metadata("Cluster Information Base", "1.0",
	"Cluster Information Base Options",
	"This is a fake resource that details the options that can be configured for the Cluster Information Base.",
	cib_opts, DIMOF(cib_opts));
	}

	void
	verify_cib_options(GHashTable * options)
	{
	verify_all_options(options, cib_opts, DIMOF(cib_opts));
	}

	const char *
	cib_pref(GHashTable * options, const char *name)
	{
	return get_cluster_pref(options, cib_opts, DIMOF(cib_opts), name);
	}

	gboolean
	cib_read_config(GHashTable * options, xmlNode * current_cib)
	{
	xmlNode *config = NULL;
	crm_time_t *now = NULL;

	if (options == NULL \|\| current_cib == NULL) {
	return FALSE;
	}

	now = crm_time_new(NULL);

	g_hash_table_remove_all(options);

	config = get_object_root(XML_CIB_TAG_CRMCONFIG, current_cib);
	if (config) {
	unpack_instance_attributes(current_cib, config, XML_CIB_TAG_PROPSET, NULL, options,
	CIB_OPTIONS_FIRST, FALSE, now);
	}

	verify_cib_options(options);

	crm_time_free(now);

	return TRUE;
	}

	int
	cib_apply_patch_event(xmlNode * event, xmlNode * input, xmlNode ** output, int level)
	{
	int rc = pcmk_err_generic;

	xmlNode *diff = NULL;

	CRM_ASSERT(event);
	CRM_ASSERT(input);
	CRM_ASSERT(output);

	crm_element_value_int(event, F_CIB_RC, &rc);
	diff = get_message_xml(event, F_CIB_UPDATE_RESULT);

	if (rc < pcmk_ok \|\| diff == NULL) {
	return rc;
	}

	if (level > LOG_CRIT) {
	xml_log_patchset(level, "Config update", diff);
	}

	if (input != NULL) {
	rc = cib_process_diff(NULL, cib_none, NULL, event, diff, input, output, NULL);

	if (rc != pcmk_ok) {
	crm_debug("Update didn't apply: %s (%d) %p", pcmk_strerror(rc), rc, *output);

	if (rc == -pcmk_err_old_data) {
	crm_trace("Masking error, we already have the supplied update");
	return pcmk_ok;
	}
	free_xml(output); output = NULL;

	return rc;
	}
	}

	return rc;
	}

	gboolean
	cib_internal_config_changed(xmlNode * diff)
	{
	gboolean changed = FALSE;
	xmlXPathObject *xpathObj = NULL;

	if (diff == NULL) {
	return FALSE;
	}

	xpathObj = xpath_search(diff, "//" XML_CIB_TAG_CRMCONFIG);
	if (numXpathResults(xpathObj) > 0) {
	changed = TRUE;
	}

	freeXpathObject(xpathObj);

	return changed;
	}

	int
	cib_internal_op(cib_t * cib, const char op, const char host,
	const char section, xmlNode data,
	xmlNode ** output_data, int call_options, const char *user_name)
	{
	int (delegate) (cib_t cib, const char op, const char host,
	const char section, xmlNode data,
	xmlNode ** output_data, int call_options, const char *user_name) =
	cib->delegate_fn;

	#if ENABLE_ACL
	if(user_name == NULL) {
	user_name = getenv("CIB_user");
	}
	#endif

	return delegate(cib, op, host, section, data, output_data, call_options, user_name);
	}
	diff --git a/lib/common/utils.c b/lib/common/utils.c
	index 4ef02e89f1..5d37c618d2 100644
	--- a/lib/common/utils.c
	+++ b/lib/common/utils.c
	@@ -1,2598 +1,2608 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <dlfcn.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#include <sys/stat.h>
	#include <sys/utsname.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <string.h>
	#include <stdlib.h>
	#include <limits.h>
	#include <ctype.h>
	#include <pwd.h>
	#include <grp.h>
	#include <time.h>
	#include <libgen.h>
	#include <signal.h>

	#include <qb/qbdefs.h>

	#include <crm/crm.h>
	#include <crm/lrmd.h>
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/cib/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/util.h>
	#include <crm/common/ipc.h>
	#include <crm/common/iso8601.h>
	#include <crm/common/mainloop.h>
	#include <crm/attrd.h>
	#include <libxml2/libxml/relaxng.h>

	#ifndef MAXLINE
	# define MAXLINE 512
	#endif

	#ifdef HAVE_GETOPT_H
	# include <getopt.h>
	#endif

	#ifndef PW_BUFFER_LEN
	# define PW_BUFFER_LEN 500
	#endif

	CRM_TRACE_INIT_DATA(common);

	gboolean crm_config_error = FALSE;
	gboolean crm_config_warning = FALSE;
	char *crm_system_name = NULL;

	int node_score_red = 0;
	int node_score_green = 0;
	int node_score_yellow = 0;
	int node_score_infinity = INFINITY;

	static struct crm_option *crm_long_options = NULL;
	static const char *crm_app_description = NULL;
	static char *crm_short_options = NULL;
	static const char *crm_app_usage = NULL;

	int
	crm_exit(int rc)
	{
	mainloop_cleanup();

	#if HAVE_LIBXML2
	crm_trace("cleaning up libxml");
	crm_xml_cleanup();
	#endif

	crm_trace("exit %d", rc);
	qb_log_fini();

	free(crm_short_options);
	free(crm_system_name);

	exit(ABS(rc)); /* Always exit with a positive value so that it can be passed to crm_error
	*
	* Otherwise the system wraps it around and people
	* have to jump through hoops figuring out what the
	* error was
	*/
	return rc; /* Can never happen, but allows return crm_exit(rc)
	* where "return rc" was used previously - which
	* keeps compilers happy.
	*/
	}

	gboolean
	check_time(const char *value)
	{
	if (crm_get_msec(value) < 5000) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	check_timer(const char *value)
	{
	if (crm_get_msec(value) < 0) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	check_boolean(const char *value)
	{
	int tmp = FALSE;

	if (crm_str_to_boolean(value, &tmp) != 1) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	check_number(const char *value)
	{
	errno = 0;
	if (value == NULL) {
	return FALSE;

	} else if (safe_str_eq(value, MINUS_INFINITY_S)) {

	} else if (safe_str_eq(value, INFINITY_S)) {

	} else {
	crm_int_helper(value, NULL);
	}

	if (errno != 0) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	check_utilization(const char *value)
	{
	char *end = NULL;
	long number = strtol(value, &end, 10);

	if(end && end[0] != '%') {
	return FALSE;
	} else if(number < 0) {
	return FALSE;
	}

	return TRUE;
	}

	int
	char2score(const char *score)
	{
	int score_f = 0;

	if (score == NULL) {

	} else if (safe_str_eq(score, MINUS_INFINITY_S)) {
	score_f = -node_score_infinity;

	} else if (safe_str_eq(score, INFINITY_S)) {
	score_f = node_score_infinity;

	} else if (safe_str_eq(score, "+" INFINITY_S)) {
	score_f = node_score_infinity;

	} else if (safe_str_eq(score, "red")) {
	score_f = node_score_red;

	} else if (safe_str_eq(score, "yellow")) {
	score_f = node_score_yellow;

	} else if (safe_str_eq(score, "green")) {
	score_f = node_score_green;

	} else {
	score_f = crm_parse_int(score, NULL);
	if (score_f > 0 && score_f > node_score_infinity) {
	score_f = node_score_infinity;

	} else if (score_f < 0 && score_f < -node_score_infinity) {
	score_f = -node_score_infinity;
	}
	}

	return score_f;
	}

	char *
	score2char_stack(int score, char *buf, size_t len)
	{
	if (score >= node_score_infinity) {
	strncpy(buf, INFINITY_S, 9);
	} else if (score <= -node_score_infinity) {
	strncpy(buf, MINUS_INFINITY_S , 10);
	} else {
	return crm_itoa_stack(score, buf, len);
	}

	return buf;
	}

	char *
	score2char(int score)
	{
	if (score >= node_score_infinity) {
	return strdup(INFINITY_S);

	} else if (score <= -node_score_infinity) {
	return strdup("-" INFINITY_S);
	}
	return crm_itoa(score);
	}

	const char *
	cluster_option(GHashTable * options, gboolean(validate) (const char ),
	const char name, const char old_name, const char *def_value)
	{
	const char *value = NULL;

	CRM_ASSERT(name != NULL);

	if (options != NULL) {
	value = g_hash_table_lookup(options, name);
	}

	if (value == NULL && old_name && options != NULL) {
	value = g_hash_table_lookup(options, old_name);
	if (value != NULL) {
	crm_config_warn("Using deprecated name '%s' for"
	" cluster option '%s'", old_name, name);
	g_hash_table_insert(options, strdup(name), strdup(value));
	value = g_hash_table_lookup(options, old_name);
	}
	}

	if (value == NULL) {
	crm_trace("Using default value '%s' for cluster option '%s'", def_value, name);

	if (options == NULL) {
	return def_value;
	}

	g_hash_table_insert(options, strdup(name), strdup(def_value));
	value = g_hash_table_lookup(options, name);
	}

	if (validate && validate(value) == FALSE) {
	crm_config_err("Value '%s' for cluster option '%s' is invalid."
	" Defaulting to %s", value, name, def_value);
	g_hash_table_replace(options, strdup(name), strdup(def_value));
	value = g_hash_table_lookup(options, name);
	}

	return value;
	}

	const char *
	get_cluster_pref(GHashTable * options, pe_cluster_option * option_list, int len, const char *name)
	{
	int lpc = 0;
	const char *value = NULL;
	gboolean found = FALSE;

	for (lpc = 0; lpc < len; lpc++) {
	if (safe_str_eq(name, option_list[lpc].name)) {
	found = TRUE;
	value = cluster_option(options,
	option_list[lpc].is_valid,
	option_list[lpc].name,
	option_list[lpc].alt_name, option_list[lpc].default_value);
	}
	}
	CRM_CHECK(found, crm_err("No option named: %s", name));
	CRM_ASSERT(value != NULL);
	return value;
	}

	void
	config_metadata(const char name, const char version, const char *desc_short,
	const char desc_long, pe_cluster_option option_list, int len)
	{
	int lpc = 0;

	fprintf(stdout, "<?xml version=\"1.0\"?>"
	"<!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n"
	"<resource-agent name=\"%s\">\n"
	" <version>%s</version>\n"
	" <longdesc lang=\"en\">%s</longdesc>\n"
	" <shortdesc lang=\"en\">%s</shortdesc>\n"
	" <parameters>\n", name, version, desc_long, desc_short);

	for (lpc = 0; lpc < len; lpc++) {
	if (option_list[lpc].description_long == NULL && option_list[lpc].description_short == NULL) {
	continue;
	}
	fprintf(stdout, " <parameter name=\"%s\" unique=\"0\">\n"
	" <shortdesc lang=\"en\">%s</shortdesc>\n"
	" <content type=\"%s\" default=\"%s\"/>\n"
	" <longdesc lang=\"en\">%s%s%s</longdesc>\n"
	" </parameter>\n",
	option_list[lpc].name,
	option_list[lpc].description_short,
	option_list[lpc].type,
	option_list[lpc].default_value,
	option_list[lpc].description_long ? option_list[lpc].
	description_long : option_list[lpc].description_short,
	option_list[lpc].values ? " Allowed values: " : "",
	option_list[lpc].values ? option_list[lpc].values : "");
	}
	fprintf(stdout, " </parameters>\n</resource-agent>\n");
	}

	void
	verify_all_options(GHashTable * options, pe_cluster_option * option_list, int len)
	{
	int lpc = 0;

	for (lpc = 0; lpc < len; lpc++) {
	cluster_option(options,
	option_list[lpc].is_valid,
	option_list[lpc].name,
	option_list[lpc].alt_name, option_list[lpc].default_value);
	}
	}

	char *
	crm_concat(const char prefix, const char suffix, char join)
	{
	int len = 0;
	char *new_str = NULL;

	CRM_ASSERT(prefix != NULL);
	CRM_ASSERT(suffix != NULL);
	len = strlen(prefix) + strlen(suffix) + 2;

	new_str = malloc(len);
	if(new_str) {
	sprintf(new_str, "%s%c%s", prefix, join, suffix);
	new_str[len - 1] = 0;
	}
	return new_str;
	}

	char *
	generate_hash_key(const char crm_msg_reference, const char sys)
	{
	char *hash_key = crm_concat(sys ? sys : "none", crm_msg_reference, '_');

	crm_trace("created hash key: (%s)", hash_key);
	return hash_key;
	}


	char *
	crm_itoa_stack(int an_int, char *buffer, size_t len)
	{
	if (buffer != NULL) {
	snprintf(buffer, len, "%d", an_int);
	}

	return buffer;
	}

	char *
	crm_itoa(int an_int)
	{
	int len = 32;
	char *buffer = NULL;

	buffer = malloc(len + 1);
	if (buffer != NULL) {
	snprintf(buffer, len, "%d", an_int);
	}

	return buffer;
	}

	void
	crm_build_path(const char *path_c, mode_t mode)
	{
	int offset = 1, len = 0;
	char *path = strdup(path_c);

	CRM_CHECK(path != NULL, return);
	for (len = strlen(path); offset < len; offset++) {
	if (path[offset] == '/') {
	path[offset] = 0;
	if (mkdir(path, mode) < 0 && errno != EEXIST) {
	crm_perror(LOG_ERR, "Could not create directory '%s'", path);
	break;
	}
	path[offset] = '/';
	}
	}
	if (mkdir(path, mode) < 0 && errno != EEXIST) {
	crm_perror(LOG_ERR, "Could not create directory '%s'", path);
	}

	free(path);
	}

	int
	crm_user_lookup(const char name, uid_t uid, gid_t * gid)
	{
	int rc = -1;
	char *buffer = NULL;
	struct passwd pwd;
	struct passwd *pwentry = NULL;

	buffer = calloc(1, PW_BUFFER_LEN);
	getpwnam_r(name, &pwd, buffer, PW_BUFFER_LEN, &pwentry);
	if (pwentry) {
	rc = 0;
	if (uid) {
	*uid = pwentry->pw_uid;
	}
	if (gid) {
	*gid = pwentry->pw_gid;
	}
	crm_trace("Cluster user %s has uid=%d gid=%d", name, pwentry->pw_uid, pwentry->pw_gid);

	} else {
	crm_err("Cluster user %s does not exist", name);
	}

	free(buffer);
	return rc;
	}

	static int
	crm_version_helper(const char text, char *end_text)
	{
	int atoi_result = -1;

	CRM_ASSERT(end_text != NULL);

	errno = 0;

	if (text != NULL && text[0] != 0) {
	atoi_result = (int)strtol(text, end_text, 10);

	if (errno == EINVAL) {
	crm_err("Conversion of '%s' %c failed", text, text[0]);
	atoi_result = -1;
	}
	}
	return atoi_result;
	}

	/*
	* version1 < version2 : -1
	* version1 = version2 : 0
	* version1 > version2 : 1
	*/
	int
	compare_version(const char version1, const char version2)
	{
	int rc = 0;
	int lpc = 0;
	char ver1_copy = NULL, ver2_copy = NULL;
	char rest1 = NULL, rest2 = NULL;

	if (version1 == NULL && version2 == NULL) {
	return 0;
	} else if (version1 == NULL) {
	return -1;
	} else if (version2 == NULL) {
	return 1;
	}

	ver1_copy = strdup(version1);
	ver2_copy = strdup(version2);
	rest1 = ver1_copy;
	rest2 = ver2_copy;

	while (1) {
	int digit1 = 0;
	int digit2 = 0;

	lpc++;

	if (rest1 == rest2) {
	break;
	}

	if (rest1 != NULL) {
	digit1 = crm_version_helper(rest1, &rest1);
	}

	if (rest2 != NULL) {
	digit2 = crm_version_helper(rest2, &rest2);
	}

	if (digit1 < digit2) {
	rc = -1;
	break;

	} else if (digit1 > digit2) {
	rc = 1;
	break;
	}

	if (rest1 != NULL && rest1[0] == '.') {
	rest1++;
	}
	if (rest1 != NULL && rest1[0] == 0) {
	rest1 = NULL;
	}

	if (rest2 != NULL && rest2[0] == '.') {
	rest2++;
	}
	if (rest2 != NULL && rest2[0] == 0) {
	rest2 = NULL;
	}
	}

	free(ver1_copy);
	free(ver2_copy);

	if (rc == 0) {
	crm_trace("%s == %s (%d)", version1, version2, lpc);
	} else if (rc < 0) {
	crm_trace("%s < %s (%d)", version1, version2, lpc);
	} else if (rc > 0) {
	crm_trace("%s > %s (%d)", version1, version2, lpc);
	}

	return rc;
	}

	gboolean do_stderr = FALSE;

	void
	g_hash_destroy_str(gpointer data)
	{
	free(data);
	}

	#include <sys/types.h>
	/* #include <stdlib.h> */
	/* #include <limits.h> */

	long long
	crm_int_helper(const char text, char *end_text)
	{
	long long result = -1;
	char *local_end_text = NULL;
	int saved_errno = 0;

	errno = 0;

	if (text != NULL) {
	#ifdef ANSI_ONLY
	if (end_text != NULL) {
	result = strtol(text, end_text, 10);
	} else {
	result = strtol(text, &local_end_text, 10);
	}
	#else
	if (end_text != NULL) {
	result = strtoll(text, end_text, 10);
	} else {
	result = strtoll(text, &local_end_text, 10);
	}
	#endif

	saved_errno = errno;
	/* CRM_CHECK(errno != EINVAL); */
	if (errno == EINVAL) {
	crm_err("Conversion of %s failed", text);
	result = -1;

	} else if (errno == ERANGE) {
	crm_err("Conversion of %s was clipped: %lld", text, result);

	} else if (errno != 0) {
	crm_perror(LOG_ERR, "Conversion of %s failed:", text);
	}

	if (local_end_text != NULL && local_end_text[0] != '\0') {
	crm_err("Characters left over after parsing '%s': '%s'", text, local_end_text);
	}

	errno = saved_errno;
	}
	return result;
	}

	int
	crm_parse_int(const char text, const char default_text)
	{
	int atoi_result = -1;

	if (text != NULL) {
	atoi_result = crm_int_helper(text, NULL);
	if (errno == 0) {
	return atoi_result;
	}
	}

	if (default_text != NULL) {
	atoi_result = crm_int_helper(default_text, NULL);
	if (errno == 0) {
	return atoi_result;
	}

	} else {
	crm_err("No default conversion value supplied");
	}

	return -1;
	}

	gboolean
	safe_str_neq(const char a, const char b)
	{
	if (a == b) {
	return FALSE;

	} else if (a == NULL \|\| b == NULL) {
	return TRUE;

	} else if (strcasecmp(a, b) == 0) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	crm_is_true(const char *s)
	{
	gboolean ret = FALSE;

	if (s != NULL) {
	crm_str_to_boolean(s, &ret);
	}
	return ret;
	}

	int
	crm_str_to_boolean(const char s, int ret)
	{
	if (s == NULL) {
	return -1;

	} else if (strcasecmp(s, "true") == 0
	\|\| strcasecmp(s, "on") == 0
	\|\| strcasecmp(s, "yes") == 0 \|\| strcasecmp(s, "y") == 0 \|\| strcasecmp(s, "1") == 0) {
	*ret = TRUE;
	return 1;

	} else if (strcasecmp(s, "false") == 0
	\|\| strcasecmp(s, "off") == 0
	\|\| strcasecmp(s, "no") == 0 \|\| strcasecmp(s, "n") == 0 \|\| strcasecmp(s, "0") == 0) {
	*ret = FALSE;
	return 1;
	}
	return -1;
	}

	#ifndef NUMCHARS
	# define NUMCHARS "0123456789."
	#endif

	#ifndef WHITESPACE
	# define WHITESPACE " \t\n\r\f"
	#endif

	unsigned long long
	crm_get_interval(const char *input)
	{
	unsigned long long msec = 0;

	if (input == NULL) {
	return msec;

	} else if (input[0] != 'P') {
	long long tmp = crm_get_msec(input);

	if(tmp > 0) {
	msec = tmp;
	}

	} else {
	crm_time_t *interval = crm_time_parse_duration(input);

	msec = 1000 * crm_time_get_seconds(interval);
	crm_time_free(interval);
	}

	return msec;
	}

	long long
	crm_get_msec(const char *input)
	{
	const char *cp = input;
	const char *units;
	long long multiplier = 1000;
	long long divisor = 1;
	long long msec = -1;
	char *end_text = NULL;

	/* double dret; */

	if (input == NULL) {
	return msec;
	}

	cp += strspn(cp, WHITESPACE);
	units = cp + strspn(cp, NUMCHARS);
	units += strspn(units, WHITESPACE);

	if (strchr(NUMCHARS, *cp) == NULL) {
	return msec;
	}

	if (strncasecmp(units, "ms", 2) == 0 \|\| strncasecmp(units, "msec", 4) == 0) {
	multiplier = 1;
	divisor = 1;
	} else if (strncasecmp(units, "us", 2) == 0 \|\| strncasecmp(units, "usec", 4) == 0) {
	multiplier = 1;
	divisor = 1000;
	} else if (strncasecmp(units, "s", 1) == 0 \|\| strncasecmp(units, "sec", 3) == 0) {
	multiplier = 1000;
	divisor = 1;
	} else if (strncasecmp(units, "m", 1) == 0 \|\| strncasecmp(units, "min", 3) == 0) {
	multiplier = 60 * 1000;
	divisor = 1;
	} else if (strncasecmp(units, "h", 1) == 0 \|\| strncasecmp(units, "hr", 2) == 0) {
	multiplier = 60 * 60 * 1000;
	divisor = 1;
	} else if (units != EOS && units != '\n' && *units != '\r') {
	return msec;
	}

	msec = crm_int_helper(cp, &end_text);
	if (msec > LLONG_MAX/multiplier) {
	/* arithmetics overflow while multiplier/divisor mutually exclusive */
	return LLONG_MAX;
	}
	msec *= multiplier;
	msec /= divisor;
	/* dret += 0.5; */
	/* msec = (long long)dret; */
	return msec;
	}

	char *
	generate_op_key(const char rsc_id, const char op_type, int interval)
	{
	int len = 35;
	char *op_id = NULL;

	CRM_CHECK(rsc_id != NULL, return NULL);
	CRM_CHECK(op_type != NULL, return NULL);

	len += strlen(op_type);
	len += strlen(rsc_id);
	op_id = malloc(len);
	CRM_CHECK(op_id != NULL, return NULL);
	sprintf(op_id, "%s_%s_%d", rsc_id, op_type, interval);
	return op_id;
	}

	gboolean
	parse_op_key(const char key, char rsc_id, char op_type, int interval)
	{
	char *notify = NULL;
	char *mutable_key = NULL;
	char *mutable_key_ptr = NULL;
	int len = 0, offset = 0, ch = 0;

	CRM_CHECK(key != NULL, return FALSE);

	*interval = 0;
	len = strlen(key);
	offset = len - 1;

	crm_trace("Source: %s", key);

	while (offset > 0 && isdigit(key[offset])) {
	int digits = len - offset;

	ch = key[offset] - '0';
	CRM_CHECK(ch < 10, return FALSE);
	CRM_CHECK(ch >= 0, return FALSE);
	while (digits > 1) {
	digits--;
	ch = ch * 10;
	}
	*interval += ch;
	offset--;
	}

	crm_trace(" Interval: %d", *interval);
	CRM_CHECK(key[offset] == '_', return FALSE);

	mutable_key = strdup(key);
	mutable_key[offset] = 0;
	offset--;

	while (offset > 0 && key[offset] != '_') {
	offset--;
	}

	CRM_CHECK(key[offset] == '_', free(mutable_key);
	return FALSE);

	mutable_key_ptr = mutable_key + offset + 1;

	crm_trace(" Action: %s", mutable_key_ptr);

	*op_type = strdup(mutable_key_ptr);

	mutable_key[offset] = 0;
	offset--;

	CRM_CHECK(mutable_key != mutable_key_ptr, free(mutable_key);
	return FALSE);

	notify = strstr(mutable_key, "_post_notify");
	if (notify && safe_str_eq(notify, "_post_notify")) {
	notify[0] = 0;
	}

	notify = strstr(mutable_key, "_pre_notify");
	if (notify && safe_str_eq(notify, "_pre_notify")) {
	notify[0] = 0;
	}

	crm_trace(" Resource: %s", mutable_key);
	*rsc_id = mutable_key;

	return TRUE;
	}

	char *
	generate_notify_key(const char rsc_id, const char notify_type, const char *op_type)
	{
	int len = 12;
	char *op_id = NULL;

	CRM_CHECK(rsc_id != NULL, return NULL);
	CRM_CHECK(op_type != NULL, return NULL);
	CRM_CHECK(notify_type != NULL, return NULL);

	len += strlen(op_type);
	len += strlen(rsc_id);
	len += strlen(notify_type);
	if(len > 0) {
	op_id = malloc(len);
	}
	if (op_id != NULL) {
	sprintf(op_id, "%s_%s_notify_%s_0", rsc_id, notify_type, op_type);
	}
	return op_id;
	}

	char *
	generate_transition_magic_v202(const char *transition_key, int op_status)
	{
	int len = 80;
	char *fail_state = NULL;

	CRM_CHECK(transition_key != NULL, return NULL);

	len += strlen(transition_key);

	fail_state = malloc(len);
	if (fail_state != NULL) {
	snprintf(fail_state, len, "%d:%s", op_status, transition_key);
	}
	return fail_state;
	}

	char *
	generate_transition_magic(const char *transition_key, int op_status, int op_rc)
	{
	int len = 80;
	char *fail_state = NULL;

	CRM_CHECK(transition_key != NULL, return NULL);

	len += strlen(transition_key);

	fail_state = malloc(len);
	if (fail_state != NULL) {
	snprintf(fail_state, len, "%d:%d;%s", op_status, op_rc, transition_key);
	}
	return fail_state;
	}

	gboolean
	decode_transition_magic(const char magic, char uuid, int transition_id, int *action_id,
	int op_status, int op_rc, int *target_rc)
	{
	int res = 0;
	char *key = NULL;
	gboolean result = TRUE;

	CRM_CHECK(magic != NULL, return FALSE);
	CRM_CHECK(op_rc != NULL, return FALSE);
	CRM_CHECK(op_status != NULL, return FALSE);

	key = calloc(1, strlen(magic) + 1);
	res = sscanf(magic, "%d:%d;%s", op_status, op_rc, key);
	if (res != 3) {
	crm_warn("Only found %d items in: '%s'", res, magic);
	free(key);
	return FALSE;
	}

	CRM_CHECK(decode_transition_key(key, uuid, transition_id, action_id, target_rc), result = FALSE);

	free(key);
	return result;
	}

	char *
	generate_transition_key(int transition_id, int action_id, int target_rc, const char *node)
	{
	int len = 40;
	char *fail_state = NULL;

	CRM_CHECK(node != NULL, return NULL);

	len += strlen(node);

	fail_state = malloc(len);
	if (fail_state != NULL) {
	snprintf(fail_state, len, "%d:%d:%d:%-*s", action_id, transition_id, target_rc, 36, node);
	}
	return fail_state;
	}

	gboolean
	decode_transition_key(const char key, char uuid, int transition_id, int *action_id,
	int *target_rc)
	{
	int res = 0;
	gboolean done = FALSE;

	CRM_CHECK(uuid != NULL, return FALSE);
	CRM_CHECK(target_rc != NULL, return FALSE);
	CRM_CHECK(action_id != NULL, return FALSE);
	CRM_CHECK(transition_id != NULL, return FALSE);

	*uuid = calloc(1, 37);
	res = sscanf(key, "%d:%d:%d:%36s", action_id, transition_id, target_rc, *uuid);
	switch (res) {
	case 4:
	/* Post Pacemaker 0.6 */
	done = TRUE;
	break;
	case 3:
	case 2:
	/* this can be tricky - the UUID might start with an integer */

	/* Until Pacemaker 0.6 */
	done = TRUE;
	*target_rc = -1;
	res = sscanf(key, "%d:%d:%36s", action_id, transition_id, *uuid);
	if (res == 2) {
	*action_id = -1;
	res = sscanf(key, "%d:%36s", transition_id, *uuid);
	CRM_CHECK(res == 2, done = FALSE);

	} else if (res != 3) {
	CRM_CHECK(res == 3, done = FALSE);
	}
	break;

	case 1:
	/* Prior to Heartbeat 2.0.8 */
	done = TRUE;
	*action_id = -1;
	*target_rc = -1;
	res = sscanf(key, "%d:%36s", transition_id, *uuid);
	CRM_CHECK(res == 2, done = FALSE);
	break;
	default:
	crm_crit("Unhandled sscanf result (%d) for %s", res, key);
	}

	if (strlen(*uuid) != 36) {
	crm_warn("Bad UUID (%s) in sscanf result (%d) for %s", *uuid, res, key);
	}

	if (done == FALSE) {
	crm_err("Cannot decode '%s' rc=%d", key, res);

	free(*uuid);
	*uuid = NULL;
	*target_rc = -1;
	*action_id = -1;
	*transition_id = -1;
	}

	return done;
	}

	void
	filter_action_parameters(xmlNode * param_set, const char *version)
	{
	char *key = NULL;
	char *timeout = NULL;
	char *interval = NULL;

	const char *attr_filter[] = {
	XML_ATTR_ID,
	XML_ATTR_CRM_VERSION,
	XML_LRM_ATTR_OP_DIGEST,
	};

	gboolean do_delete = FALSE;
	int lpc = 0;
	static int meta_len = 0;

	if (meta_len == 0) {
	meta_len = strlen(CRM_META);
	}

	if (param_set == NULL) {
	return;
	}

	for (lpc = 0; lpc < DIMOF(attr_filter); lpc++) {
	xml_remove_prop(param_set, attr_filter[lpc]);
	}

	key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
	interval = crm_element_value_copy(param_set, key);
	free(key);

	key = crm_meta_name(XML_ATTR_TIMEOUT);
	timeout = crm_element_value_copy(param_set, key);

	if (param_set) {
	xmlAttrPtr xIter = param_set->properties;

	while (xIter) {
	const char prop_name = (const char )xIter->name;

	xIter = xIter->next;
	do_delete = FALSE;
	if (strncasecmp(prop_name, CRM_META, meta_len) == 0) {
	do_delete = TRUE;
	}

	if (do_delete) {
	xml_remove_prop(param_set, prop_name);
	}
	}
	}

	if (crm_get_msec(interval) > 0 && compare_version(version, "1.0.8") > 0) {
	/* Re-instate the operation's timeout value */
	if (timeout != NULL) {
	crm_xml_add(param_set, key, timeout);
	}
	}

	free(interval);
	free(timeout);
	free(key);
	}

	void
	filter_reload_parameters(xmlNode * param_set, const char *restart_string)
	{
	int len = 0;
	char *name = NULL;
	char *match = NULL;

	if (param_set == NULL) {
	return;
	}

	if (param_set) {
	xmlAttrPtr xIter = param_set->properties;

	while (xIter) {
	const char prop_name = (const char )xIter->name;

	xIter = xIter->next;
	name = NULL;
	len = strlen(prop_name) + 3;

	name = malloc(len);
	if(name) {
	sprintf(name, " %s ", prop_name);
	name[len - 1] = 0;
	match = strstr(restart_string, name);
	}

	if (match == NULL) {
	crm_trace("%s not found in %s", prop_name, restart_string);
	xml_remove_prop(param_set, prop_name);
	}
	free(name);
	}
	}
	}

	extern bool crm_is_daemon;

	/* coverity[+kill] */
	void
	crm_abort(const char file, const char function, int line,
	const char *assert_condition, gboolean do_core, gboolean do_fork)
	{
	int rc = 0;
	int pid = 0;
	int status = 0;

	/* Implied by the parent's error logging below */
	/* crm_write_blackbox(0); */

	if(crm_is_daemon == FALSE) {
	/* This is a command line tool - do not fork */

	/* crm_add_logfile(NULL); * Record it to a file? */
	crm_enable_stderr(TRUE); /* Make sure stderr is enabled so we can tell the caller */
	do_fork = FALSE; /* Just crash if needed */
	}

	if (do_core == FALSE) {
	crm_err("%s: Triggered assert at %s:%d : %s", function, file, line, assert_condition);
	return;

	} else if (do_fork) {
	pid = fork();

	} else {
	crm_err("%s: Triggered fatal assert at %s:%d : %s", function, file, line, assert_condition);
	}

	if (pid == -1) {
	crm_crit("%s: Cannot create core for non-fatal assert at %s:%d : %s",
	function, file, line, assert_condition);
	return;

	} else if(pid == 0) {
	/* Child process */
	abort();
	return;
	}

	/* Parent process */
	crm_err("%s: Forked child %d to record non-fatal assert at %s:%d : %s",
	function, pid, file, line, assert_condition);
	crm_write_blackbox(SIGTRAP, NULL);

	do {
	rc = waitpid(pid, &status, 0);
	if(rc == pid) {
	return; /* Job done */
	}

	} while(errno == EINTR);

	if (errno == ECHILD) {
	/* crm_mon does this */
	crm_trace("Cannot wait on forked child %d - SIGCHLD is probably set to SIG_IGN", pid);
	return;
	}
	crm_perror(LOG_ERR, "Cannot wait on forked child %d", pid);
	}

	char *
	generate_series_filename(const char directory, const char series, int sequence, gboolean bzip)
	{
	int len = 40;
	char *filename = NULL;
	const char *ext = "raw";

	CRM_CHECK(directory != NULL, return NULL);
	CRM_CHECK(series != NULL, return NULL);

	#if !HAVE_BZLIB_H
	bzip = FALSE;
	#endif

	len += strlen(directory);
	len += strlen(series);
	filename = malloc(len);
	CRM_CHECK(filename != NULL, return NULL);

	if (bzip) {
	ext = "bz2";
	}
	sprintf(filename, "%s/%s-%d.%s", directory, series, sequence, ext);

	return filename;
	}

	int
	get_last_sequence(const char directory, const char series)
	{
	FILE *file_strm = NULL;
	int start = 0, length = 0, read_len = 0;
	char *series_file = NULL;
	char *buffer = NULL;
	int seq = 0;
	int len = 36;

	CRM_CHECK(directory != NULL, return 0);
	CRM_CHECK(series != NULL, return 0);

	len += strlen(directory);
	len += strlen(series);
	series_file = malloc(len);
	CRM_CHECK(series_file != NULL, return 0);
	sprintf(series_file, "%s/%s.last", directory, series);

	file_strm = fopen(series_file, "r");
	if (file_strm == NULL) {
	crm_debug("Series file %s does not exist", series_file);
	free(series_file);
	return 0;
	}

	/* see how big the file is */
	start = ftell(file_strm);
	fseek(file_strm, 0L, SEEK_END);
	length = ftell(file_strm);
	fseek(file_strm, 0L, start);

	CRM_ASSERT(length >= 0);
	CRM_ASSERT(start == ftell(file_strm));

	if (length <= 0) {
	crm_info("%s was not valid", series_file);
	free(buffer);
	buffer = NULL;

	} else {
	crm_trace("Reading %d bytes from file", length);
	buffer = calloc(1, (length + 1));
	read_len = fread(buffer, 1, length, file_strm);
	if (read_len != length) {
	crm_err("Calculated and read bytes differ: %d vs. %d", length, read_len);
	free(buffer);
	buffer = NULL;
	}
	}

	seq = crm_parse_int(buffer, "0");
	fclose(file_strm);

	crm_trace("Found %d in %s", seq, series_file);

	free(series_file);
	free(buffer);
	return seq;
	}

	void
	write_last_sequence(const char directory, const char series, int sequence, int max)
	{
	int rc = 0;
	int len = 36;
	FILE *file_strm = NULL;
	char *series_file = NULL;

	CRM_CHECK(directory != NULL, return);
	CRM_CHECK(series != NULL, return);

	if (max == 0) {
	return;
	}
	if (max > 0 && sequence >= max) {
	sequence = 0;
	}

	len += strlen(directory);
	len += strlen(series);
	series_file = malloc(len);

	if(series_file) {
	sprintf(series_file, "%s/%s.last", directory, series);
	file_strm = fopen(series_file, "w");
	}

	if (file_strm != NULL) {
	rc = fprintf(file_strm, "%d", sequence);
	if (rc < 0) {
	crm_perror(LOG_ERR, "Cannot write to series file %s", series_file);
	}

	} else {
	crm_err("Cannot open series file %s for writing", series_file);
	}

	if (file_strm != NULL) {
	fflush(file_strm);
	fclose(file_strm);
	}

	crm_trace("Wrote %d to %s", sequence, series_file);
	free(series_file);
	}

	#define LOCKSTRLEN 11

	int
	crm_pid_active(long pid)
	{
	if (pid <= 0) {
	return -1;

	} else if (kill(pid, 0) < 0 && errno == ESRCH) {
	return 0;
	}
	#ifndef HAVE_PROC_PID
	return 1;
	#else
	{
	int rc = 0;
	int running = 0;
	char proc_path[PATH_MAX], exe_path[PATH_MAX], myexe_path[PATH_MAX];

	/* check to make sure pid hasn't been reused by another process */
	snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", pid);

	rc = readlink(proc_path, exe_path, PATH_MAX - 1);
	if (rc < 0) {
	crm_perror(LOG_ERR, "Could not read from %s", proc_path);
	goto bail;
	}

	exe_path[rc] = 0;
	snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", (long unsigned int)getpid());
	rc = readlink(proc_path, myexe_path, PATH_MAX - 1);
	if (rc < 0) {
	crm_perror(LOG_ERR, "Could not read from %s", proc_path);
	goto bail;
	}

	myexe_path[rc] = 0;
	if (strcmp(exe_path, myexe_path) == 0) {
	running = 1;
	}
	}

	bail:
	return running;
	#endif
	}

	static int
	crm_read_pidfile(const char *filename)
	{
	int fd;
	long pid = -1;
	char buf[LOCKSTRLEN + 1];

	if ((fd = open(filename, O_RDONLY)) < 0) {
	goto bail;
	}

	if (read(fd, buf, sizeof(buf)) < 1) {
	goto bail;
	}

	if (sscanf(buf, "%lu", &pid) > 0) {
	if (pid <= 0) {
	pid = -ESRCH;
	}
	}

	bail:
	if (fd >= 0) {
	close(fd);
	}
	return pid;
	}

	static int
	crm_pidfile_inuse(const char *filename, long mypid)
	{
	long pid = 0;
	struct stat sbuf;
	char buf[LOCKSTRLEN + 1];
	int rc = -ENOENT, fd = 0;

	if ((fd = open(filename, O_RDONLY)) >= 0) {
	if (fstat(fd, &sbuf) >= 0 && sbuf.st_size < LOCKSTRLEN) {
	sleep(2); /* if someone was about to create one,
	* give'm a sec to do so
	*/
	}
	if (read(fd, buf, sizeof(buf)) > 0) {
	if (sscanf(buf, "%lu", &pid) > 0) {
	crm_trace("Got pid %lu from %s\n", pid, filename);
	if (pid <= 1) {
	/* Invalid pid */
	rc = -ENOENT;
	unlink(filename);

	} else if (mypid && pid == mypid) {
	/* In use by us */
	rc = pcmk_ok;

	} else if (crm_pid_active(pid) == FALSE) {
	/* Contains a stale value */
	unlink(filename);
	rc = -ENOENT;

	} else if (mypid && pid != mypid) {
	/* locked by existing process - give up */
	rc = -EEXIST;
	}
	}
	}
	close(fd);
	}
	return rc;
	}

	static int
	crm_lock_pidfile(const char *filename)
	{
	long mypid = 0;
	int fd = 0, rc = 0;
	char buf[LOCKSTRLEN + 1];

	mypid = (unsigned long)getpid();

	rc = crm_pidfile_inuse(filename, 0);
	if (rc == -ENOENT) {
	/* exists but the process is not active */

	} else if (rc != pcmk_ok) {
	/* locked by existing process - give up */
	return rc;
	}

	if ((fd = open(filename, O_CREAT \| O_WRONLY \| O_EXCL, 0644)) < 0) {
	/* Hmmh, why did we fail? Anyway, nothing we can do about it */
	return -errno;
	}

	snprintf(buf, sizeof(buf), "%*lu\n", LOCKSTRLEN - 1, mypid);
	rc = write(fd, buf, LOCKSTRLEN);
	close(fd);

	if (rc != LOCKSTRLEN) {
	crm_perror(LOG_ERR, "Incomplete write to %s", filename);
	return -errno;
	}

	return crm_pidfile_inuse(filename, mypid);
	}

	void
	crm_make_daemon(const char name, gboolean daemonize, const char pidfile)
	{
	int rc;
	long pid;
	const char *devnull = "/dev/null";

	if (daemonize == FALSE) {
	return;
	}

	/* Check before we even try... */
	rc = crm_pidfile_inuse(pidfile, 1);
	if(rc < pcmk_ok && rc != -ENOENT) {
	pid = crm_read_pidfile(pidfile);
	crm_err("%s: already running [pid %ld in %s]", name, pid, pidfile);
	printf("%s: already running [pid %ld in %s]\n", name, pid, pidfile);
	crm_exit(rc);
	}

	pid = fork();
	if (pid < 0) {
	fprintf(stderr, "%s: could not start daemon\n", name);
	crm_perror(LOG_ERR, "fork");
	crm_exit(EINVAL);

	} else if (pid > 0) {
	crm_exit(pcmk_ok);
	}

	rc = crm_lock_pidfile(pidfile);
	if(rc < pcmk_ok) {
	crm_err("Could not lock '%s' for %s: %s (%d)", pidfile, name, pcmk_strerror(rc), rc);
	printf("Could not lock '%s' for %s: %s (%d)\n", pidfile, name, pcmk_strerror(rc), rc);
	crm_exit(rc);
	}

	umask(S_IWGRP \| S_IWOTH \| S_IROTH);

	close(STDIN_FILENO);
	(void)open(devnull, O_RDONLY); /* Stdin: fd 0 */
	close(STDOUT_FILENO);
	(void)open(devnull, O_WRONLY); /* Stdout: fd 1 */
	close(STDERR_FILENO);
	(void)open(devnull, O_WRONLY); /* Stderr: fd 2 */
	}

	gboolean
	crm_is_writable(const char dir, const char file,
	const char user, const char group, gboolean need_both)
	{
	int s_res = -1;
	struct stat buf;
	char *full_file = NULL;
	const char *target = NULL;

	gboolean pass = TRUE;
	gboolean readwritable = FALSE;

	CRM_ASSERT(dir != NULL);
	if (file != NULL) {
	full_file = crm_concat(dir, file, '/');
	target = full_file;
	s_res = stat(full_file, &buf);
	if (s_res == 0 && S_ISREG(buf.st_mode) == FALSE) {
	crm_err("%s must be a regular file", target);
	pass = FALSE;
	goto out;
	}
	}

	if (s_res != 0) {
	target = dir;
	s_res = stat(dir, &buf);
	if (s_res != 0) {
	crm_err("%s must exist and be a directory", dir);
	pass = FALSE;
	goto out;

	} else if (S_ISDIR(buf.st_mode) == FALSE) {
	crm_err("%s must be a directory", dir);
	pass = FALSE;
	}
	}

	if (user) {
	struct passwd *sys_user = NULL;

	sys_user = getpwnam(user);
	readwritable = (sys_user != NULL
	&& buf.st_uid == sys_user->pw_uid && (buf.st_mode & (S_IRUSR \| S_IWUSR)));
	if (readwritable == FALSE) {
	crm_err("%s must be owned and r/w by user %s", target, user);
	if (need_both) {
	pass = FALSE;
	}
	}
	}

	if (group) {
	struct group *sys_grp = getgrnam(group);

	readwritable = (sys_grp != NULL
	&& buf.st_gid == sys_grp->gr_gid && (buf.st_mode & (S_IRGRP \| S_IWGRP)));
	if (readwritable == FALSE) {
	if (need_both \|\| user == NULL) {
	pass = FALSE;
	crm_err("%s must be owned and r/w by group %s", target, group);
	} else {
	crm_warn("%s should be owned and r/w by group %s", target, group);
	}
	}
	}

	out:
	free(full_file);
	return pass;
	}

	char *
	crm_strip_trailing_newline(char *str)
	{
	int len;

	if (str == NULL) {
	return str;
	}

	for (len = strlen(str) - 1; len >= 0 && str[len] == '\n'; len--) {
	str[len] = '\0';
	}

	return str;
	}

	gboolean
	crm_str_eq(const char a, const char b, gboolean use_case)
	{
	if (use_case) {
	return g_strcmp0(a, b) == 0;

	/* TODO - Figure out which calls, if any, really need to be case independant */
	} else if (a == b) {
	return TRUE;

	} else if (a == NULL \|\| b == NULL) {
	/* shouldn't be comparing NULLs */
	return FALSE;

	} else if (strcasecmp(a, b) == 0) {
	return TRUE;
	}
	return FALSE;
	}

	char *
	crm_meta_name(const char *field)
	{
	int lpc = 0;
	int max = 0;
	char *crm_name = NULL;

	CRM_CHECK(field != NULL, return NULL);
	crm_name = crm_concat(CRM_META, field, '_');

	/* Massage the names so they can be used as shell variables */
	max = strlen(crm_name);
	for (; lpc < max; lpc++) {
	switch (crm_name[lpc]) {
	case '-':
	crm_name[lpc] = '_';
	break;
	}
	}
	return crm_name;
	}

	const char *
	crm_meta_value(GHashTable * hash, const char *field)
	{
	char *key = NULL;
	const char *value = NULL;

	key = crm_meta_name(field);
	if (key) {
	value = g_hash_table_lookup(hash, key);
	free(key);
	}

	return value;
	}

	static struct option *
	crm_create_long_opts(struct crm_option *long_options)
	{
	struct option *long_opts = NULL;

	#ifdef HAVE_GETOPT_H
	int index = 0, lpc = 0;

	/*
	* A previous, possibly poor, choice of '?' as the short form of --help
	* means that getopt_long() returns '?' for both --help and for "unknown option"
	*
	* This dummy entry allows us to differentiate between the two in crm_get_option()
	* and exit with the correct error code
	*/
	long_opts = realloc(long_opts, (index + 1) * sizeof(struct option));
	long_opts[index].name = "__dummmy__";
	long_opts[index].has_arg = 0;
	long_opts[index].flag = 0;
	long_opts[index].val = '_';
	index++;

	for (lpc = 0; long_options[lpc].name != NULL; lpc++) {
	if (long_options[lpc].name[0] == '-') {
	continue;
	}

	long_opts = realloc(long_opts, (index + 1) * sizeof(struct option));
	/*fprintf(stderr, "Creating %d %s = %c\n", index,
	* long_options[lpc].name, long_options[lpc].val); */
	long_opts[index].name = long_options[lpc].name;
	long_opts[index].has_arg = long_options[lpc].has_arg;
	long_opts[index].flag = long_options[lpc].flag;
	long_opts[index].val = long_options[lpc].val;
	index++;
	}

	/* Now create the list terminator */
	long_opts = realloc(long_opts, (index + 1) * sizeof(struct option));
	long_opts[index].name = NULL;
	long_opts[index].has_arg = 0;
	long_opts[index].flag = 0;
	long_opts[index].val = 0;
	#endif

	return long_opts;
	}

	void
	crm_set_options(const char short_options, const char app_usage, struct crm_option *long_options,
	const char *app_desc)
	{
	if (short_options) {
	crm_short_options = strdup(short_options);

	} else if (long_options) {
	int lpc = 0;
	int opt_string_len = 0;
	char *local_short_options = NULL;

	for (lpc = 0; long_options[lpc].name != NULL; lpc++) {
	if (long_options[lpc].val && long_options[lpc].val != '-' && long_options[lpc].val < UCHAR_MAX) {
	local_short_options = realloc(local_short_options, opt_string_len + 4);
	local_short_options[opt_string_len++] = long_options[lpc].val;
	/* getopt(3) says: Two colons mean an option takes an optional arg; */
	if (long_options[lpc].has_arg == optional_argument) {
	local_short_options[opt_string_len++] = ':';
	}
	if (long_options[lpc].has_arg >= required_argument) {
	local_short_options[opt_string_len++] = ':';
	}
	local_short_options[opt_string_len] = 0;
	}
	}
	crm_short_options = local_short_options;
	crm_trace("Generated short option string: '%s'", local_short_options);
	}

	if (long_options) {
	crm_long_options = long_options;
	}
	if (app_desc) {
	crm_app_description = app_desc;
	}
	if (app_usage) {
	crm_app_usage = app_usage;
	}
	}

	int
	crm_get_option(int argc, char *argv, int index)
	{
	return crm_get_option_long(argc, argv, index, NULL);
	}

	int
	crm_get_option_long(int argc, char *argv, int index, const char **longname)
	{
	#ifdef HAVE_GETOPT_H
	static struct option *long_opts = NULL;

	if (long_opts == NULL && crm_long_options) {
	long_opts = crm_create_long_opts(crm_long_options);
	}

	if (long_opts) {
	int flag = getopt_long(argc, argv, crm_short_options, long_opts, index);

	switch (flag) {
	case 0:
	if (long_opts[*index].val) {
	return long_opts[*index].val;
	} else if (longname) {
	longname = long_opts[index].name;
	} else {
	crm_notice("Unhandled option --%s", long_opts[*index].name);
	return flag;
	}
	case -1: /* End of option processing */
	break;
	case ':':
	crm_trace("Missing argument");
	crm_help('?', 1);
	break;
	case '?':
	crm_help('?', *index ? 0 : 1);
	break;
	}
	return flag;
	}
	#endif

	if (crm_short_options) {
	return getopt(argc, argv, crm_short_options);
	}

	return -1;
	}

	int
	crm_help(char cmd, int exit_code)
	{
	int i = 0;
	FILE *stream = (exit_code ? stderr : stdout);

	if (cmd == 'v' \|\| cmd == '$') {
	fprintf(stream, "Pacemaker %s\n", VERSION);
	fprintf(stream, "Written by Andrew Beekhof\n");
	goto out;
	}

	if (cmd == '!') {
	fprintf(stream, "Pacemaker %s (Build: %s): %s\n", VERSION, BUILD_VERSION, CRM_FEATURES);
	goto out;
	}

	fprintf(stream, "%s - %s\n", crm_system_name, crm_app_description);

	if (crm_app_usage) {
	fprintf(stream, "Usage: %s %s\n", crm_system_name, crm_app_usage);
	}

	if (crm_long_options) {
	fprintf(stream, "Options:\n");
	for (i = 0; crm_long_options[i].name != NULL; i++) {
	if (crm_long_options[i].flags & pcmk_option_hidden) {

	} else if (crm_long_options[i].flags & pcmk_option_paragraph) {
	fprintf(stream, "%s\n\n", crm_long_options[i].desc);

	} else if (crm_long_options[i].flags & pcmk_option_example) {
	fprintf(stream, "\t#%s\n\n", crm_long_options[i].desc);

	} else if (crm_long_options[i].val == '-' && crm_long_options[i].desc) {
	fprintf(stream, "%s\n", crm_long_options[i].desc);

	} else {
	/* is val printable as char ? */
	if (crm_long_options[i].val && crm_long_options[i].val <= UCHAR_MAX) {
	fprintf(stream, " -%c,", crm_long_options[i].val);
	} else {
	fputs(" ", stream);
	}
	fprintf(stream, " --%s%s\t%s\n", crm_long_options[i].name,
	crm_long_options[i].has_arg == optional_argument ? "[=value]" :
	crm_long_options[i].has_arg == required_argument ? "=value" : "",
	crm_long_options[i].desc ? crm_long_options[i].desc : "");
	}
	}

	} else if (crm_short_options) {
	fprintf(stream, "Usage: %s - %s\n", crm_system_name, crm_app_description);
	for (i = 0; crm_short_options[i] != 0; i++) {
	int has_arg = no_argument /* 0 */;

	if (crm_short_options[i + 1] == ':') {
	if (crm_short_options[i + 2] == ':')
	has_arg = optional_argument /* 2 */;
	else
	has_arg = required_argument /* 1 */;
	}

	fprintf(stream, " -%c %s\n", crm_short_options[i],
	has_arg == optional_argument ? "[value]" :
	has_arg == required_argument ? "{value}" : "");
	i += has_arg;
	}
	}

	fprintf(stream, "\nReport bugs to %s\n", PACKAGE_BUGREPORT);

	out:
	return crm_exit(exit_code);
	}

	void cib_ipc_servers_init(qb_ipcs_service_t **ipcs_ro,
	qb_ipcs_service_t **ipcs_rw,
	qb_ipcs_service_t **ipcs_shm,
	struct qb_ipcs_service_handlers *ro_cb,
	struct qb_ipcs_service_handlers *rw_cb)
	{
	*ipcs_ro = mainloop_add_ipc_server(cib_channel_ro, QB_IPC_NATIVE, ro_cb);
	*ipcs_rw = mainloop_add_ipc_server(cib_channel_rw, QB_IPC_NATIVE, rw_cb);
	*ipcs_shm = mainloop_add_ipc_server(cib_channel_shm, QB_IPC_SHM, rw_cb);

	if (ipcs_ro == NULL \|\| ipcs_rw == NULL \|\| *ipcs_shm == NULL) {
	crm_err("Failed to create cib servers: exiting and inhibiting respawn.");
	crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	}

	void cib_ipc_servers_destroy(qb_ipcs_service_t *ipcs_ro,
	qb_ipcs_service_t *ipcs_rw,
	qb_ipcs_service_t *ipcs_shm)
	{
	qb_ipcs_destroy(ipcs_ro);
	qb_ipcs_destroy(ipcs_rw);
	qb_ipcs_destroy(ipcs_shm);
	}

	qb_ipcs_service_t *
	crmd_ipc_server_init(struct qb_ipcs_service_handlers *cb)
	{
	return mainloop_add_ipc_server(CRM_SYSTEM_CRMD, QB_IPC_NATIVE, cb);
	}

	void
	attrd_ipc_server_init(qb_ipcs_service_t *ipcs, struct qb_ipcs_service_handlers cb)
	{
	*ipcs = mainloop_add_ipc_server(T_ATTRD, QB_IPC_NATIVE, cb);

	if (*ipcs == NULL) {
	crm_err("Failed to create attrd servers: exiting and inhibiting respawn.");
	crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	}

	void
	stonith_ipc_server_init(qb_ipcs_service_t *ipcs, struct qb_ipcs_service_handlers cb)
	{
	*ipcs = mainloop_add_ipc_server("stonith-ng", QB_IPC_NATIVE, cb);

	if (*ipcs == NULL) {
	crm_err("Failed to create stonith-ng servers: exiting and inhibiting respawn.");
	crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	}

	int
	attrd_update_delegate(crm_ipc_t * ipc, char command, const char host, const char name,
	const char value, const char section, const char set, const char dampen,
	const char *user_name, gboolean is_remote)
	{
	int rc = -ENOTCONN;
	int max = 5;
	enum crm_ipc_flags flags = crm_ipc_flags_none;
	xmlNode *update = create_xml_node(NULL, __FUNCTION__);

	static gboolean connected = TRUE;
	static crm_ipc_t *local_ipc = NULL;

	if (ipc == NULL && local_ipc == NULL) {
	local_ipc = crm_ipc_new(T_ATTRD, 0);
	flags \|= crm_ipc_client_response;
	connected = FALSE;
	}

	if (ipc == NULL) {
	ipc = local_ipc;
	}

	/* remap common aliases */
	if (safe_str_eq(section, "reboot")) {
	section = XML_CIB_TAG_STATUS;

	} else if (safe_str_eq(section, "forever")) {
	section = XML_CIB_TAG_NODES;
	}

	crm_xml_add(update, F_TYPE, T_ATTRD);
	crm_xml_add(update, F_ORIG, crm_system_name);

	if (name == NULL && command == 'U') {
	command = 'R';
	}

	switch (command) {
	case 'D':
	case 'U':
	case 'v':
	crm_xml_add(update, F_ATTRD_TASK, "update");
	crm_xml_add(update, F_ATTRD_ATTRIBUTE, name);
	break;
	case 'R':
	crm_xml_add(update, F_ATTRD_TASK, "refresh");
	break;
	case 'q':
	crm_xml_add(update, F_ATTRD_TASK, "query");
	break;
	case 'C':
	crm_xml_add(update, F_ATTRD_TASK, "peer-remove");
	break;
	}

	crm_xml_add(update, F_ATTRD_VALUE, value);
	crm_xml_add(update, F_ATTRD_DAMPEN, dampen);
	crm_xml_add(update, F_ATTRD_SECTION, section);
	crm_xml_add(update, F_ATTRD_HOST, host);
	crm_xml_add(update, F_ATTRD_SET, set);
	crm_xml_add_int(update, F_ATTRD_IS_REMOTE, is_remote);
	#if ENABLE_ACL
	if (user_name) {
	crm_xml_add(update, F_ATTRD_USER, user_name);
	}
	#endif

	while (max > 0) {
	if (connected == FALSE) {
	crm_info("Connecting to cluster... %d retries remaining", max);
	connected = crm_ipc_connect(ipc);
	}

	if (connected) {
	rc = crm_ipc_send(ipc, update, flags, 0, NULL);
	}

	if (ipc != local_ipc) {
	break;

	} else if (rc > 0) {
	break;

	} else if (rc == -EAGAIN \|\| rc == -EALREADY) {
	sleep(5 - max);
	max--;

	} else {
	crm_ipc_close(ipc);
	connected = FALSE;
	sleep(5 - max);
	max--;
	}
	}

	free_xml(update);
	if (rc > 0) {
	crm_debug("Sent update: %s=%s for %s", name, value, host ? host : "localhost");
	rc = pcmk_ok;

	} else {
	crm_debug("Could not send update %s=%s for %s: %s (%d)", name, value,
	host ? host : "localhost", pcmk_strerror(rc), rc);
	}
	return rc;
	}

	#define FAKE_TE_ID "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
	static void
	append_digest(lrmd_event_data_t * op, xmlNode * update, const char version, const char magic,
	int level)
	{
	/* this will enable us to later determine that the
	* resource's parameters have changed and we should force
	* a restart
	*/
	char *digest = NULL;
	xmlNode *args_xml = NULL;

	if (op->params == NULL) {
	return;
	}

	args_xml = create_xml_node(NULL, XML_TAG_PARAMS);
	g_hash_table_foreach(op->params, hash2field, args_xml);
	filter_action_parameters(args_xml, version);
	digest = calculate_operation_digest(args_xml, version);

	#if 0
	if (level < get_crm_log_level()
	&& op->interval == 0 && crm_str_eq(op->op_type, CRMD_ACTION_START, TRUE)) {
	char *digest_source = dump_xml_unformatted(args_xml);

	do_crm_log(level, "Calculated digest %s for %s (%s). Source: %s\n",
	digest, ID(update), magic, digest_source);
	free(digest_source);
	}
	#endif
	crm_xml_add(update, XML_LRM_ATTR_OP_DIGEST, digest);

	free_xml(args_xml);
	free(digest);
	}

	int
	rsc_op_expected_rc(lrmd_event_data_t * op)
	{
	int rc = 0;

	if (op && op->user_data) {
	int dummy = 0;
	char *uuid = NULL;

	decode_transition_key(op->user_data, &uuid, &dummy, &dummy, &rc);
	free(uuid);
	}
	return rc;
	}

	gboolean
	did_rsc_op_fail(lrmd_event_data_t * op, int target_rc)
	{
	switch (op->op_status) {
	case PCMK_LRM_OP_CANCELLED:
	case PCMK_LRM_OP_PENDING:
	return FALSE;
	break;

	case PCMK_LRM_OP_NOTSUPPORTED:
	case PCMK_LRM_OP_TIMEOUT:
	case PCMK_LRM_OP_ERROR:
	return TRUE;
	break;

	default:
	if (target_rc != op->rc) {
	return TRUE;
	}
	}

	return FALSE;
	}

	xmlNode *
	create_operation_update(xmlNode * parent, lrmd_event_data_t * op, const char *caller_version,
	int target_rc, const char *origin, int level)
	{
	char *key = NULL;
	char *magic = NULL;
	char *op_id = NULL;
	char *op_id_additional = NULL;
	char *local_user_data = NULL;

	xmlNode *xml_op = NULL;
	const char *task = NULL;
	gboolean dc_munges_migrate_ops = (compare_version(caller_version, "3.0.3") < 0);
	gboolean dc_needs_unique_ops = (compare_version(caller_version, "3.0.6") < 0);

	CRM_CHECK(op != NULL, return NULL);
	do_crm_log(level, "%s: Updating resource %s after %s op %s (interval=%d)",
	origin, op->rsc_id, op->op_type, services_lrm_status_str(op->op_status),
	op->interval);

	crm_trace("DC version: %s", caller_version);

	task = op->op_type;
	/* remap the task name under various scenarios
	* this makes life easier for the PE when its trying determin the current state
	*/
	if (crm_str_eq(task, "reload", TRUE)) {
	if (op->op_status == PCMK_LRM_OP_DONE) {
	task = CRMD_ACTION_START;
	} else {
	task = CRMD_ACTION_STATUS;
	}

	} else if (dc_munges_migrate_ops && crm_str_eq(task, CRMD_ACTION_MIGRATE, TRUE)) {
	/* if the migrate_from fails it will have enough info to do the right thing */
	if (op->op_status == PCMK_LRM_OP_DONE) {
	task = CRMD_ACTION_STOP;
	} else {
	task = CRMD_ACTION_STATUS;
	}

	} else if (dc_munges_migrate_ops
	&& op->op_status == PCMK_LRM_OP_DONE
	&& crm_str_eq(task, CRMD_ACTION_MIGRATED, TRUE)) {
	task = CRMD_ACTION_START;
	}

	key = generate_op_key(op->rsc_id, task, op->interval);
	if (dc_needs_unique_ops && op->interval > 0) {
	op_id = strdup(key);

	} else if (crm_str_eq(task, CRMD_ACTION_NOTIFY, TRUE)) {
	const char *n_type = crm_meta_value(op->params, "notify_type");
	const char *n_task = crm_meta_value(op->params, "notify_operation");

	CRM_LOG_ASSERT(n_type != NULL);
	CRM_LOG_ASSERT(n_task != NULL);
	op_id = generate_notify_key(op->rsc_id, n_type, n_task);

	/* these are not yet allowed to fail */
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = 0;

	} else if (did_rsc_op_fail(op, target_rc)) {
	op_id = generate_op_key(op->rsc_id, "last_failure", 0);
	if (op->interval == 0) {
	/* Ensure 'last' gets updated too in case recording-pending="true" */
	op_id_additional = generate_op_key(op->rsc_id, "last", 0);
	}

	} else if (op->interval > 0) {
	op_id = strdup(key);

	} else {
	op_id = generate_op_key(op->rsc_id, "last", 0);
	}

	again:
	xml_op = find_entity(parent, XML_LRM_TAG_RSC_OP, op_id);
	if (xml_op == NULL) {
	xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP);
	}

	if (op->user_data == NULL) {
	crm_debug("Generating fake transition key for:"
	" %s_%s_%d %d from %s",
	op->rsc_id, op->op_type, op->interval, op->call_id, origin);
	local_user_data = generate_transition_key(-1, op->call_id, target_rc, FAKE_TE_ID);
	op->user_data = local_user_data;
	}

	if(magic == NULL) {
	magic = generate_transition_magic(op->user_data, op->op_status, op->rc);
	}

	crm_xml_add(xml_op, XML_ATTR_ID, op_id);
	crm_xml_add(xml_op, XML_LRM_ATTR_TASK_KEY, key);
	crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task);
	crm_xml_add(xml_op, XML_ATTR_ORIGIN, origin);
	crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, caller_version);
	crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY, op->user_data);
	crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC, magic);

	crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID, op->call_id);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op->rc);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, op->op_status);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_INTERVAL, op->interval);

	if (compare_version("2.1", caller_version) <= 0) {
	if (op->t_run \|\| op->t_rcchange \|\| op->exec_time \|\| op->queue_time) {
	crm_trace("Timing data (%s_%s_%d): last=%lu change=%lu exec=%lu queue=%lu",
	op->rsc_id, op->op_type, op->interval,
	op->t_run, op->t_rcchange, op->exec_time, op->queue_time);

	if (op->interval == 0) {
	/* The values are the same for non-recurring ops */
	crm_xml_add_int(xml_op, XML_RSC_OP_LAST_RUN, op->t_run);
	crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_run);

	} else if(op->t_rcchange) {
	/* last-run is not accurate for recurring ops */
	crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_rcchange);

	} else {
	/* ...but is better than nothing otherwise */
	crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_run);
	}

	crm_xml_add_int(xml_op, XML_RSC_OP_T_EXEC, op->exec_time);
	crm_xml_add_int(xml_op, XML_RSC_OP_T_QUEUE, op->queue_time);
	}
	}

	if (crm_str_eq(op->op_type, CRMD_ACTION_MIGRATE, TRUE)
	\|\| crm_str_eq(op->op_type, CRMD_ACTION_MIGRATED, TRUE)) {
	/*
	* Record migrate_source and migrate_target always for migrate ops.
	*/
	const char *name = XML_LRM_ATTR_MIGRATE_SOURCE;

	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));

	name = XML_LRM_ATTR_MIGRATE_TARGET;
	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
	}

	append_digest(op, xml_op, caller_version, magic, LOG_DEBUG);

	if (op_id_additional) {
	free(op_id);
	op_id = op_id_additional;
	op_id_additional = NULL;
	goto again;
	}

	if (local_user_data) {
	free(local_user_data);
	op->user_data = NULL;
	}
	free(magic);
	free(op_id);
	free(key);
	return xml_op;
	}

	bool
	pcmk_acl_required(const char *user)
	{
	#if ENABLE_ACL
	if(user == NULL \|\| strlen(user) == 0) {
	crm_trace("no user set");
	return FALSE;

	} else if (strcmp(user, CRM_DAEMON_USER) == 0) {
	return FALSE;

	} else if (strcmp(user, "root") == 0) {
	return FALSE;
	}
	crm_trace("acls required for %s", user);
	return TRUE;
	#else
	crm_trace("acls not supported");
	return FALSE;
	#endif
	}

	#if ENABLE_ACL
	char *
	uid2username(uid_t uid)
	{
	struct passwd *pwent = getpwuid(uid);

	if (pwent == NULL) {
	crm_perror(LOG_ERR, "Cannot get password entry of uid: %d", uid);
	return NULL;

	} else {
	return strdup(pwent->pw_name);
	}
	}

	const char *
	crm_acl_get_set_user(xmlNode * request, const char field, const char peer_user)
	{
	/* field is only checked for backwards compatibility */
	static const char *effective_user = NULL;
	const char *requested_user = NULL;
	const char *user = NULL;

	if(effective_user == NULL) {
	effective_user = uid2username(geteuid());
	}

	requested_user = crm_element_value(request, XML_ACL_TAG_USER);
	if(requested_user == NULL) {
	requested_user = crm_element_value(request, field);
	}

	if (is_privileged(effective_user) == FALSE) {
	/* We're not running as a privileged user, set or overwrite any existing value for $XML_ACL_TAG_USER */
	user = effective_user;

	} else if(peer_user == NULL && requested_user == NULL) {
	/* No user known or requested, use 'effective_user' and make sure one is set for the request */
	user = effective_user;

	} else if(peer_user == NULL) {
	/* No user known, trusting 'requested_user' */
	user = requested_user;

	} else if (is_privileged(peer_user) == FALSE) {
	/* The peer is not a privileged user, set or overwrite any existing value for $XML_ACL_TAG_USER */
	user = peer_user;

	} else if (requested_user == NULL) {
	/* Even if we're privileged, make sure there is always a value set */
	user = peer_user;

	} else {
	/* Legal delegation to 'requested_user' */
	user = requested_user;
	}

	/* Yes, pointer comparision */
	if(user != crm_element_value(request, XML_ACL_TAG_USER)) {
	crm_xml_add(request, XML_ACL_TAG_USER, user);
	}

	if(field != NULL && user != crm_element_value(request, field)) {
	crm_xml_add(request, field, user);
	}

	return requested_user;
	}

	void
	determine_request_user(const char user, xmlNode request, const char *field)
	{
	/* Get our internal validation out of the way first */
	CRM_CHECK(user != NULL && request != NULL && field != NULL, return);

	/* If our peer is a privileged user, we might be doing something on behalf of someone else */
	if (is_privileged(user) == FALSE) {
	/* We're not a privileged user, set or overwrite any existing value for $field */
	crm_xml_replace(request, field, user);

	} else if (crm_element_value(request, field) == NULL) {
	/* Even if we're privileged, make sure there is always a value set */
	crm_xml_replace(request, field, user);

	/* } else { Legal delegation */
	}

	crm_trace("Processing msg as user '%s'", crm_element_value(request, field));
	}
	#endif

	/*
	* This re-implements g_str_hash as it was prior to glib2-2.28:
	*
	* http://git.gnome.org/browse/glib/commit/?id=354d655ba8a54b754cb5a3efb42767327775696c
	*
	* Note that the new g_str_hash is presumably a better hash (it's actually
	* a correct implementation of DJB's hash), but we need to preserve existing
	* behaviour, because the hash key ultimately determines the "sort" order
	* when iterating through GHashTables, which affects allocation of scores to
	* clone instances when iterating through rsc->allowed_nodes. It (somehow)
	* also appears to have some minor impact on the ordering of a few
	* pseudo_event IDs in the transition graph.
	*/
	guint
	g_str_hash_traditional(gconstpointer v)
	{
	const signed char *p;
	guint32 h = 0;

	for (p = v; *p != '\0'; p++)
	h = (h << 5) - h + *p;

	return h;
	}

	guint
	crm_strcase_hash(gconstpointer v)
	{
	const signed char *p;
	guint32 h = 0;

	for (p = v; *p != '\0'; p++)
	h = (h << 5) - h + g_ascii_tolower(*p);

	return h;
	}

	void *
	find_library_function(void *handle, const char lib, const char *fn, gboolean fatal)
	{
	char *error;
	void *a_function;

	if (*handle == NULL) {
	*handle = dlopen(lib, RTLD_LAZY);
	}

	if (!(*handle)) {
	crm_err("%sCould not open %s: %s", fatal ? "Fatal: " : "", lib, dlerror());
	if (fatal) {
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	return NULL;
	}

	a_function = dlsym(*handle, fn);
	if ((error = dlerror()) != NULL) {
	crm_err("%sCould not find %s in %s: %s", fatal ? "Fatal: " : "", fn, lib, error);
	if (fatal) {
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	}

	return a_function;
	}

	char *
	add_list_element(char list, const char value)
	{
	int len = 0;
	int last = 0;

	if (value == NULL) {
	return list;
	}
	if (list) {
	last = strlen(list);
	}
	len = last + 2; /* +1 space, +1 EOS */
	len += strlen(value);
	list = realloc(list, len);
	sprintf(list + last, " %s", value);
	return list;
	}

	void *
	convert_const_pointer(const void *ptr)
	{
	/* Worst function ever */
	return (void *)ptr;
	}

	#ifdef HAVE_UUID_UUID_H
	# include <uuid/uuid.h>
	#endif

	char *
	crm_generate_uuid(void)
	{
	unsigned char uuid[16];
	char buffer = malloc(37); / Including NUL byte */

	uuid_generate(uuid);
	uuid_unparse(uuid, buffer);
	return buffer;
	}

	#include <md5.h>

	char *
	crm_md5sum(const char *buffer)
	{
	int lpc = 0, len = 0;
	char *digest = NULL;
	unsigned char raw_digest[MD5_DIGEST_SIZE];

	if(buffer != NULL) {
	len = strlen(buffer);
	}

	crm_trace("Beginning digest of %d bytes", len);
	digest = malloc(2 * MD5_DIGEST_SIZE + 1);
	if(digest) {
	md5_buffer(buffer, len, raw_digest);
	for (lpc = 0; lpc < MD5_DIGEST_SIZE; lpc++) {
	sprintf(digest + (2 * lpc), "%02x", raw_digest[lpc]);
	}
	digest[(2 * MD5_DIGEST_SIZE)] = 0;
	crm_trace("Digest %s.", digest);

	} else {
	crm_err("Could not create digest");
	}
	return digest;
	}

	#include <time.h>
	#include <bzlib.h>

	bool
	crm_compress_string(const char data, int length, int max, char result, unsigned int result_len)
	{
	int rc;
	char *compressed = NULL;
	char *uncompressed = strdup(data);
	struct timespec after_t;
	struct timespec before_t;

	if(max == 0) {
	max = (length * 1.1) + 600; /* recomended size */
	}

	#ifdef CLOCK_MONOTONIC
	clock_gettime(CLOCK_MONOTONIC, &before_t);
	#endif

	/* coverity[returned_null] Ignore */
	compressed = malloc(max);

	*result_len = max;
	rc = BZ2_bzBuffToBuffCompress(compressed, result_len, uncompressed, length, CRM_BZ2_BLOCKS, 0,
	CRM_BZ2_WORK);

	free(uncompressed);

	if (rc != BZ_OK) {
	crm_err("Compression of %d bytes failed: %s (%d)", length, bz2_strerror(rc), rc);
	free(compressed);
	return FALSE;
	}

	#ifdef CLOCK_MONOTONIC
	clock_gettime(CLOCK_MONOTONIC, &after_t);

	crm_info("Compressed %d bytes into %d (ratio %d:1) in %dms",
	length, result_len, length / (result_len),
	(after_t.tv_sec - before_t.tv_sec) * 1000 + (after_t.tv_nsec -
	before_t.tv_nsec) / 1000000);
	#else
	crm_info("Compressed %d bytes into %d (ratio %d:1)",
	length, result_len, length / (result_len));
	#endif

	*result = compressed;
	return TRUE;
	}
	+
	+#ifdef HAVE_GNUTLS_GNUTLS_H
	+void
	+crm_gnutls_global_init(void)
	+{
	+ signal(SIGPIPE, SIG_IGN);
	+ gnutls_global_init();
	+}
	+#endif
	+
	diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c
	index 50cfcf6ab8..7ba1648b8d 100644
	--- a/lib/lrmd/lrmd_client.c
	+++ b/lib/lrmd/lrmd_client.c
	@@ -1,2042 +1,2042 @@
	/*
	* Copyright (c) 2012 David Vossel <dvossel@redhat.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	#include <crm_internal.h>

	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <ctype.h>

	#include <sys/types.h>
	#include <sys/wait.h>

	#include <glib.h>
	#include <dirent.h>

	#include <crm/crm.h>
	#include <crm/lrmd.h>
	#include <crm/services.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/ipcs.h>
	#include <crm/msg_xml.h>

	#include <crm/stonith-ng.h>

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# undef KEYFILE
	# include <gnutls/gnutls.h>
	#endif

	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <netinet/ip.h>
	#include <arpa/inet.h>
	#include <netdb.h>

	CRM_TRACE_INIT_DATA(lrmd);

	static int lrmd_api_disconnect(lrmd_t * lrmd);
	static int lrmd_api_is_connected(lrmd_t * lrmd);

	/* IPC proxy functions */
	int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
	static void lrmd_internal_proxy_dispatch(lrmd_t lrmd, xmlNode msg);
	void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void userdata, void (callback)(lrmd_t lrmd, void userdata, xmlNode *msg));

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000 /* 5 seconds */
	gnutls_psk_client_credentials_t psk_cred_s;
	int lrmd_tls_set_key(gnutls_datum_t * key);
	static void lrmd_tls_disconnect(lrmd_t * lrmd);
	static int global_remote_msg_id = 0;
	int lrmd_tls_send_msg(crm_remote_t * session, xmlNode * msg, uint32_t id, const char *msg_type);
	static void lrmd_tls_connection_destroy(gpointer userdata);
	#endif

	typedef struct lrmd_private_s {
	enum client_type type;
	char *token;
	mainloop_io_t *source;

	/* IPC parameters */
	crm_ipc_t *ipc;

	crm_remote_t *remote;

	/* Extra TLS parameters */
	char *remote_nodename;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	char *server;
	int port;
	gnutls_psk_client_credentials_t psk_cred_c;

	int sock;
	/* since tls requires a round trip across the network for a
	* request/reply, there are times where we just want to be able
	* to send a request from the client and not wait around (or even care
	* about) what the reply is. */
	int expected_late_replies;
	GList *pending_notify;
	crm_trigger_t *process_notify;
	#endif

	lrmd_event_callback callback;

	/* Internal IPC proxy msg passing for remote guests */
	void (proxy_callback)(lrmd_t lrmd, void userdata, xmlNode msg);
	void *proxy_callback_userdata;
	} lrmd_private_t;

	static lrmd_list_t *
	lrmd_list_add(lrmd_list_t * head, const char *value)
	{
	lrmd_list_t p, end;

	p = calloc(1, sizeof(lrmd_list_t));
	p->val = strdup(value);

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	void
	lrmd_list_freeall(lrmd_list_t * head)
	{
	lrmd_list_t *p;

	while (head) {
	char val = (char )head->val;

	p = head->next;
	free(val);
	free(head);
	head = p;
	}
	}

	lrmd_key_value_t *
	lrmd_key_value_add(lrmd_key_value_t * head, const char key, const char value)
	{
	lrmd_key_value_t p, end;

	p = calloc(1, sizeof(lrmd_key_value_t));
	p->key = strdup(key);
	p->value = strdup(value);

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	void
	lrmd_key_value_freeall(lrmd_key_value_t * head)
	{
	lrmd_key_value_t *p;

	while (head) {
	p = head->next;
	free(head->key);
	free(head->value);
	free(head);
	head = p;
	}
	}

	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	g_hash_table_replace(user_data, strdup(key), strdup(value));
	}

	lrmd_event_data_t *
	lrmd_copy_event(lrmd_event_data_t * event)
	{
	lrmd_event_data_t *copy = NULL;

	copy = calloc(1, sizeof(lrmd_event_data_t));

	/* This will get all the int values.
	* we just have to be careful not to leave any
	* dangling pointers to strings. */
	memcpy(copy, event, sizeof(lrmd_event_data_t));

	copy->rsc_id = event->rsc_id ? strdup(event->rsc_id) : NULL;
	copy->op_type = event->op_type ? strdup(event->op_type) : NULL;
	copy->user_data = event->user_data ? strdup(event->user_data) : NULL;
	copy->output = event->output ? strdup(event->output) : NULL;
	copy->remote_nodename = event->remote_nodename ? strdup(event->remote_nodename) : NULL;

	if (event->params) {
	copy->params = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	if (copy->params != NULL) {
	g_hash_table_foreach(event->params, dup_attr, copy->params);
	}
	}

	return copy;
	}

	void
	lrmd_free_event(lrmd_event_data_t * event)
	{
	if (!event) {
	return;
	}

	/* free gives me grief if i try to cast */
	free((char *)event->rsc_id);
	free((char *)event->op_type);
	free((char *)event->user_data);
	free((char *)event->output);
	free((char *)event->remote_nodename);
	if (event->params) {
	g_hash_table_destroy(event->params);
	}
	free(event);
	}

	static int
	lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
	{
	const char *type;
	const char *proxy_session = crm_element_value(msg, F_LRMD_IPC_SESSION);
	lrmd_private_t *native = lrmd->private;
	lrmd_event_data_t event = { 0, };

	if (proxy_session != NULL) {
	/* this is proxy business */
	lrmd_internal_proxy_dispatch(lrmd, msg);
	return 1;
	} else if (!native->callback) {
	/* no callback set */
	crm_trace("notify event received but client has not set callback");
	return 1;
	}

	event.remote_nodename = native->remote_nodename;
	type = crm_element_value(msg, F_LRMD_OPERATION);
	crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
	event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);

	if (crm_str_eq(type, LRMD_OP_RSC_REG, TRUE)) {
	event.type = lrmd_event_register;
	} else if (crm_str_eq(type, LRMD_OP_RSC_UNREG, TRUE)) {
	event.type = lrmd_event_unregister;
	} else if (crm_str_eq(type, LRMD_OP_RSC_EXEC, TRUE)) {
	crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
	crm_element_value_int(msg, F_LRMD_RSC_INTERVAL, &event.interval);
	crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
	crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
	crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
	crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);

	crm_element_value_int(msg, F_LRMD_RSC_RUN_TIME, (int *)&event.t_run);
	crm_element_value_int(msg, F_LRMD_RSC_RCCHANGE_TIME, (int *)&event.t_rcchange);
	crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
	crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);

	event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
	event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
	event.output = crm_element_value(msg, F_LRMD_RSC_OUTPUT);
	event.type = lrmd_event_exec_complete;

	event.params = xml2list(msg);
	} else if (crm_str_eq(type, LRMD_OP_NEW_CLIENT, TRUE)) {
	event.type = lrmd_event_new_client;
	} else if (crm_str_eq(type, LRMD_OP_POKE, TRUE)) {
	event.type = lrmd_event_poke;
	} else {
	return 1;
	}

	crm_trace("op %s notify event received", type);
	native->callback(&event);

	if (event.params) {
	g_hash_table_destroy(event.params);
	}
	return 1;
	}

	static int
	lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->private;
	xmlNode *msg;
	int rc;

	if (!native->callback) {
	/* no callback set */
	return 1;
	}

	msg = string2xml(buffer);
	rc = lrmd_dispatch_internal(lrmd, msg);
	free_xml(msg);
	return rc;
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	lrmd_free_xml(gpointer userdata)
	{
	free_xml((xmlNode *) userdata);
	}

	static int
	lrmd_tls_connected(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->private;

	if (native->remote->tls_session) {
	return TRUE;
	}

	return FALSE;
	}

	static int
	lrmd_tls_dispatch(gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->private;
	xmlNode *xml = NULL;
	int rc = 0;
	int disconnected = 0;

	if (lrmd_tls_connected(lrmd) == FALSE) {
	crm_trace("tls dispatch triggered after disconnect");
	return 0;
	}

	crm_trace("tls_dispatch triggered");

	/* First check if there are any pending notifies to process that came
	* while we were waiting for replies earlier. */
	if (native->pending_notify) {
	GList *iter = NULL;

	crm_trace("Processing pending notifies");
	for (iter = native->pending_notify; iter; iter = iter->next) {
	lrmd_dispatch_internal(lrmd, iter->data);
	}
	g_list_free_full(native->pending_notify, lrmd_free_xml);
	native->pending_notify = NULL;
	}

	/* Next read the current buffer and see if there are any messages to handle. */
	rc = crm_remote_ready(native->remote, 0);
	if (rc == 0) {
	/* nothing to read, see if any full messages are already in buffer. */
	xml = crm_remote_parse_buffer(native->remote);
	} else if (rc < 0) {
	disconnected = 1;
	} else {
	crm_remote_recv(native->remote, -1, &disconnected);
	xml = crm_remote_parse_buffer(native->remote);
	}
	while (xml) {
	const char *msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
	if (safe_str_eq(msg_type, "notify")) {
	lrmd_dispatch_internal(lrmd, xml);
	} else if (safe_str_eq(msg_type, "reply")) {
	if (native->expected_late_replies > 0) {
	native->expected_late_replies--;
	} else {
	int reply_id = 0;
	crm_element_value_int(xml, F_LRMD_CALLID, &reply_id);
	/* if this happens, we want to know about it */
	crm_err("Got outdated reply %d", reply_id);
	}
	}
	free_xml(xml);
	xml = crm_remote_parse_buffer(native->remote);
	}

	if (disconnected) {
	crm_info("Server disconnected while reading remote server msg.");
	lrmd_tls_disconnect(lrmd);
	return 0;
	}
	return 1;
	}
	#endif

	/* Not used with mainloop */
	int
	lrmd_poll(lrmd_t * lrmd, int timeout)
	{
	lrmd_private_t *native = lrmd->private;

	switch (native->type) {
	case CRM_CLIENT_IPC:
	return crm_ipc_ready(native->ipc);

	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	if (native->pending_notify) {
	return 1;
	}

	return crm_remote_ready(native->remote, 0);
	#endif
	default:
	crm_err("Unsupported connection type: %d", native->type);
	}

	return 0;
	}

	/* Not used with mainloop */
	bool
	lrmd_dispatch(lrmd_t * lrmd)
	{
	lrmd_private_t *private = NULL;

	CRM_ASSERT(lrmd != NULL);

	private = lrmd->private;
	switch (private->type) {
	case CRM_CLIENT_IPC:
	while (crm_ipc_ready(private->ipc)) {
	if (crm_ipc_read(private->ipc) > 0) {
	const char *msg = crm_ipc_buffer(private->ipc);

	lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
	}
	}
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	lrmd_tls_dispatch(lrmd);
	break;
	#endif
	default:
	crm_err("Unsupported connection type: %d", private->type);
	}

	if (lrmd_api_is_connected(lrmd) == FALSE) {
	crm_err("Connection closed");
	return FALSE;
	}

	return TRUE;
	}

	static xmlNode *
	lrmd_create_op(const char token, const char op, xmlNode * data, enum lrmd_call_options options)
	{
	xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");

	CRM_CHECK(op_msg != NULL, return NULL);
	CRM_CHECK(token != NULL, return NULL);

	crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");

	crm_xml_add(op_msg, F_TYPE, T_LRMD);
	crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
	crm_xml_add(op_msg, F_LRMD_OPERATION, op);
	crm_trace("Sending call options: %.8lx, %d", (long)options, options);
	crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);

	if (data != NULL) {
	add_message_xml(op_msg, F_LRMD_CALLDATA, data);
	}

	return op_msg;
	}

	static void
	lrmd_ipc_connection_destroy(gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->private;

	crm_info("IPC connection destroyed");

	/* Prevent these from being cleaned up in lrmd_api_disconnect() */
	native->ipc = NULL;
	native->source = NULL;

	if (native->callback) {
	lrmd_event_data_t event = { 0, };
	event.type = lrmd_event_disconnect;
	event.remote_nodename = native->remote_nodename;
	native->callback(&event);
	}
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	lrmd_tls_connection_destroy(gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->private;

	crm_info("TLS connection destroyed");

	if (native->remote->tls_session) {
	gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	}
	if (native->psk_cred_c) {
	gnutls_psk_free_client_credentials(native->psk_cred_c);
	}
	if (native->sock) {
	close(native->sock);
	}
	if (native->process_notify) {
	mainloop_destroy_trigger(native->process_notify);
	native->process_notify = NULL;
	}
	if (native->pending_notify) {
	g_list_free_full(native->pending_notify, lrmd_free_xml);
	native->pending_notify = NULL;
	}

	free(native->remote->buffer);
	native->remote->buffer = NULL;
	native->source = 0;
	native->sock = 0;
	native->psk_cred_c = NULL;
	native->remote->tls_session = NULL;
	native->sock = 0;

	if (native->callback) {
	lrmd_event_data_t event = { 0, };
	event.remote_nodename = native->remote_nodename;
	event.type = lrmd_event_disconnect;
	native->callback(&event);
	}
	return;
	}

	int
	lrmd_tls_send_msg(crm_remote_t * session, xmlNode * msg, uint32_t id, const char *msg_type)
	{
	int rc = -1;

	crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
	crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);

	rc = crm_remote_send(session, msg);

	if (rc < 0) {
	crm_err("Failed to send remote lrmd tls msg, rc = %d", rc);
	return rc;
	}

	return rc;
	}

	static xmlNode *
	lrmd_tls_recv_reply(lrmd_t * lrmd, int total_timeout, int expected_reply_id, int *disconnected)
	{
	lrmd_private_t *native = lrmd->private;
	xmlNode *xml = NULL;
	time_t start = time(NULL);
	const char *msg_type = NULL;
	int reply_id = 0;
	int remaining_timeout = 0;

	/* A timeout of 0 here makes no sense. We have to wait a period of time
	* for the response to come back. If -1 or 0, default to 10 seconds. */
	if (total_timeout <= 0) {
	total_timeout = 10000;
	}

	while (!xml) {

	xml = crm_remote_parse_buffer(native->remote);
	if (!xml) {
	/* read some more off the tls buffer if we still have time left. */
	if (remaining_timeout) {
	remaining_timeout = remaining_timeout - ((time(NULL) - start) * 1000);
	} else {
	remaining_timeout = total_timeout;
	}
	if (remaining_timeout <= 0) {
	return NULL;
	}

	crm_remote_recv(native->remote, remaining_timeout, disconnected);
	xml = crm_remote_parse_buffer(native->remote);
	if (!xml \|\| *disconnected) {
	return NULL;
	}
	}

	CRM_ASSERT(xml != NULL);

	crm_element_value_int(xml, F_LRMD_REMOTE_MSG_ID, &reply_id);
	msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);

	if (!msg_type) {
	crm_err("Empty msg type received while waiting for reply");
	free_xml(xml);
	xml = NULL;
	} else if (safe_str_eq(msg_type, "notify")) {
	/* got a notify while waiting for reply, trigger the notify to be processed later */
	crm_info("queueing notify");
	native->pending_notify = g_list_append(native->pending_notify, xml);
	if (native->process_notify) {
	crm_info("notify trigger set.");
	mainloop_set_trigger(native->process_notify);
	}
	xml = NULL;
	} else if (safe_str_neq(msg_type, "reply")) {
	/* msg isn't a reply, make some noise */
	crm_err("Expected a reply, got %s", msg_type);
	free_xml(xml);
	xml = NULL;
	} else if (reply_id != expected_reply_id) {
	if (native->expected_late_replies > 0) {
	native->expected_late_replies--;
	} else {
	crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
	}
	free_xml(xml);
	xml = NULL;
	}
	}

	if (native->remote->buffer && native->process_notify) {
	mainloop_set_trigger(native->process_notify);
	}

	return xml;
	}

	static int
	lrmd_tls_send(lrmd_t * lrmd, xmlNode * msg)
	{
	int rc = 0;
	lrmd_private_t *native = lrmd->private;

	global_remote_msg_id++;
	if (global_remote_msg_id <= 0) {
	global_remote_msg_id = 1;
	}

	rc = lrmd_tls_send_msg(native->remote, msg, global_remote_msg_id, "request");
	if (rc <= 0) {
	crm_err("Remote lrmd send failed, disconnecting");
	lrmd_tls_disconnect(lrmd);
	return -ENOTCONN;
	}
	return pcmk_ok;
	}

	static int
	lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
	{
	int rc = 0;
	int disconnected = 0;
	xmlNode *xml = NULL;

	if (lrmd_tls_connected(lrmd) == FALSE) {
	return -1;
	}

	rc = lrmd_tls_send(lrmd, msg);
	if (rc < 0) {
	return rc;
	}

	xml = lrmd_tls_recv_reply(lrmd, timeout, global_remote_msg_id, &disconnected);

	if (disconnected) {
	crm_err("Remote lrmd server disconnected while waiting for reply with id %d. ",
	global_remote_msg_id);
	lrmd_tls_disconnect(lrmd);
	rc = -ENOTCONN;
	} else if (!xml) {
	crm_err("Remote lrmd never received reply for request id %d. timeout: %dms ",
	global_remote_msg_id, timeout);
	rc = -ECOMM;
	}

	if (reply) {
	*reply = xml;
	} else {
	free_xml(xml);
	}

	return rc;
	}
	#endif

	static int
	lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
	{
	int rc = -1;
	lrmd_private_t *native = lrmd->private;

	switch (native->type) {
	case CRM_CLIENT_IPC:
	rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
	break;
	#endif
	default:
	crm_err("Unsupported connection type: %d", native->type);
	}

	return rc;
	}

	static int
	lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
	{
	int rc = -1;
	lrmd_private_t *native = lrmd->private;

	switch (native->type) {
	case CRM_CLIENT_IPC:
	rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	rc = lrmd_tls_send(lrmd, msg);
	if (rc == pcmk_ok) {
	/* we don't want to wait around for the reply, but
	* since the request/reply protocol needs to behave the same
	* as libqb, a reply will eventually come later anyway. */
	native->expected_late_replies++;
	}
	break;
	#endif
	default:
	crm_err("Unsupported connection type: %d", native->type);
	}

	return rc;
	}

	static int
	lrmd_api_is_connected(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->private;

	switch (native->type) {
	case CRM_CLIENT_IPC:
	return crm_ipc_connected(native->ipc);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	return lrmd_tls_connected(lrmd);
	break;
	#endif
	default:
	crm_err("Unsupported connection type: %d", native->type);
	}

	return 0;
	}

	static int
	lrmd_send_command(lrmd_t * lrmd, const char op, xmlNode data, xmlNode ** output_data, int timeout, /* ms. defaults to 1000 if set to 0 */
	enum lrmd_call_options options, gboolean expect_reply)
	{ /* TODO we need to reduce usage of this boolean */
	int rc = pcmk_ok;
	int reply_id = -1;
	lrmd_private_t *native = lrmd->private;
	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	if (!lrmd_api_is_connected(lrmd)) {
	return -ENOTCONN;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	CRM_CHECK(native->token != NULL,;
	);
	crm_trace("sending %s op to lrmd", op);

	op_msg = lrmd_create_op(native->token, op, data, options);

	if (op_msg == NULL) {
	return -EINVAL;
	}

	crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);

	if (expect_reply) {
	rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
	} else {
	rc = lrmd_send_xml_no_reply(lrmd, op_msg);
	goto done;
	}

	if (rc < 0) {
	crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
	rc = -ECOMM;
	goto done;

	} else if(op_reply == NULL) {
	rc = -ENOMSG;
	goto done;
	}

	rc = pcmk_ok;
	crm_element_value_int(op_reply, F_LRMD_CALLID, &reply_id);
	crm_trace("%s op reply received", op);
	if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
	rc = -ENOMSG;
	goto done;
	}

	crm_log_xml_trace(op_reply, "Reply");

	if (output_data) {
	*output_data = op_reply;
	op_reply = NULL; /* Prevent subsequent free */
	}

	done:
	if (lrmd_api_is_connected(lrmd) == FALSE) {
	crm_err("LRMD disconnected");
	}

	free_xml(op_msg);
	free_xml(op_reply);
	return rc;
	}

	static int
	lrmd_api_poke_connection(lrmd_t * lrmd)
	{
	int rc;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0, FALSE);
	free_xml(data);

	return rc;
	}

	static int
	lrmd_handshake(lrmd_t * lrmd, const char *name)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->private;
	xmlNode *reply = NULL;
	xmlNode *hello = create_xml_node(NULL, "lrmd_command");

	crm_xml_add(hello, F_TYPE, T_LRMD);
	crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(hello, F_LRMD_CLIENTNAME, name);
	crm_xml_add(hello, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);

	/* advertise that we are a proxy provider */
	if (native->proxy_callback) {
	crm_xml_add(hello, F_LRMD_IS_IPC_PROVIDER, "true");
	}

	rc = lrmd_send_xml(lrmd, hello, -1, &reply);

	if (rc < 0) {
	crm_perror(LOG_DEBUG, "Couldn't complete registration with the lrmd API: %d", rc);
	rc = -ECOMM;
	} else if (reply == NULL) {
	crm_err("Did not receive registration reply");
	rc = -EPROTO;
	} else {
	const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
	const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);

	crm_element_value_int(reply, F_LRMD_RC, &rc);

	if (rc == -EPROTO) {
	crm_err("LRMD protocol mismatch client version %s, server version %s",
	LRMD_PROTOCOL_VERSION, crm_element_value(reply, F_LRMD_PROTOCOL_VERSION));
	crm_log_xml_err(reply, "Protocol Error");

	} else if (safe_str_neq(msg_type, CRM_OP_REGISTER)) {
	crm_err("Invalid registration message: %s", msg_type);
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;
	} else if (tmp_ticket == NULL) {
	crm_err("No registration token provided");
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;
	} else {
	crm_trace("Obtained registration token: %s", tmp_ticket);
	native->token = strdup(tmp_ticket);
	rc = pcmk_ok;
	}
	}

	free_xml(reply);
	free_xml(hello);

	if (rc != pcmk_ok) {
	lrmd_api_disconnect(lrmd);
	}
	return rc;
	}

	static int
	lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->private;

	static struct ipc_client_callbacks lrmd_callbacks = {
	.dispatch = lrmd_ipc_dispatch,
	.destroy = lrmd_ipc_connection_destroy
	};

	crm_info("Connecting to lrmd");

	if (fd) {
	/* No mainloop */
	native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
	if (native->ipc && crm_ipc_connect(native->ipc)) {
	*fd = crm_ipc_get_fd(native->ipc);
	} else if (native->ipc) {
	rc = -ENOTCONN;
	}
	} else {
	native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
	native->ipc = mainloop_get_ipc_client(native->source);
	}

	if (native->ipc == NULL) {
	crm_debug("Could not connect to the LRMD API");
	rc = -ENOTCONN;
	}

	return rc;
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static int
	set_key(gnutls_datum_t * key, const char *location)
	{
	FILE *stream;
	int read_len = 256;
	int cur_len = 0;
	int buf_len = read_len;
	static char *key_cache = NULL;
	static size_t key_cache_len = 0;
	static time_t key_cache_updated;

	if (location == NULL) {
	return -1;
	}

	if (key_cache) {
	time_t now = time(NULL);

	if ((now - key_cache_updated) < 60) {
	key->data = gnutls_malloc(key_cache_len + 1);
	key->size = key_cache_len;
	memcpy(key->data, key_cache, key_cache_len);

	crm_debug("using cached LRMD key");
	return 0;
	} else {
	key_cache_len = 0;
	key_cache_updated = 0;
	free(key_cache);
	key_cache = NULL;
	crm_debug("clearing lrmd key cache");
	}
	}

	stream = fopen(location, "r");
	if (!stream) {
	return -1;
	}

	key->data = gnutls_malloc(read_len);
	while (!feof(stream)) {
	int next;

	if (cur_len == buf_len) {
	buf_len = cur_len + read_len;
	key->data = gnutls_realloc(key->data, buf_len);
	}
	next = fgetc(stream);
	if (next == EOF && feof(stream)) {
	break;
	}

	key->data[cur_len] = next;
	cur_len++;
	}
	fclose(stream);

	key->size = cur_len;
	if (!cur_len) {
	gnutls_free(key->data);
	key->data = 0;
	return -1;
	}

	if (!key_cache) {
	key_cache = calloc(1, key->size + 1);
	memcpy(key_cache, key->data, key->size);

	key_cache_len = key->size;
	key_cache_updated = time(NULL);
	}

	return 0;
	}

	int
	lrmd_tls_set_key(gnutls_datum_t * key)
	{
	int rc = 0;
	const char *specific_location = getenv("PCMK_authkey_location");

	if (set_key(key, specific_location) == 0) {
	crm_debug("Using custom authkey location %s", specific_location);
	return 0;
	}

	if (set_key(key, DEFAULT_REMOTE_KEY_LOCATION)) {
	rc = set_key(key, ALT_REMOTE_KEY_LOCATION);
	}
	if (rc) {
	crm_err("No lrmd remote key found");
	return -1;
	}

	return rc;
	}

	static void
	lrmd_gnutls_global_init(void)
	{
	static int gnutls_init = 0;

	if (!gnutls_init) {
	- gnutls_global_init();
	+ crm_gnutls_global_init();
	}
	gnutls_init = 1;
	}
	#endif

	static void
	report_async_connection_result(lrmd_t * lrmd, int rc)
	{
	lrmd_private_t *native = lrmd->private;

	if (native->callback) {
	lrmd_event_data_t event = { 0, };
	event.type = lrmd_event_connect;
	event.remote_nodename = native->remote_nodename;
	event.connection_rc = rc;
	native->callback(&event);
	}
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	lrmd_tcp_connect_cb(void *userdata, int sock)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->private;
	char name[256] = { 0, };
	static struct mainloop_fd_callbacks lrmd_tls_callbacks = {
	.dispatch = lrmd_tls_dispatch,
	.destroy = lrmd_tls_connection_destroy,
	};
	int rc = sock;
	gnutls_datum_t psk_key = { NULL, 0 };

	if (rc < 0) {
	lrmd_tls_connection_destroy(lrmd);
	crm_info("remote lrmd connect to %s at port %d failed", native->server, native->port);
	report_async_connection_result(lrmd, rc);
	return;
	}

	/* TODO continue with tls stuff now that tcp connect passed. make this async as well soon
	* to avoid all blocking code in the client. */
	native->sock = sock;

	if (lrmd_tls_set_key(&psk_key) != 0) {
	lrmd_tls_connection_destroy(lrmd);
	return;
	}

	gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
	gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
	gnutls_free(psk_key.data);

	native->remote->tls_session = create_psk_tls_session(sock, GNUTLS_CLIENT, native->psk_cred_c);

	if (crm_initiate_client_tls_handshake(native->remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT) != 0) {
	crm_warn("Client tls handshake failed for server %s:%d. Disconnecting", native->server,
	native->port);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	native->remote->tls_session = NULL;
	lrmd_tls_connection_destroy(lrmd);
	report_async_connection_result(lrmd, -1);
	return;
	}

	crm_info("Remote lrmd client TLS connection established with server %s:%d", native->server,
	native->port);

	snprintf(name, 128, "remote-lrmd-%s:%d", native->server, native->port);

	native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_tls_dispatch, lrmd);
	native->source =
	mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd, &lrmd_tls_callbacks);

	rc = lrmd_handshake(lrmd, name);
	report_async_connection_result(lrmd, rc);

	return;
	}

	static int
	lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /ms / )
	{
	int rc = 0;
	lrmd_private_t *native = lrmd->private;

	lrmd_gnutls_global_init();

	rc = crm_remote_tcp_connect_async(native->server, native->port, timeout, lrmd,
	lrmd_tcp_connect_cb);

	return rc;
	}

	static int
	lrmd_tls_connect(lrmd_t * lrmd, int *fd)
	{
	static struct mainloop_fd_callbacks lrmd_tls_callbacks = {
	.dispatch = lrmd_tls_dispatch,
	.destroy = lrmd_tls_connection_destroy,
	};

	lrmd_private_t *native = lrmd->private;
	int sock;
	gnutls_datum_t psk_key = { NULL, 0 };

	lrmd_gnutls_global_init();

	sock = crm_remote_tcp_connect(native->server, native->port);
	if (sock < 0) {
	crm_warn("Could not establish remote lrmd connection to %s", native->server);
	lrmd_tls_connection_destroy(lrmd);
	return -ENOTCONN;
	}

	native->sock = sock;

	if (lrmd_tls_set_key(&psk_key) != 0) {
	lrmd_tls_connection_destroy(lrmd);
	return -1;
	}

	gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
	gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
	gnutls_free(psk_key.data);

	native->remote->tls_session = create_psk_tls_session(sock, GNUTLS_CLIENT, native->psk_cred_c);

	if (crm_initiate_client_tls_handshake(native->remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT) != 0) {
	crm_err("Session creation for %s:%d failed", native->server, native->port);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	native->remote->tls_session = NULL;
	lrmd_tls_connection_destroy(lrmd);
	return -1;
	}

	crm_info("Remote lrmd client TLS connection established with server %s:%d", native->server,
	native->port);

	if (fd) {
	*fd = sock;
	} else {
	char name[256] = { 0, };
	snprintf(name, 128, "remote-lrmd-%s:%d", native->server, native->port);

	native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_tls_dispatch, lrmd);
	native->source =
	mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd, &lrmd_tls_callbacks);
	}
	return pcmk_ok;
	}
	#endif

	static int
	lrmd_api_connect(lrmd_t * lrmd, const char name, int fd)
	{
	int rc = -ENOTCONN;
	lrmd_private_t *native = lrmd->private;

	switch (native->type) {
	case CRM_CLIENT_IPC:
	rc = lrmd_ipc_connect(lrmd, fd);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	rc = lrmd_tls_connect(lrmd, fd);
	break;
	#endif
	default:
	crm_err("Unsupported connection type: %d", native->type);
	}

	if (rc == pcmk_ok) {
	rc = lrmd_handshake(lrmd, name);
	}

	return rc;
	}

	static int
	lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
	{
	int rc = 0;
	lrmd_private_t *native = lrmd->private;

	if (!native->callback) {
	crm_err("Async connect not possible, no lrmd client callback set.");
	return -1;
	}

	switch (native->type) {
	case CRM_CLIENT_IPC:
	/* fake async connection with ipc. it should be fast
	* enough that we gain very little from async */
	rc = lrmd_api_connect(lrmd, name, NULL);
	if (!rc) {
	report_async_connection_result(lrmd, rc);
	}
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	rc = lrmd_tls_connect_async(lrmd, timeout);
	if (rc) {
	/* connection failed, report rc now */
	report_async_connection_result(lrmd, rc);
	}
	break;
	#endif
	default:
	crm_err("Unsupported connection type: %d", native->type);
	}

	return rc;
	}

	static void
	lrmd_ipc_disconnect(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->private;

	if (native->source != NULL) {
	/* Attached to mainloop */
	mainloop_del_ipc_client(native->source);
	native->source = NULL;
	native->ipc = NULL;

	} else if (native->ipc) {
	/* Not attached to mainloop */
	crm_ipc_t *ipc = native->ipc;

	native->ipc = NULL;
	crm_ipc_close(ipc);
	crm_ipc_destroy(ipc);
	}
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	lrmd_tls_disconnect(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->private;

	if (native->remote->tls_session) {
	gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	native->remote->tls_session = 0;
	}

	if (native->source != NULL) {
	/* Attached to mainloop */
	mainloop_del_ipc_client(native->source);
	native->source = NULL;

	} else if (native->sock) {
	close(native->sock);
	}

	if (native->pending_notify) {
	g_list_free_full(native->pending_notify, lrmd_free_xml);
	native->pending_notify = NULL;
	}
	}
	#endif

	static int
	lrmd_api_disconnect(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->private;

	crm_info("Disconnecting from lrmd service");
	switch (native->type) {
	case CRM_CLIENT_IPC:
	lrmd_ipc_disconnect(lrmd);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	lrmd_tls_disconnect(lrmd);
	break;
	#endif
	default:
	crm_err("Unsupported connection type: %d", native->type);
	}

	free(native->token);
	native->token = NULL;
	return 0;
	}

	static int
	lrmd_api_register_rsc(lrmd_t * lrmd,
	const char *rsc_id,
	const char *class,
	const char provider, const char type, enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	xmlNode *data = NULL;

	if (!class \|\| !type \|\| !rsc_id) {
	return -EINVAL;
	}
	if (safe_str_eq(class, "ocf") && !provider) {
	return -EINVAL;
	}

	data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add(data, F_LRMD_CLASS, class);
	crm_xml_add(data, F_LRMD_PROVIDER, provider);
	crm_xml_add(data, F_LRMD_TYPE, type);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
	free_xml(data);

	return rc;
	}

	static int
	lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
	free_xml(data);

	return rc;
	}

	lrmd_rsc_info_t *
	lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
	{
	lrmd_rsc_info_t *copy = NULL;

	copy = calloc(1, sizeof(lrmd_rsc_info_t));

	copy->id = strdup(rsc_info->id);
	copy->type = strdup(rsc_info->type);
	copy->class = strdup(rsc_info->class);
	if (rsc_info->provider) {
	copy->provider = strdup(rsc_info->provider);
	}

	return copy;
	}

	void
	lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
	{
	if (!rsc_info) {
	return;
	}
	free(rsc_info->id);
	free(rsc_info->type);
	free(rsc_info->class);
	free(rsc_info->provider);
	free(rsc_info);
	}

	static lrmd_rsc_info_t *
	lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
	{
	lrmd_rsc_info_t *rsc_info = NULL;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
	xmlNode *output = NULL;
	const char *class = NULL;
	const char *provider = NULL;
	const char *type = NULL;

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 30000, options, TRUE);
	free_xml(data);

	if (!output) {
	return NULL;
	}

	class = crm_element_value(output, F_LRMD_CLASS);
	provider = crm_element_value(output, F_LRMD_PROVIDER);
	type = crm_element_value(output, F_LRMD_TYPE);

	if (!class \|\| !type) {
	free_xml(output);
	return NULL;
	} else if (safe_str_eq(class, "ocf") && !provider) {
	free_xml(output);
	return NULL;
	}

	rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
	rsc_info->id = strdup(rsc_id);
	rsc_info->class = strdup(class);
	if (provider) {
	rsc_info->provider = strdup(provider);
	}
	rsc_info->type = strdup(type);

	free_xml(output);
	return rsc_info;
	}

	static void
	lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
	{
	lrmd_private_t *native = lrmd->private;

	native->callback = callback;
	}

	void
	lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void userdata, void (callback)(lrmd_t lrmd, void userdata, xmlNode *msg))
	{
	lrmd_private_t *native = lrmd->private;

	native->proxy_callback = callback;
	native->proxy_callback_userdata = userdata;
	}

	void
	lrmd_internal_proxy_dispatch(lrmd_t lrmd, xmlNode msg)
	{
	lrmd_private_t *native = lrmd->private;

	if (native->proxy_callback) {
	crm_log_xml_trace(msg, "PROXY_INBOUND");
	native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
	}
	}

	int
	lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
	{
	if (lrmd == NULL) {
	return -ENOTCONN;
	}
	crm_xml_add(msg, F_LRMD_OPERATION, CRM_OP_IPC_FWD);

	crm_log_xml_trace(msg, "PROXY_OUTBOUND");
	return lrmd_send_xml_no_reply(lrmd, msg);
	}

	static int
	stonith_get_metadata(const char provider, const char type, char **output)
	{
	int rc = pcmk_ok;
	stonith_t *stonith_api = stonith_api_new();

	if(stonith_api) {
	stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type, provider, output, 0);
	stonith_api->cmds->free(stonith_api);
	}
	if (*output == NULL) {
	rc = -EIO;
	}
	return rc;
	}

	#define lsb_metadata_template \
	"<?xml version='1.0'?>\n" \
	"<!DOCTYPE resource-agent SYSTEM 'ra-api-1.dtd'>\n" \
	"<resource-agent name='%s' version='0.1'>\n" \
	" <version>1.0</version>\n" \
	" <longdesc lang='en'>\n" \
	" %s\n" \
	" </longdesc>\n" \
	" <shortdesc lang='en'>%s</shortdesc>\n" \
	" <parameters>\n" \
	" </parameters>\n" \
	" <actions>\n" \
	" <action name='meta-data' timeout='5' />\n" \
	" <action name='start' timeout='15' />\n" \
	" <action name='stop' timeout='15' />\n" \
	" <action name='status' timeout='15' />\n" \
	" <action name='restart' timeout='15' />\n" \
	" <action name='force-reload' timeout='15' />\n" \
	" <action name='monitor' timeout='15' interval='15' />\n" \
	" </actions>\n" \
	" <special tag='LSB'>\n" \
	" <Provides>%s</Provides>\n" \
	" <Required-Start>%s</Required-Start>\n" \
	" <Required-Stop>%s</Required-Stop>\n" \
	" <Should-Start>%s</Should-Start>\n" \
	" <Should-Stop>%s</Should-Stop>\n" \
	" <Default-Start>%s</Default-Start>\n" \
	" <Default-Stop>%s</Default-Stop>\n" \
	" </special>\n" \
	"</resource-agent>\n"

	#define LSB_INITSCRIPT_INFOBEGIN_TAG "### BEGIN INIT INFO"
	#define LSB_INITSCRIPT_INFOEND_TAG "### END INIT INFO"
	#define PROVIDES "# Provides:"
	#define REQ_START "# Required-Start:"
	#define REQ_STOP "# Required-Stop:"
	#define SHLD_START "# Should-Start:"
	#define SHLD_STOP "# Should-Stop:"
	#define DFLT_START "# Default-Start:"
	#define DFLT_STOP "# Default-Stop:"
	#define SHORT_DSCR "# Short-Description:"
	#define DESCRIPTION "# Description:"

	#define lsb_meta_helper_free_value(m) \
	if ((m) != NULL) { \
	xmlFree(m); \
	(m) = NULL; \
	}

	#define lsb_meta_helper_get_value(buffer, ptr, keyword) \
	if (!ptr && !strncasecmp(buffer, keyword, strlen(keyword))) { \
	(ptr) = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST buffer+strlen(keyword)); \
	continue; \
	}

	static int
	lsb_get_metadata(const char type, char *output)
	{
	char ra_pathname[PATH_MAX] = { 0, };
	FILE *fp;
	GString *meta_data = NULL;
	char buffer[1024];
	char *provides = NULL;
	char *req_start = NULL;
	char *req_stop = NULL;
	char *shld_start = NULL;
	char *shld_stop = NULL;
	char *dflt_start = NULL;
	char *dflt_stop = NULL;
	char *s_dscrpt = NULL;
	char *xml_l_dscrpt = NULL;
	GString *l_dscrpt = NULL;

	if(type[0] == '/') {
	snprintf(ra_pathname, sizeof(ra_pathname), "%s", type);
	} else {
	snprintf(ra_pathname, sizeof(ra_pathname), "%s/%s", LSB_ROOT_DIR, type);
	}

	crm_trace("Looking into %s", ra_pathname);
	if (!(fp = fopen(ra_pathname, "r"))) {
	return -errno;
	}

	/* Enter into the lsb-compliant comment block */
	while (fgets(buffer, sizeof(buffer), fp)) {
	/* Now suppose each of the following eight arguments contain only one line */
	lsb_meta_helper_get_value(buffer, provides, PROVIDES)
	lsb_meta_helper_get_value(buffer, req_start, REQ_START)
	lsb_meta_helper_get_value(buffer, req_stop, REQ_STOP)
	lsb_meta_helper_get_value(buffer, shld_start, SHLD_START)
	lsb_meta_helper_get_value(buffer, shld_stop, SHLD_STOP)
	lsb_meta_helper_get_value(buffer, dflt_start, DFLT_START)
	lsb_meta_helper_get_value(buffer, dflt_stop, DFLT_STOP)
	lsb_meta_helper_get_value(buffer, s_dscrpt, SHORT_DSCR)

	/* Long description may cross multiple lines */
	if ((l_dscrpt == NULL) && (0 == strncasecmp(buffer, DESCRIPTION, strlen(DESCRIPTION)))) {
	l_dscrpt = g_string_new(buffer + strlen(DESCRIPTION));
	/* Between # and keyword, more than one space, or a tab character,
	* indicates the continuation line. Extracted from LSB init script standard */
	while (fgets(buffer, sizeof(buffer), fp)) {
	if (!strncmp(buffer, "# ", 3) \|\| !strncmp(buffer, "#\t", 2)) {
	buffer[0] = ' ';
	l_dscrpt = g_string_append(l_dscrpt, buffer);
	} else {
	fputs(buffer, fp);
	break; /* Long description ends */
	}
	}
	continue;
	}
	if (l_dscrpt) {
	xml_l_dscrpt = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST(l_dscrpt->str));
	}
	if (!strncasecmp(buffer, LSB_INITSCRIPT_INFOEND_TAG, strlen(LSB_INITSCRIPT_INFOEND_TAG))) {
	/* Get to the out border of LSB comment block */
	break;
	}
	if (buffer[0] != '#') {
	break; /* Out of comment block in the beginning */
	}
	}
	fclose(fp);

	meta_data = g_string_new("");
	g_string_sprintf(meta_data, lsb_metadata_template, type,
	(xml_l_dscrpt == NULL) ? type : xml_l_dscrpt,
	(s_dscrpt == NULL) ? type : s_dscrpt, (provides == NULL) ? "" : provides,
	(req_start == NULL) ? "" : req_start, (req_stop == NULL) ? "" : req_stop,
	(shld_start == NULL) ? "" : shld_start, (shld_stop == NULL) ? "" : shld_stop,
	(dflt_start == NULL) ? "" : dflt_start, (dflt_stop == NULL) ? "" : dflt_stop);

	lsb_meta_helper_free_value(xml_l_dscrpt);
	lsb_meta_helper_free_value(s_dscrpt);
	lsb_meta_helper_free_value(provides);
	lsb_meta_helper_free_value(req_start);
	lsb_meta_helper_free_value(req_stop);
	lsb_meta_helper_free_value(shld_start);
	lsb_meta_helper_free_value(shld_stop);
	lsb_meta_helper_free_value(dflt_start);
	lsb_meta_helper_free_value(dflt_stop);

	if (l_dscrpt) {
	g_string_free(l_dscrpt, TRUE);
	}

	*output = strdup(meta_data->str);
	g_string_free(meta_data, TRUE);

	crm_trace("Created fake metadata: %d", strlen(*output));
	return pcmk_ok;
	}

	#if SUPPORT_NAGIOS
	static int
	nagios_get_metadata(const char type, char *output)
	{
	int rc = pcmk_ok;
	FILE *file_strm = NULL;
	int start = 0, length = 0, read_len = 0;
	char *metadata_file = NULL;
	int len = 36;

	len += strlen(NAGIOS_METADATA_DIR);
	len += strlen(type);
	metadata_file = calloc(1, len);
	CRM_CHECK(metadata_file != NULL, return -ENOMEM);

	sprintf(metadata_file, "%s/%s.xml", NAGIOS_METADATA_DIR, type);
	file_strm = fopen(metadata_file, "r");
	if (file_strm == NULL) {
	crm_err("Metadata file %s does not exist", metadata_file);
	free(metadata_file);
	return -EIO;
	}

	/* see how big the file is */
	start = ftell(file_strm);
	fseek(file_strm, 0L, SEEK_END);
	length = ftell(file_strm);
	fseek(file_strm, 0L, start);

	CRM_ASSERT(length >= 0);
	CRM_ASSERT(start == ftell(file_strm));

	if (length <= 0) {
	crm_info("%s was not valid", metadata_file);
	free(*output);
	*output = NULL;
	rc = -EIO;

	} else {
	crm_trace("Reading %d bytes from file", length);
	*output = calloc(1, (length + 1));
	read_len = fread(*output, 1, length, file_strm);
	if (read_len != length) {
	crm_err("Calculated and read bytes differ: %d vs. %d", length, read_len);
	free(*output);
	*output = NULL;
	rc = -EIO;
	}
	}

	fclose(file_strm);
	free(metadata_file);
	return rc;
	}
	#endif

	static int
	generic_get_metadata(const char standard, const char provider, const char type, char *output)
	{
	svc_action_t *action = resources_action_create(type,
	standard,
	provider,
	type,
	"meta-data",
	0,
	30000,
	NULL);

	if (!(services_action_sync(action))) {
	crm_err("Failed to retrieve meta-data for %s:%s:%s", standard, provider, type);
	services_action_free(action);
	return -EIO;
	}

	if (!action->stdout_data) {
	crm_err("Failed to retrieve meta-data for %s:%s:%s", standard, provider, type);
	services_action_free(action);
	return -EIO;
	}

	*output = strdup(action->stdout_data);
	services_action_free(action);

	return pcmk_ok;
	}

	static int
	lrmd_api_get_metadata(lrmd_t * lrmd,
	const char *class,
	const char *provider,
	const char type, char *output, enum lrmd_call_options options)
	{
	if (!class \|\| !type) {
	return -EINVAL;
	}

	if (safe_str_eq(class, "service")) {
	class = resources_find_service_class(type);
	}

	if (safe_str_eq(class, "stonith")) {
	return stonith_get_metadata(provider, type, output);
	} else if (safe_str_eq(class, "lsb")) {
	return lsb_get_metadata(type, output);
	#if SUPPORT_NAGIOS
	} else if (safe_str_eq(class, "nagios")) {
	return nagios_get_metadata(type, output);
	#endif
	}
	return generic_get_metadata(class, provider, type, output);
	}

	static int
	lrmd_api_exec(lrmd_t * lrmd, const char rsc_id, const char action, const char userdata, int interval, / ms */
	int timeout, /* ms */
	int start_delay, /* ms */
	enum lrmd_call_options options, lrmd_key_value_t * params)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
	xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
	lrmd_key_value_t *tmp = NULL;

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add(data, F_LRMD_RSC_ACTION, action);
	crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
	crm_xml_add_int(data, F_LRMD_RSC_INTERVAL, interval);
	crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
	crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);

	for (tmp = params; tmp; tmp = tmp->next) {
	hash2field((gpointer) tmp->key, (gpointer) tmp->value, args);
	}

	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
	free_xml(data);

	lrmd_key_value_freeall(params);
	return rc;
	}

	static int
	lrmd_api_cancel(lrmd_t * lrmd, const char rsc_id, const char action, int interval)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ACTION, action);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add_int(data, F_LRMD_RSC_INTERVAL, interval);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
	free_xml(data);
	return rc;
	}

	static int
	list_stonith_agents(lrmd_list_t ** resources)
	{
	int rc = 0;
	stonith_t *stonith_api = stonith_api_new();
	stonith_key_value_t *stonith_resources = NULL;
	stonith_key_value_t *dIter = NULL;

	if(stonith_api) {
	stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL, &stonith_resources, 0);
	stonith_api->cmds->free(stonith_api);
	}

	for (dIter = stonith_resources; dIter; dIter = dIter->next) {
	rc++;
	if (resources) {
	resources = lrmd_list_add(resources, dIter->value);
	}
	}

	stonith_key_value_freeall(stonith_resources, 1, 0);
	return rc;
	}

	static int
	lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
	const char *provider)
	{
	int rc = 0;

	if (safe_str_eq(class, "stonith")) {
	rc += list_stonith_agents(resources);

	} else {
	GListPtr gIter = NULL;
	GList *agents = resources_list_agents(class, provider);

	for (gIter = agents; gIter != NULL; gIter = gIter->next) {
	resources = lrmd_list_add(resources, (const char *)gIter->data);
	rc++;
	}
	g_list_free_full(agents, free);

	if (!class) {
	rc += list_stonith_agents(resources);
	}
	}

	if (rc == 0) {
	crm_notice("No agents found for class %s", class);
	rc = -EPROTONOSUPPORT;
	}
	return rc;
	}

	static int
	does_provider_have_agent(const char agent, const char provider, const char *class)
	{
	int found = 0;
	GList *agents = NULL;
	GListPtr gIter2 = NULL;

	agents = resources_list_agents(class, provider);
	for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
	if (safe_str_eq(agent, gIter2->data)) {
	found = 1;
	}
	}
	g_list_free_full(agents, free);

	return found;
	}

	static int
	lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char agent, lrmd_list_t * providers)
	{
	int rc = pcmk_ok;
	char *provider = NULL;
	GList *ocf_providers = NULL;
	GListPtr gIter = NULL;

	ocf_providers = resources_list_providers("ocf");

	for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
	provider = gIter->data;
	if (!agent \|\| does_provider_have_agent(agent, provider, "ocf")) {
	providers = lrmd_list_add(providers, (const char *)gIter->data);
	rc++;
	}
	}

	g_list_free_full(ocf_providers, free);
	return rc;
	}

	static int
	lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
	{
	int rc = 0;
	GList *standards = NULL;
	GListPtr gIter = NULL;

	standards = resources_list_standards();

	for (gIter = standards; gIter != NULL; gIter = gIter->next) {
	supported = lrmd_list_add(supported, (const char *)gIter->data);
	rc++;
	}

	if (list_stonith_agents(NULL) > 0) {
	supported = lrmd_list_add(supported, "stonith");
	rc++;
	}

	g_list_free_full(standards, free);
	return rc;
	}

	lrmd_t *
	lrmd_api_new(void)
	{
	lrmd_t *new_lrmd = NULL;
	lrmd_private_t *pvt = NULL;

	new_lrmd = calloc(1, sizeof(lrmd_t));
	pvt = calloc(1, sizeof(lrmd_private_t));
	pvt->remote = calloc(1, sizeof(crm_remote_t));
	new_lrmd->cmds = calloc(1, sizeof(lrmd_api_operations_t));

	pvt->type = CRM_CLIENT_IPC;
	new_lrmd->private = pvt;

	new_lrmd->cmds->connect = lrmd_api_connect;
	new_lrmd->cmds->connect_async = lrmd_api_connect_async;
	new_lrmd->cmds->is_connected = lrmd_api_is_connected;
	new_lrmd->cmds->poke_connection = lrmd_api_poke_connection;
	new_lrmd->cmds->disconnect = lrmd_api_disconnect;
	new_lrmd->cmds->register_rsc = lrmd_api_register_rsc;
	new_lrmd->cmds->unregister_rsc = lrmd_api_unregister_rsc;
	new_lrmd->cmds->get_rsc_info = lrmd_api_get_rsc_info;
	new_lrmd->cmds->set_callback = lrmd_api_set_callback;
	new_lrmd->cmds->get_metadata = lrmd_api_get_metadata;
	new_lrmd->cmds->exec = lrmd_api_exec;
	new_lrmd->cmds->cancel = lrmd_api_cancel;
	new_lrmd->cmds->list_agents = lrmd_api_list_agents;
	new_lrmd->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
	new_lrmd->cmds->list_standards = lrmd_api_list_standards;

	return new_lrmd;
	}

	lrmd_t *
	lrmd_remote_api_new(const char nodename, const char server, int port)
	{
	#ifdef HAVE_GNUTLS_GNUTLS_H
	lrmd_t *new_lrmd = lrmd_api_new();
	lrmd_private_t *native = new_lrmd->private;

	if (!nodename && !server) {
	lrmd_api_delete(new_lrmd);
	return NULL;
	}

	native->type = CRM_CLIENT_TLS;
	native->remote_nodename = nodename ? strdup(nodename) : strdup(server);
	native->server = server ? strdup(server) : strdup(nodename);
	native->port = port;
	if (native->port == 0) {
	const char *remote_port_str = getenv("PCMK_remote_port");
	native->port = remote_port_str ? atoi(remote_port_str) : DEFAULT_REMOTE_PORT;
	}

	return new_lrmd;
	#else
	crm_err("GNUTLS is not enabled for this build, remote LRMD client can not be created");
	return NULL;
	#endif

	}

	void
	lrmd_api_delete(lrmd_t * lrmd)
	{
	if (!lrmd) {
	return;
	}
	lrmd->cmds->disconnect(lrmd); /* no-op if already disconnected */
	free(lrmd->cmds);
	if (lrmd->private) {
	lrmd_private_t *native = lrmd->private;

	#ifdef HAVE_GNUTLS_GNUTLS_H
	free(native->server);
	#endif
	free(native->remote_nodename);
	free(native->remote);
	}

	free(lrmd->private);
	free(lrmd);
	}
	diff --git a/lrmd/main.c b/lrmd/main.c
	index 1c5726109d..af4bebbfb9 100644
	--- a/lrmd/main.c
	+++ b/lrmd/main.c
	@@ -1,358 +1,358 @@
	/*
	* Copyright (c) 2012 David Vossel <dvossel@redhat.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <unistd.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/services.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipcs.h>

	#include <lrmd_private.h>

	#if defined(HAVE_GNUTLS_GNUTLS_H) && defined(SUPPORT_REMOTE)
	# define ENABLE_PCMK_REMOTE
	#endif

	GMainLoop *mainloop = NULL;
	static qb_ipcs_service_t *ipcs = NULL;
	stonith_t *stonith_api = NULL;
	int lrmd_call_id = 0;

	static void
	stonith_connection_destroy_cb(stonith_t * st, stonith_event_t * e)
	{
	stonith_api->state = stonith_disconnected;
	crm_err("LRMD lost STONITH connection");
	stonith_connection_failed();
	}

	stonith_t *
	get_stonith_connection(void)
	{
	if (stonith_api && stonith_api->state == stonith_disconnected) {
	stonith_api_delete(stonith_api);
	stonith_api = NULL;
	}

	if (!stonith_api) {
	int rc = 0;
	int tries = 10;

	stonith_api = stonith_api_new();
	do {
	rc = stonith_api->cmds->connect(stonith_api, "lrmd", NULL);
	if (rc == pcmk_ok) {
	stonith_api->cmds->register_notification(stonith_api,
	T_STONITH_NOTIFY_DISCONNECT,
	stonith_connection_destroy_cb);
	break;
	}
	sleep(1);
	tries--;
	} while (tries);

	if (rc) {
	crm_err("Unable to connect to stonith daemon to execute command. error: %s",
	pcmk_strerror(rc));
	stonith_api_delete(stonith_api);
	stonith_api = NULL;
	}
	}
	return stonith_api;
	}

	static int32_t
	lrmd_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	crm_trace("Connection %p", c);
	if (crm_client_new(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	static void
	lrmd_ipc_created(qb_ipcs_connection_t * c)
	{
	crm_client_t *new_client = crm_client_get(c);

	crm_trace("Connection %p", c);
	CRM_ASSERT(new_client != NULL);
	/* Now that the connection is offically established, alert
	* the other clients a new connection exists. */

	notify_of_new_client(new_client);
	}

	static int32_t
	lrmd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	crm_client_t *client = crm_client_get(c);
	xmlNode *request = crm_ipcs_recv(client, data, size, &id, &flags);

	CRM_CHECK(client != NULL, crm_err("Invalid client");
	return FALSE);
	CRM_CHECK(client->id != NULL, crm_err("Invalid client: %p", client);
	return FALSE);

	CRM_CHECK(flags & crm_ipc_client_response, crm_err("Invalid client request: %p", client);
	return FALSE);

	if (!request) {
	return 0;
	}

	if (!client->name) {
	const char *value = crm_element_value(request, F_LRMD_CLIENTNAME);

	if (value == NULL) {
	client->name = crm_itoa(crm_ipcs_client_pid(c));
	} else {
	client->name = strdup(value);
	}
	}

	lrmd_call_id++;
	if (lrmd_call_id < 1) {
	lrmd_call_id = 1;
	}

	crm_xml_add(request, F_LRMD_CLIENTID, client->id);
	crm_xml_add(request, F_LRMD_CLIENTNAME, client->name);
	crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id);

	process_lrmd_message(client, id, request);

	free_xml(request);
	return 0;
	}

	static int32_t
	lrmd_ipc_closed(qb_ipcs_connection_t * c)
	{
	crm_client_t *client = crm_client_get(c);

	if (client == NULL) {
	return 0;
	}

	crm_trace("Connection %p", c);
	client_disconnect_cleanup(client->id);
	#ifdef ENABLE_PCMK_REMOTE
	ipc_proxy_remove_provider(client);
	#endif
	crm_client_destroy(client);
	return 0;
	}

	static void
	lrmd_ipc_destroy(qb_ipcs_connection_t * c)
	{
	lrmd_ipc_closed(c);
	crm_trace("Connection %p", c);
	}

	static struct qb_ipcs_service_handlers lrmd_ipc_callbacks = {
	.connection_accept = lrmd_ipc_accept,
	.connection_created = lrmd_ipc_created,
	.msg_process = lrmd_ipc_dispatch,
	.connection_closed = lrmd_ipc_closed,
	.connection_destroyed = lrmd_ipc_destroy
	};

	int
	lrmd_server_send_reply(crm_client_t * client, uint32_t id, xmlNode * reply)
	{

	crm_trace("sending reply to client (%s) with msg id %d", client->id, id);
	switch (client->kind) {
	case CRM_CLIENT_IPC:
	return crm_ipcs_send(client, id, reply, FALSE);
	#ifdef ENABLE_PCMK_REMOTE
	case CRM_CLIENT_TLS:
	return lrmd_tls_send_msg(client->remote, reply, id, "reply");
	#endif
	default:
	crm_err("Unknown lrmd client type %d", client->kind);
	}
	return -1;
	}

	int
	lrmd_server_send_notify(crm_client_t * client, xmlNode * msg)
	{
	crm_trace("sending notify to client (%s)", client->id);
	switch (client->kind) {
	case CRM_CLIENT_IPC:
	if (client->ipcs == NULL) {
	crm_trace("Asked to send event to disconnected local client");
	return -1;
	}
	return crm_ipcs_send(client, 0, msg, crm_ipc_server_event);
	#ifdef ENABLE_PCMK_REMOTE
	case CRM_CLIENT_TLS:
	if (client->remote == NULL) {
	crm_trace("Asked to send event to disconnected remote client");
	return -1;
	}
	return lrmd_tls_send_msg(client->remote, msg, 0, "notify");
	#endif
	default:
	crm_err("Unknown lrmd client type %d", client->kind);
	}
	return -1;
	}

	void
	lrmd_shutdown(int nsig)
	{
	crm_info("Terminating with %d clients", crm_hash_table_size(client_connections));
	if (ipcs) {
	mainloop_del_ipc_server(ipcs);
	}
	crm_exit(pcmk_ok);
	}

	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	/* Top-level Options */
	{"help", 0, 0, '?', "\tThis text"},
	{"version", 0, 0, '$', "\tVersion information" },
	{"verbose", 0, 0, 'V', "\tIncrease debug output"},

	{"logfile", 1, 0, 'l', "\tSend logs to the additional named logfile"},

	/* For compatibility with the original lrmd */
	{"dummy", 0, 0, 'r', NULL, 1},
	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	int
	main(int argc, char **argv)
	{
	int rc = 0;
	int flag = 0;
	int index = 0;
	const char *option = NULL;

	#ifdef ENABLE_PCMK_REMOTE
	crm_log_init("pacemaker_remoted", LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
	crm_set_options(NULL, "[options]", long_options,
	"Daemon for controlling services confirming to different standards");
	#else
	crm_log_init("lrmd", LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
	crm_set_options(NULL, "[options]", long_options,
	"Pacemaker Remote daemon for extending pacemaker functionality to remote nodes.");
	#endif

	while (1) {
	flag = crm_get_option(argc, argv, &index);
	if (flag == -1) {
	break;
	}

	switch (flag) {
	case 'r':
	break;
	case 'l':
	crm_add_logfile(optarg);
	break;
	case 'V':
	crm_bump_log_level(argc, argv);
	break;
	case '?':
	case '$':
	crm_help(flag, EX_OK);
	break;
	default:
	crm_help('?', EX_USAGE);
	break;
	}
	}

	option = daemon_option("logfacility");
	- if(safe_str_neq(option, "none")) {
	+ if(option && safe_str_neq(option, "none")) {
	setenv("HA_LOGFACILITY", option, 1); /* Used by the ocf_log/ha_log OCF macro */
	}

	option = daemon_option("logfile");
	- if(safe_str_neq(option, "none")) {
	+ if(option && safe_str_neq(option, "none")) {
	setenv("HA_LOGFILE", option, 1); /* Used by the ocf_log/ha_log OCF macro */

	if (daemon_option_enabled(crm_system_name, "debug")) {
	setenv("HA_DEBUGLOG", option, 1); /* Used by the ocf_log/ha_debug OCF macro */
	}
	}

	/* Used by RAs - Leave owned by root */
	crm_build_path(CRM_RSCTMP_DIR, 0755);

	/* Legacy: Used by RAs - Leave owned by root */
	crm_build_path(HA_STATE_DIR"/heartbeat/rsctmp", 0755);

	rsc_list = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_rsc);
	ipcs = mainloop_add_ipc_server(CRM_SYSTEM_LRMD, QB_IPC_SHM, &lrmd_ipc_callbacks);
	if (ipcs == NULL) {
	crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
	crm_exit(DAEMON_RESPAWN_STOP);
	}

	#ifdef ENABLE_PCMK_REMOTE
	{
	const char *remote_port_str = getenv("PCMK_remote_port");
	int remote_port = remote_port_str ? atoi(remote_port_str) : DEFAULT_REMOTE_PORT;

	if (lrmd_init_remote_tls_server(remote_port) < 0) {
	crm_err("Failed to create TLS server on port %d: shutting down and inhibiting respawn", remote_port);
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	ipc_proxy_init();
	}
	#endif

	mainloop_add_signal(SIGTERM, lrmd_shutdown);
	mainloop = g_main_new(FALSE);
	crm_info("Starting");
	g_main_run(mainloop);

	mainloop_del_ipc_server(ipcs);
	#ifdef ENABLE_PCMK_REMOTE
	lrmd_tls_server_destroy();
	ipc_proxy_cleanup();
	#endif
	crm_client_cleanup();

	g_hash_table_destroy(rsc_list);

	if (stonith_api) {
	stonith_api->cmds->disconnect(stonith_api);
	stonith_api_delete(stonith_api);
	}

	return rc;
	}
	diff --git a/lrmd/tls_backend.c b/lrmd/tls_backend.c
	index 384607d3aa..c7e787521c 100644
	--- a/lrmd/tls_backend.c
	+++ b/lrmd/tls_backend.c
	@@ -1,410 +1,410 @@
	/*
	* Copyright (c) 2012 David Vossel <dvossel@redhat.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <unistd.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/mainloop.h>

	#include <lrmd_private.h>

	#include <netdb.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <netinet/ip.h>
	#include <arpa/inet.h>

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# define LRMD_REMOTE_AUTH_TIMEOUT 10000
	gnutls_psk_server_credentials_t psk_cred_s;
	gnutls_dh_params_t dh_params;
	static int ssock = -1;
	extern int lrmd_call_id;

	static void
	debug_log(int level, const char *str)
	{
	fputs(str, stderr);
	}

	static int
	lrmd_remote_client_msg(gpointer data)
	{
	int id = 0;
	int rc = 0;
	int disconnected = 0;
	xmlNode *request = NULL;
	crm_client_t *client = data;

	if (client->remote->tls_handshake_complete == FALSE) {
	int rc = 0;

	/* Muliple calls to handshake will be required, this callback
	* will be invoked once the client sends more handshake data. */
	do {
	rc = gnutls_handshake(*client->remote->tls_session);

	if (rc < 0 && rc != GNUTLS_E_AGAIN) {
	crm_err("Remote lrmd tls handshake failed");
	return -1;
	}
	} while (rc == GNUTLS_E_INTERRUPTED);

	if (rc == 0) {
	crm_debug("Remote lrmd tls handshake completed");
	client->remote->tls_handshake_complete = TRUE;
	if (client->remote->auth_timeout) {
	g_source_remove(client->remote->auth_timeout);
	}
	client->remote->auth_timeout = 0;
	}
	return 0;
	}

	rc = crm_remote_ready(client->remote, 0);
	if (rc == 0) {
	/* no msg to read */
	return 0;
	} else if (rc < 0) {
	crm_info("Client disconnected during remote client read");
	return -1;
	}

	crm_remote_recv(client->remote, -1, &disconnected);

	request = crm_remote_parse_buffer(client->remote);
	while (request) {
	crm_element_value_int(request, F_LRMD_REMOTE_MSG_ID, &id);
	crm_trace("processing request from remote client with remote msg id %d", id);
	if (!client->name) {
	const char *value = crm_element_value(request, F_LRMD_CLIENTNAME);

	if (value) {
	client->name = strdup(value);
	}
	}

	lrmd_call_id++;
	if (lrmd_call_id < 1) {
	lrmd_call_id = 1;
	}

	crm_xml_add(request, F_LRMD_CLIENTID, client->id);
	crm_xml_add(request, F_LRMD_CLIENTNAME, client->name);
	crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id);

	process_lrmd_message(client, id, request);
	free_xml(request);

	/* process all the messages in the current buffer */
	request = crm_remote_parse_buffer(client->remote);
	}

	if (disconnected) {
	crm_info("Client disconnect detected in tls msg dispatcher.");
	return -1;
	}

	return 0;
	}

	static void
	lrmd_remote_client_destroy(gpointer user_data)
	{
	crm_client_t *client = user_data;

	if (client == NULL) {
	return;
	}

	ipc_proxy_remove_provider(client);

	/* if this is the last remote connection, stop recurring
	* operations */
	if (crm_hash_table_size(client_connections) == 1) {
	client_disconnect_cleanup(NULL);
	}

	crm_notice("LRMD client disconnecting remote client - name: %s id: %s",
	client->name ? client->name : "<unknown>", client->id);

	if (client->remote->tls_session) {
	void *sock_ptr;
	int csock;

	sock_ptr = gnutls_transport_get_ptr(*client->remote->tls_session);
	csock = GPOINTER_TO_INT(sock_ptr);

	gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_RDWR);
	gnutls_deinit(*client->remote->tls_session);
	gnutls_free(client->remote->tls_session);
	close(csock);
	}

	crm_client_destroy(client);

	return;
	}

	static gboolean
	lrmd_auth_timeout_cb(gpointer data)
	{
	crm_client_t *client = data;

	client->remote->auth_timeout = 0;

	if (client->remote->tls_handshake_complete == TRUE) {
	return FALSE;
	}

	mainloop_del_fd(client->remote->source);
	client->remote->source = NULL;
	crm_err("Remote client authentication timed out");

	return FALSE;
	}

	static int
	lrmd_remote_listen(gpointer data)
	{
	int csock = 0;
	int flag = 0;
	unsigned laddr;
	struct sockaddr_in addr;
	gnutls_session_t *session = NULL;
	crm_client_t *new_client = NULL;

	static struct mainloop_fd_callbacks lrmd_remote_fd_cb = {
	.dispatch = lrmd_remote_client_msg,
	.destroy = lrmd_remote_client_destroy,
	};

	/* accept the connection */
	laddr = sizeof(addr);
	memset(&addr, 0, sizeof(addr));
	csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
	crm_debug("New remote connection from %s", inet_ntoa(addr.sin_addr));

	if (csock == -1) {
	crm_err("accept socket failed");
	return TRUE;
	}

	if ((flag = fcntl(csock, F_GETFL)) >= 0) {
	if (fcntl(csock, F_SETFL, flag \| O_NONBLOCK) < 0) {
	crm_err("fcntl() write failed");
	close(csock);
	return TRUE;
	}
	} else {
	crm_err("fcntl() read failed");
	close(csock);
	return TRUE;
	}

	session = create_psk_tls_session(csock, GNUTLS_SERVER, psk_cred_s);
	if (session == NULL) {
	crm_err("TLS session creation failed");
	close(csock);
	return TRUE;
	}

	new_client = calloc(1, sizeof(crm_client_t));
	new_client->remote = calloc(1, sizeof(crm_remote_t));
	new_client->kind = CRM_CLIENT_TLS;
	new_client->remote->tls_session = session;
	new_client->id = crm_generate_uuid();
	new_client->remote->auth_timeout =
	g_timeout_add(LRMD_REMOTE_AUTH_TIMEOUT, lrmd_auth_timeout_cb, new_client);
	crm_notice("LRMD client connection established. %p id: %s", new_client, new_client->id);

	new_client->remote->source =
	mainloop_add_fd("lrmd-remote-client", G_PRIORITY_DEFAULT, csock, new_client,
	&lrmd_remote_fd_cb);
	g_hash_table_insert(client_connections, new_client->id, new_client);

	/* Alert other clients of the new connection */
	notify_of_new_client(new_client);
	return TRUE;
	}

	static void
	lrmd_remote_connection_destroy(gpointer user_data)
	{
	crm_notice("Remote tls server disconnected");
	return;
	}

	static int
	lrmd_tls_server_key_cb(gnutls_session_t session, const char username, gnutls_datum_t key)
	{
	return lrmd_tls_set_key(key);
	}

	static int
	bind_and_listen(struct addrinfo *addr)
	{
	int optval;
	int fd;
	int rc;
	char buffer[256] = { 0, };

	if (addr->ai_family == AF_INET6) {
	struct sockaddr_in6 addr_in = (struct sockaddr_in6 )(void*)addr->ai_addr;
	inet_ntop(addr->ai_family, &addr_in->sin6_addr, buffer, DIMOF(buffer));

	} else {
	struct sockaddr_in addr_in = (struct sockaddr_in )(void*)addr->ai_addr;
	inet_ntop(addr->ai_family, &addr_in->sin_addr, buffer, DIMOF(buffer));
	}

	crm_trace("Attempting to bind on address %s", buffer);

	fd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
	if (fd < 0) {
	return -1;
	}

	/* reuse address */
	optval = 1;
	rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
	if (rc < 0) {
	crm_perror(LOG_INFO, "Couldn't allow the reuse of local addresses by our remote listener, bind address %s", buffer);
	close(fd);
	return -1;
	}

	if (addr->ai_family == AF_INET6) {
	optval = 0;
	rc = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &optval, sizeof(optval));
	if (rc < 0) {
	crm_perror(LOG_INFO, "Couldn't disable IPV6 only on address %s", buffer);
	close(fd);
	return -1;
	}
	}

	if (bind(fd, addr->ai_addr, addr->ai_addrlen) != 0) {
	close(fd);
	return -1;
	}

	if (listen(fd, 10) == -1) {
	crm_err("Can not start listen on address %s", buffer);
	close(fd);
	return -1;
	}

	crm_notice("Listening on address %s", buffer);

	return fd;
	}

	int
	lrmd_init_remote_tls_server(int port)
	{
	int rc;
	int filter;
	struct addrinfo hints, res = NULL, iter;
	char port_str[16];

	static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
	.dispatch = lrmd_remote_listen,
	.destroy = lrmd_remote_connection_destroy,
	};

	crm_notice("Starting a tls listener on port %d.", port);
	- gnutls_global_init();
	+ crm_gnutls_global_init();
	gnutls_global_set_log_function(debug_log);

	gnutls_dh_params_init(&dh_params);
	gnutls_dh_params_generate2(dh_params, 1024);
	gnutls_psk_allocate_server_credentials(&psk_cred_s);
	gnutls_psk_set_server_credentials_function(psk_cred_s, lrmd_tls_server_key_cb);
	gnutls_psk_set_server_dh_params(psk_cred_s, dh_params);

	memset(&hints, 0, sizeof(struct addrinfo));
	hints.ai_flags = AI_PASSIVE; /* Only return socket addresses with wildcard INADDR_ANY or IN6ADDR_ANY_INIT */
	hints.ai_family = AF_UNSPEC; /* Return IPv6 or IPv4 */
	hints.ai_socktype = SOCK_STREAM;
	hints.ai_protocol = IPPROTO_TCP;

	snprintf(port_str, sizeof(port_str), "%d", port);
	rc = getaddrinfo(NULL, port_str, &hints, &res);
	if (rc) {
	crm_err("getaddrinfo: %s", gai_strerror(rc));
	return -1;
	}

	iter = res;
	filter = AF_INET6;
	/* Try IPv6 addresses first, then IPv4 */
	while (iter) {
	if (iter->ai_family == filter) {
	ssock = bind_and_listen(iter);
	}
	if (ssock != -1) {
	break;
	}

	iter = iter->ai_next;
	if (iter == NULL && filter == AF_INET6) {
	iter = res;
	filter = AF_INET;
	}
	}

	if (ssock < 0) {
	crm_err("unable to bind to address");
	goto init_remote_cleanup;
	}

	mainloop_add_fd("lrmd-remote", G_PRIORITY_DEFAULT, ssock, NULL, &remote_listen_fd_callbacks);

	rc = ssock;
	init_remote_cleanup:
	if (rc < 0) {
	close(ssock);
	ssock = 0;
	}
	freeaddrinfo(res);
	return rc;

	}

	void
	lrmd_tls_server_destroy(void)
	{
	if (psk_cred_s) {
	gnutls_psk_free_server_credentials(psk_cred_s);
	psk_cred_s = 0;
	}

	if (ssock > 0) {
	close(ssock);
	ssock = 0;
	}
	}
	#endif
	diff --git a/tools/regression.acls.exp b/tools/regression.acls.exp
	index 85c05d1334..2abdf974d3 100644
	--- a/tools/regression.acls.exp
	+++ b/tools/regression.acls.exp
	@@ -1,1773 +1,1773 @@
	Created new pacemaker-1.3 configuration
	Setting up shadow instance
	A new shadow instance was created. To begin using it paste the following into your shell:
	CIB_shadow=tools-regression ; export CIB_shadow
	=#=#=#= Begin test: Configure some ACLs =#=#=#=
	=#=#=#= Current cib after: Configure some ACLs =#=#=#=
	<cib epoch="1" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config/>
	<nodes/>
	<resources/>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Configure some ACLs - OK (0) =#=#=#=
	* Passed: cibadmin - Configure some ACLs
	=#=#=#= Begin test: Enable ACLs =#=#=#=
	=#=#=#= Current cib after: Enable ACLs =#=#=#=
	<cib epoch="2" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Enable ACLs - OK (0) =#=#=#=
	* Passed: crm_attribute - Enable ACLs
	=#=#=#= Begin test: Set cluster option =#=#=#=
	=#=#=#= Current cib after: Set cluster option =#=#=#=
	<cib epoch="3" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Set cluster option - OK (0) =#=#=#=
	* Passed: crm_attribute - Set cluster option
	=#=#=#= Begin test: New ACL =#=#=#=
	=#=#=#= Current cib after: New ACL =#=#=#=
	<cib epoch="4" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: New ACL - OK (0) =#=#=#=
	* Passed: cibadmin - New ACL
	=#=#=#= Begin test: Another ACL =#=#=#=
	=#=#=#= Current cib after: Another ACL =#=#=#=
	<cib epoch="5" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Another ACL - OK (0) =#=#=#=
	* Passed: cibadmin - Another ACL
	=#=#=#= Begin test: Updated ACL =#=#=#=
	=#=#=#= Current cib after: Updated ACL =#=#=#=
	<cib epoch="6" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<deny id="betteridea-nothing" xpath="/cib"/>
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Updated ACL - OK (0) =#=#=#=
	* Passed: cibadmin - Updated ACL
	=#=#=#= Begin test: unknownguy: Query configuration =#=#=#=
	Call failed: Permission denied
	=#=#=#= End test: unknownguy: Query configuration - Permission denied (13) =#=#=#=
	* Passed: cibadmin - unknownguy: Query configuration
	=#=#=#= Begin test: unknownguy: Set enable-acl =#=#=#=
	Error performing operation: Permission denied
	=#=#=#= End test: unknownguy: Set enable-acl - Permission denied (13) =#=#=#=
	* Passed: crm_attribute - unknownguy: Set enable-acl
	=#=#=#= Begin test: unknownguy: Set stonith-enabled =#=#=#=
	Error performing operation: Permission denied
	=#=#=#= End test: unknownguy: Set stonith-enabled - Permission denied (13) =#=#=#=
	* Passed: crm_attribute - unknownguy: Set stonith-enabled
	=#=#=#= Begin test: unknownguy: Create a resource =#=#=#=
	__xml_acl_check: Ordinary user unknownguy cannot access the CIB without any defined ACLs
	__xml_acl_check: Ordinary user unknownguy cannot access the CIB without any defined ACLs
	__xml_acl_check: Ordinary user unknownguy cannot access the CIB without any defined ACLs
	__xml_acl_check: Ordinary user unknownguy cannot access the CIB without any defined ACLs
	Call failed: Permission denied
	=#=#=#= End test: unknownguy: Create a resource - Permission denied (13) =#=#=#=
	* Passed: cibadmin - unknownguy: Create a resource
	=#=#=#= Begin test: l33t-haxor: Query configuration =#=#=#=
	Call failed: Permission denied
	=#=#=#= End test: l33t-haxor: Query configuration - Permission denied (13) =#=#=#=
	* Passed: cibadmin - l33t-haxor: Query configuration
	=#=#=#= Begin test: l33t-haxor: Set enable-acl =#=#=#=
	Error performing operation: Permission denied
	=#=#=#= End test: l33t-haxor: Set enable-acl - Permission denied (13) =#=#=#=
	* Passed: crm_attribute - l33t-haxor: Set enable-acl
	=#=#=#= Begin test: l33t-haxor: Set stonith-enabled =#=#=#=
	Error performing operation: Permission denied
	=#=#=#= End test: l33t-haxor: Set stonith-enabled - Permission denied (13) =#=#=#=
	* Passed: crm_attribute - l33t-haxor: Set stonith-enabled
	=#=#=#= Begin test: l33t-haxor: Create a resource =#=#=#=
	__xml_acl_check: 400 access denied to /cib/configuration/resources/primitive[@id='dummy']: parent
	__xml_acl_post_process: Cannot add new node primitive at /cib/configuration/resources/primitive[@id='dummy']
	Call failed: Permission denied
	=#=#=#= End test: l33t-haxor: Create a resource - Permission denied (13) =#=#=#=
	* Passed: cibadmin - l33t-haxor: Create a resource
	=#=#=#= Begin test: niceguy: Query configuration =#=#=#=
	<cib epoch="6" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<deny id="betteridea-nothing" xpath="/cib"/>
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: niceguy: Query configuration - OK (0) =#=#=#=
	* Passed: cibadmin - niceguy: Query configuration
	=#=#=#= Begin test: niceguy: Set enable-acl =#=#=#=
	__xml_acl_check: 400 access denied to /cib/configuration/crm_config/cluster_property_set[@id='cib-bootstrap-options']/nvpair[@id='cib-bootstrap-options-enable-acl'][@value]: default
	Error performing operation: Permission denied
	Error setting enable-acl=false (section=crm_config, set=<null>): Permission denied
	=#=#=#= End test: niceguy: Set enable-acl - Permission denied (13) =#=#=#=
	* Passed: crm_attribute - niceguy: Set enable-acl
	=#=#=#= Begin test: niceguy: Set stonith-enabled =#=#=#=
	__xml_acl_post_process: Creation of nvpair=cib-bootstrap-options-stonith-enabled is allowed
	=#=#=#= Current cib after: niceguy: Set stonith-enabled =#=#=#=
	<cib epoch="7" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<deny id="betteridea-nothing" xpath="/cib"/>
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: niceguy: Set stonith-enabled - OK (0) =#=#=#=
	* Passed: crm_attribute - niceguy: Set stonith-enabled
	=#=#=#= Begin test: niceguy: Create a resource =#=#=#=
	__xml_acl_check: 400 access denied to /cib/configuration/resources/primitive[@id='dummy']: default
	__xml_acl_post_process: Cannot add new node primitive at /cib/configuration/resources/primitive[@id='dummy']
	Call failed: Permission denied
	=#=#=#= End test: niceguy: Create a resource - Permission denied (13) =#=#=#=
	* Passed: cibadmin - niceguy: Create a resource
	=#=#=#= Begin test: root: Query configuration =#=#=#=
	<cib epoch="7" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<deny id="betteridea-nothing" xpath="/cib"/>
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: root: Query configuration - OK (0) =#=#=#=
	* Passed: cibadmin - root: Query configuration
	=#=#=#= Begin test: root: Set stonith-enabled =#=#=#=
	=#=#=#= Current cib after: root: Set stonith-enabled =#=#=#=
	<cib epoch="8" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<deny id="betteridea-nothing" xpath="/cib"/>
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: root: Set stonith-enabled - OK (0) =#=#=#=
	* Passed: crm_attribute - root: Set stonith-enabled
	=#=#=#= Begin test: root: Create a resource =#=#=#=
	=#=#=#= Current cib after: root: Create a resource =#=#=#=
	<cib epoch="9" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy"/>
	</resources>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<deny id="betteridea-nothing" xpath="/cib"/>
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: root: Create a resource - OK (0) =#=#=#=
	* Passed: cibadmin - root: Create a resource
	=#=#=#= Begin test: l33t-haxor: Create a resource meta attribute =#=#=#=
	Error performing operation: Permission denied
	=#=#=#= End test: l33t-haxor: Create a resource meta attribute - Permission denied (13) =#=#=#=
	* Passed: crm_resource - l33t-haxor: Create a resource meta attribute
	=#=#=#= Begin test: l33t-haxor: Query a resource meta attribute =#=#=#=
	Error performing operation: Permission denied
	=#=#=#= End test: l33t-haxor: Query a resource meta attribute - Permission denied (13) =#=#=#=
	* Passed: crm_resource - l33t-haxor: Query a resource meta attribute
	=#=#=#= Begin test: l33t-haxor: Remove a resource meta attribute =#=#=#=
	Error performing operation: Permission denied
	=#=#=#= End test: l33t-haxor: Remove a resource meta attribute - Permission denied (13) =#=#=#=
	* Passed: crm_resource - l33t-haxor: Remove a resource meta attribute
	=#=#=#= Begin test: niceguy: Create a resource meta attribute =#=#=#=
	-( unpack.c:711 ) error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	-( unpack.c:712 ) error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	-( unpack.c:713 ) error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	+error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	+error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	+error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	__xml_acl_post_process: Creation of nvpair=dummy-meta_attributes-target-role is allowed
	=#=#=#= Current cib after: niceguy: Create a resource meta attribute =#=#=#=
	<cib epoch="10" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Stopped"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<deny id="betteridea-nothing" xpath="/cib"/>
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: niceguy: Create a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - niceguy: Create a resource meta attribute
	=#=#=#= Begin test: niceguy: Query a resource meta attribute =#=#=#=
	-( unpack.c:711 ) error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	-( unpack.c:712 ) error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	-( unpack.c:713 ) error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	+error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	+error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	+error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	Stopped
	=#=#=#= Current cib after: niceguy: Query a resource meta attribute =#=#=#=
	<cib epoch="10" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Stopped"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<deny id="betteridea-nothing" xpath="/cib"/>
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: niceguy: Query a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - niceguy: Query a resource meta attribute
	=#=#=#= Begin test: niceguy: Remove a resource meta attribute =#=#=#=
	-( unpack.c:711 ) error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	-( unpack.c:712 ) error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	-( unpack.c:713 ) error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	+error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	+error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	+error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	Deleted dummy option: id=dummy-meta_attributes-target-role name=target-role
	=#=#=#= Current cib after: niceguy: Remove a resource meta attribute =#=#=#=
	<cib epoch="11" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<deny id="betteridea-nothing" xpath="/cib"/>
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: niceguy: Remove a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - niceguy: Remove a resource meta attribute
	=#=#=#= Begin test: niceguy: Create a resource meta attribute =#=#=#=
	-( unpack.c:711 ) error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	-( unpack.c:712 ) error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	-( unpack.c:713 ) error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	+error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	+error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	+error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	__xml_acl_post_process: Creation of nvpair=dummy-meta_attributes-target-role is allowed
	=#=#=#= Current cib after: niceguy: Create a resource meta attribute =#=#=#=
	<cib epoch="12" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<deny id="betteridea-nothing" xpath="/cib"/>
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: niceguy: Create a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - niceguy: Create a resource meta attribute
	=#=#=#= Begin test: badidea: Query configuration - implied deny =#=#=#=
	<cib>
	<configuration>
	<resources>
	<primitive id="dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	</configuration>
	</cib>
	=#=#=#= End test: badidea: Query configuration - implied deny - OK (0) =#=#=#=
	* Passed: cibadmin - badidea: Query configuration - implied deny
	=#=#=#= Begin test: betteridea: Query configuration - explicit deny =#=#=#=
	<cib>
	<configuration>
	<resources>
	<primitive id="dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	</configuration>
	</cib>
	=#=#=#= End test: betteridea: Query configuration - explicit deny - OK (0) =#=#=#=
	* Passed: cibadmin - betteridea: Query configuration - explicit deny
	<cib epoch="13" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= Begin test: niceguy: Replace - remove acls =#=#=#=
	__xml_acl_check: 400 access denied to /cib[@epoch]: default
	__xml_acl_check: 400 access denied to /cib/configuration/acls: default
	Call failed: Permission denied
	=#=#=#= End test: niceguy: Replace - remove acls - Permission denied (13) =#=#=#=
	* Passed: cibadmin - niceguy: Replace - remove acls
	<cib epoch="13" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	<primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy"/>
	</resources>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<deny id="betteridea-nothing" xpath="/cib"/>
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= Begin test: niceguy: Replace - create resource =#=#=#=
	__xml_acl_check: 400 access denied to /cib[@epoch]: default
	__xml_acl_check: 400 access denied to /cib/configuration/resources/primitive[@id='dummy2']: default
	__xml_acl_post_process: Cannot add new node primitive at /cib/configuration/resources/primitive[@id='dummy2']
	Call failed: Permission denied
	=#=#=#= End test: niceguy: Replace - create resource - Permission denied (13) =#=#=#=
	* Passed: cibadmin - niceguy: Replace - create resource
	<cib epoch="13" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="false"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<deny id="betteridea-nothing" xpath="/cib"/>
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= Begin test: niceguy: Replace - modify attribute =#=#=#=
	__xml_acl_check: 400 access denied to /cib[@epoch]: default
	__xml_acl_check: 400 access denied to /cib/configuration/crm_config/cluster_property_set[@id='cib-bootstrap-options']/nvpair[@id='cib-bootstrap-options-enable-acl'][@value]: default
	Call failed: Permission denied
	=#=#=#= End test: niceguy: Replace - modify attribute - Permission denied (13) =#=#=#=
	* Passed: cibadmin - niceguy: Replace - modify attribute
	<cib epoch="13" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<deny id="betteridea-nothing" xpath="/cib"/>
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= Begin test: niceguy: Replace - delete attribute =#=#=#=
	__xml_acl_check: 400 access denied to /cib[@epoch]: default
	__xml_acl_check: 400 access denied to /cib/configuration/crm_config/cluster_property_set[@id='cib-bootstrap-options']/nvpair[@id='cib-bootstrap-options-enable-acl']: default
	Call failed: Permission denied
	=#=#=#= End test: niceguy: Replace - delete attribute - Permission denied (13) =#=#=#=
	* Passed: cibadmin - niceguy: Replace - delete attribute
	<cib epoch="13" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy" description="nothing interesting">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_user id="badidea">
	<read id="badidea-resources" xpath="//meta_attributes"/>
	</acl_user>
	<acl_user id="betteridea">
	<deny id="betteridea-nothing" xpath="/cib"/>
	<read id="betteridea-resources" xpath="//meta_attributes"/>
	</acl_user>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= Begin test: niceguy: Replace - create attribute =#=#=#=
	__xml_acl_check: 400 access denied to /cib[@epoch]: default
	__xml_acl_check: 400 access denied to /cib/configuration/resources/primitive[@id='dummy'][@description]: default
	Call failed: Permission denied
	=#=#=#= End test: niceguy: Replace - create attribute - Permission denied (13) =#=#=#=
	* Passed: cibadmin - niceguy: Replace - create attribute


	!#!#!#!#! Upgrading to pacemaker-2.0 and retesting !#!#!#!#!
	=#=#=#= Begin test: root: Upgrade to pacemaker-2.0 =#=#=#=
	__xml_acl_post_process: Creation of acl_permission=observer-read-1 is allowed
	__xml_acl_post_process: Creation of acl_permission=observer-write-1 is allowed
	__xml_acl_post_process: Creation of acl_permission=observer-write-2 is allowed
	__xml_acl_post_process: Creation of acl_permission=crook-nothing is allowed
	__xml_acl_post_process: Creation of acl_permission=badidea-resources is allowed
	__xml_acl_post_process: Creation of acl_permission=betteridea-nothing is allowed
	__xml_acl_post_process: Creation of acl_permission=betteridea-resources is allowed
	=#=#=#= Current cib after: root: Upgrade to pacemaker-2.0 =#=#=#=
	<cib epoch="2" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_target id="l33t-haxor">
	<role id="auto-l33t-haxor"/>
	</acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea">
	<role id="auto-badidea"/>
	</acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	<acl_target id="betteridea">
	<role id="auto-betteridea"/>
	</acl_target>
	<acl_role id="auto-betteridea">
	<acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
	<acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: root: Upgrade to pacemaker-2.0 - OK (0) =#=#=#=
	* Passed: cibadmin - root: Upgrade to pacemaker-2.0
	=#=#=#= Begin test: unknownguy: Query configuration =#=#=#=
	Call failed: Permission denied
	=#=#=#= End test: unknownguy: Query configuration - Permission denied (13) =#=#=#=
	* Passed: cibadmin - unknownguy: Query configuration
	=#=#=#= Begin test: unknownguy: Set enable-acl =#=#=#=
	Error performing operation: Permission denied
	=#=#=#= End test: unknownguy: Set enable-acl - Permission denied (13) =#=#=#=
	* Passed: crm_attribute - unknownguy: Set enable-acl
	=#=#=#= Begin test: unknownguy: Set stonith-enabled =#=#=#=
	Error performing operation: Permission denied
	=#=#=#= End test: unknownguy: Set stonith-enabled - Permission denied (13) =#=#=#=
	* Passed: crm_attribute - unknownguy: Set stonith-enabled
	=#=#=#= Begin test: unknownguy: Create a resource =#=#=#=
	__xml_acl_check: Ordinary user unknownguy cannot access the CIB without any defined ACLs
	__xml_acl_check: Ordinary user unknownguy cannot access the CIB without any defined ACLs
	__xml_acl_check: Ordinary user unknownguy cannot access the CIB without any defined ACLs
	__xml_acl_check: Ordinary user unknownguy cannot access the CIB without any defined ACLs
	Call failed: Permission denied
	=#=#=#= End test: unknownguy: Create a resource - Permission denied (13) =#=#=#=
	* Passed: cibadmin - unknownguy: Create a resource
	=#=#=#= Begin test: l33t-haxor: Query configuration =#=#=#=
	Call failed: Permission denied
	=#=#=#= End test: l33t-haxor: Query configuration - Permission denied (13) =#=#=#=
	* Passed: cibadmin - l33t-haxor: Query configuration
	=#=#=#= Begin test: l33t-haxor: Set enable-acl =#=#=#=
	Error performing operation: Permission denied
	=#=#=#= End test: l33t-haxor: Set enable-acl - Permission denied (13) =#=#=#=
	* Passed: crm_attribute - l33t-haxor: Set enable-acl
	=#=#=#= Begin test: l33t-haxor: Set stonith-enabled =#=#=#=
	Error performing operation: Permission denied
	=#=#=#= End test: l33t-haxor: Set stonith-enabled - Permission denied (13) =#=#=#=
	* Passed: crm_attribute - l33t-haxor: Set stonith-enabled
	=#=#=#= Begin test: l33t-haxor: Create a resource =#=#=#=
	__xml_acl_check: 400 access denied to /cib/configuration/resources/primitive[@id='dummy']: parent
	__xml_acl_post_process: Cannot add new node primitive at /cib/configuration/resources/primitive[@id='dummy']
	Call failed: Permission denied
	=#=#=#= End test: l33t-haxor: Create a resource - Permission denied (13) =#=#=#=
	* Passed: cibadmin - l33t-haxor: Create a resource
	=#=#=#= Begin test: niceguy: Query configuration =#=#=#=
	<cib epoch="7" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	<acls>
	<acl_target id="l33t-haxor">
	<role id="auto-l33t-haxor"/>
	</acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea">
	<role id="auto-badidea"/>
	</acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	<acl_target id="betteridea">
	<role id="auto-betteridea"/>
	</acl_target>
	<acl_role id="auto-betteridea">
	<acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
	<acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: niceguy: Query configuration - OK (0) =#=#=#=
	* Passed: cibadmin - niceguy: Query configuration
	=#=#=#= Begin test: niceguy: Set enable-acl =#=#=#=
	__xml_acl_check: 400 access denied to /cib/configuration/crm_config/cluster_property_set[@id='cib-bootstrap-options']/nvpair[@id='cib-bootstrap-options-enable-acl'][@value]: default
	Error performing operation: Permission denied
	Error setting enable-acl=false (section=crm_config, set=<null>): Permission denied
	=#=#=#= End test: niceguy: Set enable-acl - Permission denied (13) =#=#=#=
	* Passed: crm_attribute - niceguy: Set enable-acl
	=#=#=#= Begin test: niceguy: Set stonith-enabled =#=#=#=
	=#=#=#= Current cib after: niceguy: Set stonith-enabled =#=#=#=
	<cib epoch="8" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	<acls>
	<acl_target id="l33t-haxor">
	<role id="auto-l33t-haxor"/>
	</acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea">
	<role id="auto-badidea"/>
	</acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	<acl_target id="betteridea">
	<role id="auto-betteridea"/>
	</acl_target>
	<acl_role id="auto-betteridea">
	<acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
	<acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: niceguy: Set stonith-enabled - OK (0) =#=#=#=
	* Passed: crm_attribute - niceguy: Set stonith-enabled
	=#=#=#= Begin test: niceguy: Create a resource =#=#=#=
	__xml_acl_check: 400 access denied to /cib/configuration/resources/primitive[@id='dummy']: default
	__xml_acl_post_process: Cannot add new node primitive at /cib/configuration/resources/primitive[@id='dummy']
	Call failed: Permission denied
	=#=#=#= End test: niceguy: Create a resource - Permission denied (13) =#=#=#=
	* Passed: cibadmin - niceguy: Create a resource
	=#=#=#= Begin test: root: Query configuration =#=#=#=
	<cib epoch="8" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	<acls>
	<acl_target id="l33t-haxor">
	<role id="auto-l33t-haxor"/>
	</acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea">
	<role id="auto-badidea"/>
	</acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	<acl_target id="betteridea">
	<role id="auto-betteridea"/>
	</acl_target>
	<acl_role id="auto-betteridea">
	<acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
	<acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: root: Query configuration - OK (0) =#=#=#=
	* Passed: cibadmin - root: Query configuration
	=#=#=#= Begin test: root: Set stonith-enabled =#=#=#=
	=#=#=#= Current cib after: root: Set stonith-enabled =#=#=#=
	<cib epoch="9" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	<acls>
	<acl_target id="l33t-haxor">
	<role id="auto-l33t-haxor"/>
	</acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea">
	<role id="auto-badidea"/>
	</acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	<acl_target id="betteridea">
	<role id="auto-betteridea"/>
	</acl_target>
	<acl_role id="auto-betteridea">
	<acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
	<acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: root: Set stonith-enabled - OK (0) =#=#=#=
	* Passed: crm_attribute - root: Set stonith-enabled
	=#=#=#= Begin test: root: Create a resource =#=#=#=
	=#=#=#= Current cib after: root: Create a resource =#=#=#=
	<cib epoch="10" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy"/>
	</resources>
	<constraints/>
	<acls>
	<acl_target id="l33t-haxor">
	<role id="auto-l33t-haxor"/>
	</acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea">
	<role id="auto-badidea"/>
	</acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	<acl_target id="betteridea">
	<role id="auto-betteridea"/>
	</acl_target>
	<acl_role id="auto-betteridea">
	<acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
	<acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: root: Create a resource - OK (0) =#=#=#=
	* Passed: cibadmin - root: Create a resource
	=#=#=#= Begin test: l33t-haxor: Create a resource meta attribute =#=#=#=
	Error performing operation: Permission denied
	=#=#=#= End test: l33t-haxor: Create a resource meta attribute - Permission denied (13) =#=#=#=
	* Passed: crm_resource - l33t-haxor: Create a resource meta attribute
	=#=#=#= Begin test: l33t-haxor: Query a resource meta attribute =#=#=#=
	Error performing operation: Permission denied
	=#=#=#= End test: l33t-haxor: Query a resource meta attribute - Permission denied (13) =#=#=#=
	* Passed: crm_resource - l33t-haxor: Query a resource meta attribute
	=#=#=#= Begin test: l33t-haxor: Remove a resource meta attribute =#=#=#=
	Error performing operation: Permission denied
	=#=#=#= End test: l33t-haxor: Remove a resource meta attribute - Permission denied (13) =#=#=#=
	* Passed: crm_resource - l33t-haxor: Remove a resource meta attribute
	=#=#=#= Begin test: niceguy: Create a resource meta attribute =#=#=#=
	-( unpack.c:711 ) error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	-( unpack.c:712 ) error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	-( unpack.c:713 ) error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	+error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	+error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	+error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	__xml_acl_post_process: Creation of nvpair=dummy-meta_attributes-target-role is allowed
	=#=#=#= Current cib after: niceguy: Create a resource meta attribute =#=#=#=
	<cib epoch="11" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Stopped"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_target id="l33t-haxor">
	<role id="auto-l33t-haxor"/>
	</acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea">
	<role id="auto-badidea"/>
	</acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	<acl_target id="betteridea">
	<role id="auto-betteridea"/>
	</acl_target>
	<acl_role id="auto-betteridea">
	<acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
	<acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: niceguy: Create a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - niceguy: Create a resource meta attribute
	=#=#=#= Begin test: niceguy: Query a resource meta attribute =#=#=#=
	-( unpack.c:711 ) error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	-( unpack.c:712 ) error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	-( unpack.c:713 ) error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	+error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	+error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	+error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	Stopped
	=#=#=#= Current cib after: niceguy: Query a resource meta attribute =#=#=#=
	<cib epoch="11" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Stopped"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_target id="l33t-haxor">
	<role id="auto-l33t-haxor"/>
	</acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea">
	<role id="auto-badidea"/>
	</acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	<acl_target id="betteridea">
	<role id="auto-betteridea"/>
	</acl_target>
	<acl_role id="auto-betteridea">
	<acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
	<acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: niceguy: Query a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - niceguy: Query a resource meta attribute
	=#=#=#= Begin test: niceguy: Remove a resource meta attribute =#=#=#=
	-( unpack.c:711 ) error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	-( unpack.c:712 ) error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	-( unpack.c:713 ) error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	+error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	+error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	+error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	Deleted dummy option: id=dummy-meta_attributes-target-role name=target-role
	=#=#=#= Current cib after: niceguy: Remove a resource meta attribute =#=#=#=
	<cib epoch="12" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_target id="l33t-haxor">
	<role id="auto-l33t-haxor"/>
	</acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea">
	<role id="auto-badidea"/>
	</acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	<acl_target id="betteridea">
	<role id="auto-betteridea"/>
	</acl_target>
	<acl_role id="auto-betteridea">
	<acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
	<acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: niceguy: Remove a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - niceguy: Remove a resource meta attribute
	=#=#=#= Begin test: niceguy: Create a resource meta attribute =#=#=#=
	-( unpack.c:711 ) error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	-( unpack.c:712 ) error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	-( unpack.c:713 ) error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	+error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
	+error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
	+error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
	__xml_acl_post_process: Creation of nvpair=dummy-meta_attributes-target-role is allowed
	=#=#=#= Current cib after: niceguy: Create a resource meta attribute =#=#=#=
	<cib epoch="13" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_target id="l33t-haxor">
	<role id="auto-l33t-haxor"/>
	</acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea">
	<role id="auto-badidea"/>
	</acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	<acl_target id="betteridea">
	<role id="auto-betteridea"/>
	</acl_target>
	<acl_role id="auto-betteridea">
	<acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
	<acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: niceguy: Create a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - niceguy: Create a resource meta attribute
	=#=#=#= Begin test: badidea: Query configuration - implied deny =#=#=#=
	<cib>
	<configuration>
	<resources>
	<primitive id="dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	</configuration>
	</cib>
	=#=#=#= End test: badidea: Query configuration - implied deny - OK (0) =#=#=#=
	* Passed: cibadmin - badidea: Query configuration - implied deny
	=#=#=#= Begin test: betteridea: Query configuration - explicit deny =#=#=#=
	<cib>
	<configuration>
	<resources>
	<primitive id="dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	</configuration>
	</cib>
	=#=#=#= End test: betteridea: Query configuration - explicit deny - OK (0) =#=#=#=
	* Passed: cibadmin - betteridea: Query configuration - explicit deny
	<cib epoch="14" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= Begin test: niceguy: Replace - remove acls =#=#=#=
	__xml_acl_check: 400 access denied to /cib[@epoch]: default
	__xml_acl_check: 400 access denied to /cib/configuration/acls: default
	Call failed: Permission denied
	=#=#=#= End test: niceguy: Replace - remove acls - Permission denied (13) =#=#=#=
	* Passed: cibadmin - niceguy: Replace - remove acls
	<cib epoch="14" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	<primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy"/>
	</resources>
	<constraints/>
	<acls>
	<acl_target id="l33t-haxor">
	<role id="auto-l33t-haxor"/>
	</acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea">
	<role id="auto-badidea"/>
	</acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	<acl_target id="betteridea">
	<role id="auto-betteridea"/>
	</acl_target>
	<acl_role id="auto-betteridea">
	<acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
	<acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= Begin test: niceguy: Replace - create resource =#=#=#=
	__xml_acl_check: 400 access denied to /cib[@epoch]: default
	__xml_acl_check: 400 access denied to /cib/configuration/resources/primitive[@id='dummy2']: default
	__xml_acl_post_process: Cannot add new node primitive at /cib/configuration/resources/primitive[@id='dummy2']
	Call failed: Permission denied
	=#=#=#= End test: niceguy: Replace - create resource - Permission denied (13) =#=#=#=
	* Passed: cibadmin - niceguy: Replace - create resource
	<cib epoch="14" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="false"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_target id="l33t-haxor">
	<role id="auto-l33t-haxor"/>
	</acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea">
	<role id="auto-badidea"/>
	</acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	<acl_target id="betteridea">
	<role id="auto-betteridea"/>
	</acl_target>
	<acl_role id="auto-betteridea">
	<acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
	<acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= Begin test: niceguy: Replace - modify attribute =#=#=#=
	__xml_acl_check: 400 access denied to /cib[@epoch]: default
	__xml_acl_check: 400 access denied to /cib/configuration/crm_config/cluster_property_set[@id='cib-bootstrap-options']/nvpair[@id='cib-bootstrap-options-enable-acl'][@value]: default
	Call failed: Permission denied
	=#=#=#= End test: niceguy: Replace - modify attribute - Permission denied (13) =#=#=#=
	* Passed: cibadmin - niceguy: Replace - modify attribute
	<cib epoch="14" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_target id="l33t-haxor">
	<role id="auto-l33t-haxor"/>
	</acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea">
	<role id="auto-badidea"/>
	</acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	<acl_target id="betteridea">
	<role id="auto-betteridea"/>
	</acl_target>
	<acl_role id="auto-betteridea">
	<acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
	<acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= Begin test: niceguy: Replace - delete attribute =#=#=#=
	__xml_acl_check: 400 access denied to /cib[@epoch]: default
	__xml_acl_check: 400 access denied to /cib/configuration/crm_config/cluster_property_set[@id='cib-bootstrap-options']/nvpair[@id='cib-bootstrap-options-enable-acl']: default
	Call failed: Permission denied
	=#=#=#= End test: niceguy: Replace - delete attribute - Permission denied (13) =#=#=#=
	* Passed: cibadmin - niceguy: Replace - delete attribute
	<cib epoch="14" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy" description="nothing interesting">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	<acls>
	<acl_target id="l33t-haxor">
	<role id="auto-l33t-haxor"/>
	</acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea">
	<role id="auto-badidea"/>
	</acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	<acl_target id="betteridea">
	<role id="auto-betteridea"/>
	</acl_target>
	<acl_role id="auto-betteridea">
	<acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
	<acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
	</acl_role>
	</acls>
	</configuration>
	<status/>
	</cib>
	=#=#=#= Begin test: niceguy: Replace - create attribute =#=#=#=
	__xml_acl_check: 400 access denied to /cib[@epoch]: default
	__xml_acl_check: 400 access denied to /cib/configuration/resources/primitive[@id='dummy'][@description]: default
	Call failed: Permission denied
	=#=#=#= End test: niceguy: Replace - create attribute - Permission denied (13) =#=#=#=
	* Passed: cibadmin - niceguy: Replace - create attribute
	diff --git a/tools/regression.sh b/tools/regression.sh
	index 9f290eac2a..285dcbacae 100755
	--- a/tools/regression.sh
	+++ b/tools/regression.sh
	@@ -1,622 +1,623 @@
	#!/bin/bash

	: ${shadow=tools-regression}
	test_home=`dirname $0`
	num_errors=0
	num_passed=0
	GREP_OPTIONS=
	verbose=0
	tests="dates tools acls"

	function test_assert() {
	target=$1; shift
	cib=$1; shift
	app=`echo "$cmd" \| sed 's/\ .*//'`
	printf "* Running: $app - $desc\n" 1>&2

	printf "=#=#=#= Begin test: $desc =#=#=#=\n"
	eval $VALGRIND_CMD $cmd 2>&1
	rc=$?

	if [ x$cib != x0 ]; then
	printf "=#=#=#= Current cib after: $desc =#=#=#=\n"
	CIB_user=root cibadmin -Q
	fi

	printf "=#=#=#= End test: $desc - `crm_error $rc` ($rc) =#=#=#=\n"

	if [ $rc -ne $target ]; then
	num_errors=`expr $num_errors + 1`
	printf "* Failed (rc=%.3d): %-14s - %s\n" $rc $app "$desc"
	printf "* Failed (rc=%.3d): %-14s - %s\n" $rc $app "$desc (`which $app`)" 1>&2
	return
	exit 1
	else
	printf "* Passed: %-14s - %s\n" $app "$desc"

	num_passed=`expr $num_passed + 1`
	fi
	}

	function usage() {
	echo "Usage: ./regression.sh [-s(ave)] [-x] [-v(erbose)]"
	exit $1
	}

	done=0
	do_save=0
	VALGRIND_CMD=
	while test "$done" = "0"; do
	case "$1" in
	-t) tests=$2; shift; shift;;
	-V\|--verbose) verbose=1; shift;;
	-v\|--valgrind)
	export G_SLICE=always-malloc
	VALGRIND_CMD="valgrind -q --gen-suppressions=all --show-reachable=no --leak-check=full --trace-children=no --time-stamp=yes --num-callers=20 --suppressions=/usr/share/pacemaker/tests/valgrind-pcmk.suppressions"
	shift;;
	-x) set -x; shift;;
	-s) do_save=1; shift;;
	-p) PATH="$2:$PATH"; export PATH; shift 1;;
	-?) usage 0;;
	-*) echo "unknown option: $1"; usage 1;;
	*) done=1;;
	esac
	done

	if [ "x$VALGRIND_CMD" = "x" -a -x $test_home/crm_simulate ]; then
	xml_home=`dirname ${test_home}`
	echo "Using local binaries from: $test_home, schemas from $xml_home"
	export PATH="$test_home:$PATH"
	export PCMK_schema_directory=${xml_home}/xml
	fi

	function test_tools() {
	export CIB_shadow_dir=$test_home
	$VALGRIND_CMD crm_shadow --batch --force --create-empty $shadow 2>&1
	export CIB_shadow=$shadow

	desc="Validate CIB"
	cmd="cibadmin -Q"
	test_assert 0

	desc="Configure something before erasing"
	cmd="crm_attribute -n cluster-delay -v 60s"
	test_assert 0

	desc="Require --force for CIB erasure"
	cmd="cibadmin -E"
	test_assert 22

	desc="Allow CIB erasure with --force"
	cmd="cibadmin -E --force"
	test_assert 0

	desc="Query CIB"
	cmd="cibadmin -Q > /tmp/$$.existing.xml"
	test_assert 0

	desc="Set cluster option"
	cmd="crm_attribute -n cluster-delay -v 60s"
	test_assert 0

	desc="Query new cluster option"
	cmd="cibadmin -Q -o crm_config \| grep cib-bootstrap-options-cluster-delay"
	test_assert 0

	desc="Query cluster options"
	cmd="cibadmin -Q -o crm_config > /tmp/$$.opt.xml"
	test_assert 0

	desc="Set no-quorum policy"
	cmd="crm_attribute -n no-quorum-policy -v ignore"
	test_assert 0

	desc="Delete nvpair"
	cmd="cibadmin -D -o crm_config --xml-text '<nvpair id=\"cib-bootstrap-options-cluster-delay\"/>'"
	test_assert 0

	desc="Create operaton should fail"
	cmd="cibadmin -C -o crm_config --xml-file /tmp/$$.opt.xml"
	test_assert 76

	desc="Modify cluster options section"
	cmd="cibadmin -M -o crm_config --xml-file /tmp/$$.opt.xml"
	test_assert 0

	desc="Query updated cluster option"
	cmd="cibadmin -Q -o crm_config \| grep cib-bootstrap-options-cluster-delay"
	test_assert 0

	desc="Set duplicate cluster option"
	cmd="crm_attribute -n cluster-delay -v 40s -s duplicate"
	test_assert 0

	desc="Setting multiply defined cluster option should fail"
	cmd="crm_attribute -n cluster-delay -v 30s"
	test_assert 76

	desc="Set cluster option with -s"
	cmd="crm_attribute -n cluster-delay -v 30s -s duplicate"
	test_assert 0

	desc="Delete cluster option with -i"
	cmd="crm_attribute -n cluster-delay -D -i cib-bootstrap-options-cluster-delay"
	test_assert 0

	desc="Create node1 and bring it online"
	cmd="crm_simulate --live-check --in-place --node-up=node1"
	test_assert 0

	desc="Create node attribute"
	cmd="crm_attribute -n ram -v 1024M -U node1 -t nodes"
	test_assert 0

	desc="Query new node attribute"
	cmd="cibadmin -Q -o nodes \| grep node1-ram"
	test_assert 0

	desc="Digest calculation"
	cmd="cibadmin -Q \| cibadmin -5 -p 2>&1 > /dev/null"
	test_assert 0

	# This update will fail because it has version numbers
	desc="Replace operation should fail"
	cmd="cibadmin -R --xml-file /tmp/$$.existing.xml"
	test_assert 205

	desc="Default standby value"
	cmd="crm_standby -N node1 -G"
	test_assert 0

	desc="Set standby status"
	cmd="crm_standby -N node1 -v true"
	test_assert 0

	desc="Query standby value"
	cmd="crm_standby -N node1 -G"
	test_assert 0

	desc="Delete standby value"
	cmd="crm_standby -N node1 -D"
	test_assert 0

	desc="Create a resource"
	cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
	test_assert 0

	desc="Create a resource meta attribute"
	cmd="crm_resource -r dummy --meta -p is-managed -v false"
	test_assert 0

	desc="Query a resource meta attribute"
	cmd="crm_resource -r dummy --meta -g is-managed"
	test_assert 0

	desc="Remove a resource meta attribute"
	cmd="crm_resource -r dummy --meta -d is-managed"
	test_assert 0

	desc="Create a resource attribute"
	cmd="crm_resource -r dummy -p delay -v 10s"
	test_assert 0

	desc="List the configured resources"
	cmd="crm_resource -L"
	test_assert 0

	desc="Set a resource's fail-count"
	cmd="crm_failcount -r dummy -v 10 -N node1"
	test_assert 0

	desc="Require a destination when migrating a resource that is stopped"
	cmd="crm_resource -r dummy -M"
	test_assert 22

	desc="Don't support migration to non-existant locations"
	cmd="crm_resource -r dummy -M -N i.dont.exist"
	test_assert 6

	desc="Create a fencing resource"
	cmd="cibadmin -C -o resources --xml-text '<primitive id=\"Fence\" class=\"stonith\" type=\"fence_true\"/>'"
	test_assert 0

	desc="Bring resources online"
	cmd="crm_simulate --live-check --in-place -S"
	test_assert 0

	desc="Try to move a resource to its existing location"
	cmd="crm_resource -r dummy --move --host node1"
	test_assert 22

	desc="Move a resource from its existing location"
	cmd="crm_resource -r dummy --move"
	test_assert 0

	desc="Clear out constraints generated by --move"
	cmd="crm_resource -r dummy --clear"
	test_assert 0

	desc="Default ticket granted state"
	cmd="crm_ticket -t ticketA -G granted -d false"
	test_assert 0

	desc="Set ticket granted state"
	cmd="crm_ticket -t ticketA -r --force"
	test_assert 0

	desc="Query ticket granted state"
	cmd="crm_ticket -t ticketA -G granted"
	test_assert 0

	desc="Delete ticket granted state"
	cmd="crm_ticket -t ticketA -D granted --force"
	test_assert 0

	desc="Make a ticket standby"
	cmd="crm_ticket -t ticketA -s"
	test_assert 0

	desc="Query ticket standby state"
	cmd="crm_ticket -t ticketA -G standby"
	test_assert 0

	desc="Activate a ticket"
	cmd="crm_ticket -t ticketA -a"
	test_assert 0

	desc="Delete ticket standby state"
	cmd="crm_ticket -t ticketA -D standby"
	test_assert 0

	desc="Ban a resource on unknown node"
	cmd="crm_resource -r dummy -B -N host1"
	test_assert 6

	desc="Create two more nodes and bring them online"
	cmd="crm_simulate --live-check --in-place --node-up=node2 --node-up=node3"
	test_assert 0

	desc="Ban dummy from node1"
	cmd="crm_resource -r dummy -B -N node1"
	test_assert 0

	desc="Ban dummy from node2"
	cmd="crm_resource -r dummy -B -N node2"
	test_assert 0

	desc="Relocate resources due to ban"
	cmd="crm_simulate --live-check --in-place -S"
	test_assert 0

	desc="Move dummy to node1"
	cmd="crm_resource -r dummy -M -N node1"
	test_assert 0

	desc="Clear implicit constraints for dummy on node2"
	cmd="crm_resource -r dummy -U -N node2"
	test_assert 0
	}

	function test_dates() {
	desc="2014-01-01 00:30:00 - 1 Hour"
	cmd="iso8601 -d '2014-01-01 00:30:00Z' -D P-1H -E '2013-12-31 23:30:00Z'"
	test_assert 0 0

	for y in 06 07 08 09 10 11 12 13 14 15 16 17 18; do
	desc="20$y-W01-7"
	cmd="iso8601 -d '20$y-W01-7 00Z'"
	test_assert 0 0

	desc="20$y-W01-7 - round-trip"
	cmd="iso8601 -d '20$y-W01-7 00Z' -W -E '20$y-W01-7 00:00:00Z'"
	test_assert 0 0

	desc="20$y-W01-1"
	cmd="iso8601 -d '20$y-W01-1 00Z'"
	test_assert 0 0

	desc="20$y-W01-1 - round-trip"
	cmd="iso8601 -d '20$y-W01-1 00Z' -W -E '20$y-W01-1 00:00:00Z'"
	test_assert 0 0
	done

	desc="2009-W53-07"
	cmd="iso8601 -d '2009-W53-7 00:00:00Z' -W -E '2009-W53-7 00:00:00Z'"
	test_assert 0 0

	desc="2009-01-31 + 1 Month"
	cmd="iso8601 -d '2009-01-31 00:00:00Z' -D P1M -E '2009-02-28 00:00:00Z'"
	test_assert 0 0

	desc="2009-01-31 + 2 Months"
	cmd="iso8601 -d '2009-01-31 00:00:00Z' -D P2M -E '2009-03-31 00:00:00Z'"
	test_assert 0 0

	desc="2009-01-31 + 3 Months"
	cmd="iso8601 -d '2009-01-31 00:00:00Z' -D P3M -E '2009-04-30 00:00:00Z'"
	test_assert 0 0

	desc="2009-03-31 - 1 Month"
	cmd="iso8601 -d '2009-03-31 00:00:00Z' -D P-1M -E '2009-02-28 00:00:00Z'"
	test_assert 0 0
	}

	function get_epoch() {
	CIB_user=root CIB_file=$1 CIB_shadow="" cibadmin -Q \| head -n 1 \| sed -e 's/.* epoch=\"$[0-9]$./\1/'
	}

	function restore_epoch() {
	infile=$1; shift
	old=$1; shift
	new=$(get_epoch $infile)

	sed -i 's/epoch=.$old/epoch=\"$new/g' $infile
	}

	function test_acl_loop() {
	# Make sure we're rejecting things for the right reasons
	export PCMK_trace_functions=__xml_acl_check,__xml_acl_post_process
	export PCMK_stderr=1

	CIB_user=root cibadmin --replace --xml-text '<resources/>'

	export CIB_user=unknownguy
	desc="$CIB_user: Query configuration"
	cmd="cibadmin -Q"
	test_assert 13 0

	desc="$CIB_user: Set enable-acl"
	cmd="crm_attribute -n enable-acl -v false"
	test_assert 13 0

	desc="$CIB_user: Set stonith-enabled"
	cmd="crm_attribute -n stonith-enabled -v false"
	test_assert 13 0

	desc="$CIB_user: Create a resource"
	cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
	test_assert 13 0

	export CIB_user=l33t-haxor
	desc="$CIB_user: Query configuration"
	cmd="cibadmin -Q"
	test_assert 13 0

	desc="$CIB_user: Set enable-acl"
	cmd="crm_attribute -n enable-acl -v false"
	test_assert 13 0

	desc="$CIB_user: Set stonith-enabled"
	cmd="crm_attribute -n stonith-enabled -v false"
	test_assert 13 0

	desc="$CIB_user: Create a resource"
	cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
	test_assert 13 0

	export CIB_user=niceguy
	desc="$CIB_user: Query configuration"
	cmd="cibadmin -Q"
	test_assert 0 0

	desc="$CIB_user: Set enable-acl"
	cmd="crm_attribute -n enable-acl -v false"
	test_assert 13 0

	desc="$CIB_user: Set stonith-enabled"
	cmd="crm_attribute -n stonith-enabled -v false"
	test_assert 0

	desc="$CIB_user: Create a resource"
	cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
	test_assert 13 0

	export CIB_user=root
	desc="$CIB_user: Query configuration"
	cmd="cibadmin -Q"
	test_assert 0 0

	desc="$CIB_user: Set stonith-enabled"
	cmd="crm_attribute -n stonith-enabled -v true"
	test_assert 0

	desc="$CIB_user: Create a resource"
	cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
	test_assert 0

	export CIB_user=l33t-haxor

	desc="$CIB_user: Create a resource meta attribute"
	cmd="crm_resource -r dummy --meta -p target-role -v Stopped"
	test_assert 13 0

	desc="$CIB_user: Query a resource meta attribute"
	cmd="crm_resource -r dummy --meta -g target-role"
	test_assert 13 0

	desc="$CIB_user: Remove a resource meta attribute"
	cmd="crm_resource -r dummy --meta -d target-role"
	test_assert 13 0

	export CIB_user=niceguy

	desc="$CIB_user: Create a resource meta attribute"
	cmd="crm_resource -r dummy --meta -p target-role -v Stopped"
	test_assert 0

	desc="$CIB_user: Query a resource meta attribute"
	cmd="crm_resource -r dummy --meta -g target-role"
	test_assert 0

	desc="$CIB_user: Remove a resource meta attribute"
	cmd="crm_resource -r dummy --meta -d target-role"
	test_assert 0

	desc="$CIB_user: Create a resource meta attribute"
	cmd="crm_resource -r dummy --meta -p target-role -v Started"
	test_assert 0

	export CIB_user=badidea
	desc="$CIB_user: Query configuration - implied deny"
	cmd="cibadmin -Q"
	test_assert 0 0

	export CIB_user=betteridea
	desc="$CIB_user: Query configuration - explicit deny"
	cmd="cibadmin -Q"
	test_assert 0 0

	CIB_user=root cibadmin -Q > /tmp/$$.haxor.xml
	CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin --delete --xml-text '<acls/>'
	CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -Ql

	export CIB_user=niceguy
	desc="$CIB_user: Replace - remove acls"
	cmd="cibadmin --replace --xml-file /tmp/$$.haxor.xml"
	test_assert 13 0

	CIB_user=root cibadmin -Q > /tmp/$$.haxor.xml
	CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -C -o resources --xml-text '<primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy"/>'
	CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -Ql

	desc="$CIB_user: Replace - create resource"
	cmd="cibadmin --replace --xml-file /tmp/$$.haxor.xml"
	test_assert 13 0

	CIB_user=root cibadmin -Q > /tmp/$$.haxor.xml
	CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" crm_attribute -n enable-acl -v false
	CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -Ql

	desc="$CIB_user: Replace - modify attribute"
	cmd="cibadmin --replace --xml-file /tmp/$$.haxor.xml"
	test_assert 13 0

	CIB_user=root cibadmin -Q > /tmp/$$.haxor.xml
	CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin --replace --xml-text '<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl"/>'
	CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -Ql

	desc="$CIB_user: Replace - delete attribute"
	cmd="cibadmin --replace --xml-file /tmp/$$.haxor.xml"
	test_assert 13 0

	CIB_user=root cibadmin -Q > /tmp/$$.haxor.xml
	CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin --modify --xml-text '<primitive id="dummy" description="nothing interesting"/>'
	CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -Ql

	desc="$CIB_user: Replace - create attribute"
	cmd="cibadmin --replace --xml-file /tmp/$$.haxor.xml"
	test_assert 13 0
	rm -rf /tmp/$$.haxor.xml
	}

	function test_acls() {
	export CIB_shadow_dir=$test_home
	$VALGRIND_CMD crm_shadow --batch --force --create-empty $shadow --validate-with pacemaker-1.3 2>&1
	export CIB_shadow=$shadow

	cat<<EOF>/tmp/$$.acls.xml
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	</acls>
	EOF

	desc="Configure some ACLs"
	cmd="cibadmin -M -o acls --xml-file /tmp/$$.acls.xml"
	test_assert 0

	desc="Enable ACLs"
	cmd="crm_attribute -n enable-acl -v true"
	test_assert 0

	desc="Set cluster option"
	cmd="crm_attribute -n no-quorum-policy -v ignore"
	test_assert 0

	desc="New ACL"
	cmd="cibadmin --create -o acls --xml-text '<acl_user id=\"badidea\"><read id=\"badidea-resources\" xpath=\"//meta_attributes\"/></acl_user>'"
	test_assert 0

	desc="Another ACL"
	cmd="cibadmin --create -o acls --xml-text '<acl_user id=\"betteridea\"><read id=\"betteridea-resources\" xpath=\"//meta_attributes\"/></acl_user>'"
	test_assert 0

	desc="Updated ACL"
	cmd="cibadmin --replace -o acls --xml-text '<acl_user id=\"betteridea\"><deny id=\"betteridea-nothing\" xpath=\"/cib\"/><read id=\"betteridea-resources\" xpath=\"//meta_attributes\"/></acl_user>'"
	test_assert 0

	test_acl_loop

	printf "\n\n !#!#!#!#! Upgrading to pacemaker-2.0 and retesting !#!#!#!#!\n"
	printf "\nUpgrading to pacemaker-2.0 and re-testing\n" 1>&2

	export CIB_user=root
	desc="$CIB_user: Upgrade to pacemaker-2.0"
	cmd="cibadmin --upgrade --force -V"
	test_assert 0

	sed -i 's/epoch=.2/epoch=\"6/g' $CIB_shadow_dir/shadow.$CIB_shadow
	sed -i 's/admin_epoch=.1/admin_epoch=\"0/g' $CIB_shadow_dir/shadow.$CIB_shadow

	test_acl_loop
	}

	for t in $tests; do
	echo "Testing $t"
	test_$t > $test_home/regression.$t.out

	sed -i -e 's/cib-last-written.*>/>/'\
	-e 's/ last-run=\"[0-9]*\"//' \
	-e 's/crm_feature_set="[^"]*" //'\
	-e 's/validate-with="[^"]*" //'\
	-e 's/.*__xml_acl_check/__xml_acl_check/g'\
	-e 's/.*__xml_acl_post_process/__xml_acl_post_process/g'\
	+ -e 's/.*error: unpack_resources:/error: unpack_resources:/g'\
	-e 's/ last-rc-change=\"[0-9]*\"//' $test_home/regression.$t.out

	if [ $do_save = 1 ]; then
	cp $test_home/regression.$t.out $test_home/regression.$t.exp
	fi
	done

	failed=0

	echo -e "\n\nResults"
	for t in $tests; do
	if [ $do_save = 1 ]; then
	cp $test_home/regression.$t.out $test_home/regression.$t.exp
	fi
	if [ $verbose = 1 ]; then
	diff -wu $test_home/regression.$t.exp $test_home/regression.$t.out
	else
	diff -wu $test_home/regression.$t.exp $test_home/regression.$t.out
	fi
	if [ $? != 0 ]; then
	failed=1
	fi
	done


	echo -e "\n\nSummary"
	for t in $tests; do
	grep -e "^*" $test_home/regression.$t.out
	done

	if [ $num_errors != 0 ]; then
	echo $num_errors tests failed
	exit 1
	elif [ $failed = 1 ]; then
	echo $num_passed tests passed but diff failed
	exit 2
	else
	echo $num_passed tests passed
	exit 0
	fi
	diff --git a/tools/report.common.in b/tools/report.common.in
	index 78c8370f90..9dc57695d2 100644
	--- a/tools/report.common.in
	+++ b/tools/report.common.in
	@@ -1,804 +1,805 @@
	# Copyright (C) 2007 Dejan Muhamedagic <dmuhamedagic@suse.de>
	# Almost everything as part of hb_report
	# Copyright (C) 2010 Andrew Beekhof <andrew@beekhof.net>
	# Cleanups, refactoring, extensions
	#
	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public
	# License as published by the Free Software Foundation; either
	# version 2.1 of the License, or (at your option) any later version.
	#
	# This software is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	# General Public License for more details.
	#
	# You should have received a copy of the GNU General Public
	# License along with this library; if not, write to the Free Software
	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	#

	host=`uname -n`
	shorthost=`echo $host \| sed s:\\\\..*::`
	if [ -z $verbose ]; then
	verbose=0
	fi

	# Target Files
	EVENTS_F=events.txt
	ANALYSIS_F=analysis.txt
	DESCRIPTION_F=description.txt
	HALOG_F=cluster-log.txt
	BT_F=backtraces.txt
	SYSINFO_F=sysinfo.txt
	SYSSTATS_F=sysstats.txt
	DLM_DUMP_F=dlm_dump.txt
	CRM_MON_F=crm_mon.txt
	MEMBERSHIP_F=members.txt
	HB_UUID_F=hb_uuid.txt
	HOSTCACHE=hostcache
	CRM_VERIFY_F=crm_verify.txt
	PERMISSIONS_F=permissions.txt
	CIB_F=cib.xml
	CIB_TXT_F=cib.txt

	EVENT_PATTERNS="
	state do_state_transition
	membership pcmk_peer_update.*(lost\|memb):
	quorum crmd.crm_update_quorum\|crmd.ais.disp.*quorum.(lost\|ac?quir)
	pause Process.pause.detected
	resources lrmd.*rsc:(start\|stop)
	stonith te_fence_node\|stonith-ng.log_oper.report\|stonithd.*(requests\|(Succeeded\|Failed).to.STONITH\|result=)
	start_stop sutdown.decision\|Starting.heartbeat\|Corosync.Cluster.Engine\|corosync.*Initializing.transport\|Executive.Service.RELEASE\|crm_shutdown:.Requesting.shutdown\|pcmk_shutdown:.Shutdown.complete
	"

	PACKAGES="pacemaker pacemaker-libs libpacemaker3
	pacemaker-pygui pacemaker-pymgmt pymgmt-client
	openais libopenais2 libopenais3 corosync libcorosync4
	resource-agents cluster-glue-libs cluster-glue libglue2 ldirectord
	heartbeat heartbeat-common heartbeat-resources libheartbeat2
	ocfs2-tools ocfs2-tools-o2cb ocfs2console
	ocfs2-kmp-default ocfs2-kmp-pae ocfs2-kmp-xen ocfs2-kmp-debug ocfs2-kmp-trace
	drbd drbd-kmp-xen drbd-kmp-pae drbd-kmp-default drbd-kmp-debug drbd-kmp-trace
	drbd-heartbeat drbd-pacemaker drbd-utils drbd-bash-completion drbd-xen
	lvm2 lvm2-clvm cmirrord
	libdlm libdlm2 libdlm3
	hawk ruby lighttpd
	kernel-default kernel-pae kernel-xen
	glibc
	"

	#
	# keep the user posted
	#
	record() {
	if [ x != x"$REPORT_HOME" -a -d "${REPORT_HOME}/$shorthost" ]; then
	rec="${REPORT_HOME}/$shorthost/report.out"

	elif [ x != x"${l_base}" -a -d "${l_base}" ]; then
	rec="${l_base}/report.summary"

	else
	rec="/dev/null"
	fi
	printf "%-10s $*\n" "$shorthost:" 2>&1 >> "${rec}"
	}

	log() {
	printf "%-10s $*\n" "$shorthost:" 1>&2
	record "$*"
	}

	debug() {
	if [ $verbose -gt 0 ]; then
	log "Debug: $*"
	fi
	record "Debug: $*"
	}

	info() {
	log "$*"
	}

	warning() {
	log "WARN: $*"
	}

	fatal() {
	log "ERROR: $*"
	exit 1
	}

	detect_host() {
	if [ -z "$maxdepth" ]; then
	depth="-maxdepth 5"
	else
	depth="-maxdepth $maxdepth"
	fi

	local_state_dir=@localstatedir@

	if [ -d $local_state_dir/run ]; then
	CRM_STATE_DIR=$local_state_dir/run/crm
	else
	info "Searching for where Pacemaker keeps runtime data... this may take a while"
	for d in `find / $depth -type d -name run`; do
	local_state_dir=`dirname $d`
	CRM_STATE_DIR=$d/crm
	break
	done
	info "Found: $CRM_STATE_DIR"
	fi
	debug "Machine runtime directory: $local_state_dir"
	debug "Pacemaker runtime data located in: $CRM_STATE_DIR"

	CRM_DAEMON_DIR=
	for p in /usr /usr/local /opt/local @exec_prefix@; do
	for d in libexec lib64 lib; do
	if [ -e $p/$d/pacemaker/pengine ]; then
	CRM_DAEMON_DIR=$p/$d/pacemaker
	break
	elif [ -e $p/$d/heartbeat/pengine ]; then
	CRM_DAEMON_DIR=$p/$d/heartbeat
	break
	fi
	done
	done

	if [ ! -d $CRM_DAEMON_DIR ]; then
	info "Searching for where Pacemaker daemons live... this may take a while"
	for f in `find / $depth -type f -name pengine`; do
	CRM_DAEMON_DIR=`dirname $f`
	break
	done
	info "Found: $CRM_DAEMON_DIR"
	fi

	if [ -z $CRM_DAEMON_DIR ]; then
	fatal "Non-standard Pacemaker installation: daemons not found"
	else
	debug "Pacemaker daemons located under: $CRM_DAEMON_DIR"
	fi

	CRM_CONFIG_DIR=
	for d in pacemaker/cib heartbeat/crm; do
	if [ -f $local_state_dir/lib/$d/cib.xml ]; then
	CRM_CONFIG_DIR=$local_state_dir/lib/$d
	break
	fi
	done

	if [ ! -d $CRM_CONFIG_DIR ]; then
	info "Detecting where Pacemaker keeps config information... this may take a while"
	for f in `find / $depth -type f -name cib.xml`; do
	CRM_CONFIG_DIR=`dirname $f`
	break
	done
	info "Found: $CRM_CONFIG_DIR"
	fi
	if [ -z $CRM_CONFIG_DIR ]; then
	warning "Non-standard Pacemaker installation: config not found"
	else
	debug "Pacemaker config files located in: $CRM_CONFIG_DIR"
	fi

	# Assume new layout
	# $local_state_dir/lib/pacemaker/(cib,pengine,blackbox,cores)
	config_root=`dirname $CRM_CONFIG_DIR`

	# Older versions had none
	BLACKBOX_DIR=$config_root/blackbox
	debug "Pacemaker blackboxes (if any) located in: $BLACKBOX_DIR"

	PE_STATE_DIR=$config_root/pengine
	if [ ! -d $PE_STATE_DIR ]; then
	info "Detecting where Pacemaker keeps Policy Engine inputs... this may take a while"
	for d in `find / $depth -type d -name pengine`; do
	PE_STATE_DIR=$d
	break
	done
	info "Found: $PE_STATE_DIR"
	fi
	if [ -z $PE_STATE_DIR ]; then
	fatal "Non-standard Pacemaker installation: Policy Engine directory not found"
	else
	debug "PE files located in: $PE_STATE_DIR"
	fi

	HA_STATE_DIR=$local_state_dir/lib/heartbeat
	debug "Assuming Heartbeat state files, if any, are located in: $HA_STATE_DIR"

	CRM_CORE_DIRS=""
	for d in $config_root/cores $HA_STATE_DIR/cores $local_state_dir/lib/corosync $local_state_dir/lib/openais; do
	if [ -d $d ]; then
	CRM_CORE_DIRS="$CRM_CORE_DIRS $d"
	fi
	done
	debug "Core files located under: $CRM_CORE_DIRS"
	}

	time2str() {
	perl -e "use POSIX; print strftime('%x %X',localtime($1));"
	}

	get_time() {
	perl -e "\$time=\"$*\";" -e '
	$unix_tm = 0;
	eval "use Date::Parse";
	if (index($time, ":") < 0) {
	} elsif (!$@) {
	$unix_tm = str2time($time);
	} else {
	eval "use Date::Manip";
	if (!$@) {
	$unix_tm = UnixDate(ParseDateString($time), "%s");
	}
	}
	if ($unix_tm != "") {
	print int($unix_tm);
	} else {
	print "";
	}
	'
	}

	get_time_() {
	warning "Unknown time format used by: $*"
	}

	get_time_syslog() {
	awk '{print $1,$2,$3}'
	}

	get_time_legacy() {
	awk '{print $2}' \| sed 's/_/ /'
	}

	get_time_iso8601() {
	awk '{print $1}'
	}

	get_time_format_for_string() {
	l="$*"
	t=$(get_time `echo $l \| get_time_syslog`)
	if [ "x$t" != x ]; then
	echo syslog
	return
	fi

	t=$(get_time `echo $l \| get_time_iso8601`)
	if [ "x$t" != x ]; then
	echo iso8601
	return
	fi

	t=$(get_time `echo $l \| get_time_legacy`)
	if [ "x$t" != x ]; then
	echo legacy
	return
	fi
	}

	get_time_format() {
	t=0 l="" func=""
	trycnt=10
	while [ $trycnt -gt 0 ] && read l; do
	func=$(get_time_format_for_string $l)
	if [ "x$func" != x ]; then
	break
	fi
	trycnt=$(($trycnt-1))
	done
	#debug "Logfile uses the $func time format"
	echo $func
	}

	get_first_time() {
	l=""
	format=$1
	while read l; do
	t=$(echo $l \| get_time_$format)
	ts=$(get_time $t)
	if [ "x$ts" != x ]; then
	echo "$ts"
	return
	fi
	done
	}

	get_last_time() {
	l=""
	best=`date +%s` # Now
	format=$1
	while read l; do
	t=$(echo $l \| get_time_$format)
	ts=$(get_time $t)
	if [ "x$ts" != x ]; then
	best=$ts
	fi
	done
	echo $best
	}

	linetime() {
	l=`tail -n +$2 $1 \| grep -a ":[0-5][0-9]:" \| head -n 1`
	format=`get_time_format_for_string $l`
	t=`echo $l \| get_time_$format`
	get_time "$t"
	}

	# Find pattern in a logfile somewhere
	# Return $max ordered results by age (newest first)
	findmsg() {
	max=$1
	pattern=$2
	logfiles=""
	syslogdirs="/var/log /var/logs /var/syslog /var/adm /var/log/ha /var/log/cluster"

	for d in $syslogdirs; do
	if [ -d $d ]; then
	files=`find $d -type f -maxdepth 1`
	for f in $files; do
	local cat=`find_decompressor $f`
	$cat $f \| grep -l -e "$pattern"
	if [ $? = 0 ]; then
	logfiles="$logfiles $f"
	fi
	done
	fi
	done 2>/dev/null

	if [ "x$logfiles" != "x" ]; then
	list=`ls -t $logfiles \| head -n $max \| tr '\n' ' '`
	echo $list
	debug "Pattern \'$pattern\' found in: [ $list ]"
	else
	debug "Pattern \'$pattern\' not found anywhere"
	fi
	}

	node_events() {
	if [ -e $1 ]; then
	Epatt=`echo "$EVENT_PATTERNS" \|
	while read title p; do [ -n "$p" ] && echo -n "\|$p"; done \|
	sed 's/.//'
	`
	grep -E "$Epatt" $1
	fi
	}

	pickfirst() {
	for x; do
	which $x >/dev/null 2>&1 && {
	echo $x
	return 0
	}
	done
	return 1
	}

	shrink() {
	olddir=$PWD
	dir=`dirname $1`
	base=`basename $1`

	target=$1.tar
	tar_options="cf"

	variant=`pickfirst bzip2 gzip xz false`
	case $variant in
	bz*)
	tar_options="jcf"
	target="$target.bz2"
	;;
	gz*)
	tar_options="zcf"
	target="$target.gz"
	;;
	xz*)
	tar_options="Jcf"
	target="$target.xz"
	;;
	*)
	warning "Could not find a compression program, the resulting tarball may be huge"
	;;
	esac

	if [ -e $target ]; then
	fatal "Destination $target already exists, specify an alternate name with --dest"
	fi

	cd $dir >/dev/null 2>&1
	tar $tar_options $target $base >/dev/null 2>&1
	cd $olddir >/dev/null 2>&1

	echo $target
	}

	findln_by_time() {
	local logf=$1
	local tm=$2
	local first=1

	- local fileSize=`ls --block-size=M $logf \| awk '{ print $5 }' \| sed s/M//`
	- if [ $fileSize -gt 1024 ]; then
	- # Some logs can be massive (over 1,500,000,000 lines have been seen in the wild)
	- # Even just 'wc -l' on these files can take 10+ minutes
	- warning "$logf is ${fileSize}M in size and could take hours to process. Skipping."
	+ # Some logs can be massive (over 1,500,000,000 lines have been seen in the wild)
	+ # Even just 'wc -l' on these files can take 10+ minutes
	+
	+ local fileSize=`ls -lh \| awk '{ print $5 }' \| grep -ie G`
	+ if [ x$fileSize != x ]; then
	+ warning "$logf is ${fileSize} in size and could take many hours to process. Skipping."
	return
	fi

	local last=`wc -l < $logf`
	while [ $first -le $last ]; do
	mid=$((($last+$first)/2))
	trycnt=10
	while [ $trycnt -gt 0 ]; do
	tmid=`linetime $logf $mid`
	[ "$tmid" ] && break
	warning "cannot extract time: $logf:$mid; will try the next one"
	trycnt=$(($trycnt-1))
	# shift the whole first-last segment
	first=$(($first-1))
	last=$(($last-1))
	mid=$((($last+$first)/2))
	done
	if [ -z "$tmid" ]; then
	warning "giving up on log..."
	return
	fi
	if [ $tmid -gt $tm ]; then
	last=$(($mid-1))
	elif [ $tmid -lt $tm ]; then
	first=$(($mid+1))
	else
	break
	fi
	done
	echo $mid
	}

	dumplog() {
	local logf=$1
	local from_line=$2
	local to_line=$3
	[ "$from_line" ] \|\|
	return
	tail -n +$from_line $logf \|
	if [ "$to_line" ]; then
	head -$(($to_line-$from_line+1))
	else
	cat
	fi
	}

	#
	# find log/set of logs which are interesting for us
	#
	#
	# find log slices
	#

	find_decompressor() {
	if echo $1 \| grep -qs 'bz2$'; then
	echo "bzip2 -dc"
	elif echo $1 \| grep -qs 'gz$'; then
	echo "gzip -dc"
	elif echo $1 \| grep -qs 'xz$'; then
	echo "xz -dc"
	else
	echo "cat"
	fi
	}
	#
	# check if the log contains a piece of our segment
	#
	is_our_log() {
	local logf=$1
	local from_time=$2
	local to_time=$3

	local cat=`find_decompressor $logf`
	local format=`$cat $logf \| get_time_format`
	local first_time=`$cat $logf \| head -10 \| get_first_time $format`
	local last_time=`$cat $logf \| tail -10 \| get_last_time $format`

	if [ x = "x$first_time" -o x = "x$last_time" ]; then
	warning "Skipping bad logfile '$1': Could not determine log dates"
	return 0 # skip (empty log?)
	fi
	if [ $from_time -gt $last_time ]; then
	# we shouldn't get here anyway if the logs are in order
	return 2 # we're past good logs; exit
	fi
	if [ $from_time -ge $first_time ]; then
	return 3 # this is the last good log
	fi
	# have to go further back
	if [ x = "x$to_time" -o $to_time -ge $first_time ]; then
	return 1 # include this log
	else
	return 0 # don't include this log
	fi
	}
	#
	# go through archived logs (timewise backwards) and see if there
	# are lines belonging to us
	# (we rely on untouched log files, i.e. that modify time
	# hasn't been changed)
	#
	arch_logs() {
	local logf=$1
	local from_time=$2
	local to_time=$3

	# look for files such as: ha-log-20090308 or
	# ha-log-20090308.gz (.bz2) or ha-log.0, etc
	ls -t $logf $logf*[0-9z] 2>/dev/null \|
	while read next_log; do
	is_our_log $next_log $from_time $to_time
	case $? in
	0) ;; # noop, continue
	1) echo $next_log # include log and continue
	debug "Found log $next_log"
	;;
	2) break;; # don't go through older logs!
	3) echo $next_log # include log and continue
	debug "Found log $next_log"
	break
	;; # don't go through older logs!
	esac
	done
	}

	#
	# print part of the log
	#
	drop_tmp_file() {
	[ -z "$tmp" ] \|\| rm -f "$tmp"
	}

	print_logseg() {
	local logf=$1
	local from_time=$2
	local to_time=$3

	# uncompress to a temp file (if necessary)
	local cat=`find_decompressor $logf`
	if [ "$cat" != "cat" ]; then
	tmp=`mktemp`
	$cat $logf > $tmp
	trap drop_tmp_file 0
	sourcef=$tmp
	else
	sourcef=$logf
	tmp=""
	fi

	if [ "$from_time" = 0 ]; then
	FROM_LINE=1
	else
	FROM_LINE=`findln_by_time $sourcef $from_time`
	fi
	if [ -z "$FROM_LINE" ]; then
	warning "couldn't find line for time $from_time; corrupt log file?"
	return
	fi

	TO_LINE=""
	if [ "$to_time" != 0 ]; then
	TO_LINE=`findln_by_time $sourcef $to_time`
	if [ -z "$TO_LINE" ]; then
	warning "couldn't find line for time $to_time; corrupt log file?"
	return
	fi
	if [ $FROM_LINE -lt $TO_LINE ]; then
	dumplog $sourcef $FROM_LINE $TO_LINE
	log "Including segment [$FROM_LINE-$TO_LINE] from $logf"
	else
	debug "Empty segment [$FROM_LINE-$TO_LINE] from $logf"
	fi
	else
	dumplog $sourcef $FROM_LINE $TO_LINE
	log "Including all logs after line $FROM_LINE from $logf"
	fi
	drop_tmp_file
	trap "" 0
	}

	#
	# find log/set of logs which are interesting for us
	#
	dumplogset() {
	local logf=$1
	local from_time=$2
	local to_time=$3

	local logf_set=`arch_logs $logf $from_time $to_time`
	if [ x = "x$logf_set" ]; then
	return
	fi

	local num_logs=`echo "$logf_set" \| wc -l`
	local oldest=`echo $logf_set \| awk '{print $NF}'`
	local newest=`echo $logf_set \| awk '{print $1}'`
	local mid_logfiles=`echo $logf_set \| awk '{for(i=NF-1; i>1; i--) print $i}'`

	# the first logfile: from $from_time to $to_time (or end)
	# logfiles in the middle: all
	# the last logfile: from beginning to $to_time (or end)
	case $num_logs in
	1) print_logseg $newest $from_time $to_time;;
	*)
	print_logseg $oldest $from_time 0
	for f in $mid_logfiles; do
	`find_decompressor $f` $f
	debug "including complete $f logfile"
	done
	print_logseg $newest 0 $to_time
	;;
	esac
	}

	# cut out a stanza
	getstanza() {
	awk -v name="$1" '
	!in_stanza && NF==2 && /^[a-z][a-z][[:space:]]{/ { # stanza start
	if ($1 == name)
	in_stanza = 1
	}
	in_stanza { print }
	in_stanza && NF==1 && $1 == "}" { exit }
	'
	}
	# supply stanza in $1 and variable name in $2
	# (stanza is optional)
	getcfvar() {
	cf_type=$1; shift;
	cf_var=$1; shift;
	cf_file=$*

	[ -f "$cf_file" ] \|\| return
	case $cf_type in
	cman)
	grep $cf_var $cf_file \| sed s/.$cf_var=\"// \| sed s/\".//
	;;
	corosync\|openais)
	sed 's/#.*//' < $cf_file \|
	if [ $# -eq 2 ]; then
	getstanza "$cf_var"
	shift 1
	else
	cat
	fi \|
	awk -v varname="$cf_var" '
	NF==2 && match($1,varname":$")==1 { print $2; exit; }
	'
	;;
	heartbeat)
	sed 's/#.*//' < $cf_file \|
	grep -w "^$cf_var" \|
	sed 's/^[^[:space:]][[:space:]]//'

	;;
	logd)
	sed 's/#.*//' < $cf_file \|
	grep -w "^$cf_var" \|
	sed 's/^[^[:space:]][[:space:]]//'

	;;
	esac
	}

	pickfirst() {
	for x; do
	which $x >/dev/null 2>&1 && {
	echo $x
	return 0
	}
	done
	return 1
	}

	#
	# figure out the cluster type, depending on the process list
	# and existence of configuration files
	#
	get_cluster_type() {
	if
	ps -ef \| egrep -qs '[c]orosync'
	then
	tool=`pickfirst corosync-objctl corosync-cmapctl`
	case $tool in
	objctl) quorum=`$tool -a \| grep quorum.provider \| sed s/.=//`;;
	cmapctl) quorum=`$tool \| grep quorum.provider \| sed s/.=//`;;
	esac
	if [ x"$quorum" = x"quorum_cman" ]; then
	stack="cman"
	else
	stack="corosync"
	fi

	elif
	ps -ef \| egrep -qs '[a]isexec'
	then
	stack="openais"
	elif
	ps -ef \| grep -v -e grep -e "eartbeat/[clasp]" \| egrep -qs '[h]eartbeat'
	then
	stack="heartbeat"

	# Now we're guessing...

	elif [ -f /etc/cluster/cluster.conf ]; then
	stack="cman"

	# TODO: Technically these could be anywhere :-/
	elif [ -f /etc/corosync/corosync.conf ]; then
	stack="corosync"

	elif [ -f /etc/ais/openais.conf ]; then
	stack="openais"

	else
	stack="heartbeat"
	fi

	debug "Detected the '$stack' cluster stack"
	echo $stack
	}

	find_cluster_cf() {
	case $1 in
	cman) echo "/etc/cluster/cluster.conf";;
	corosync)
	best_size=0
	best_file=""

	# TODO: Technically these could be anywhere :-/
	for cf in /etc/ais/openais.conf /etc/corosync/corosync.conf; do
	if [ -f $cf ]; then
	size=`wc -l $cf \| awk '{print $1}'`
	if [ $size -gt $best_size ]; then
	best_size=$size
	best_file=$cf
	fi
	fi
	done
	if [ -z "$best_file" ]; then
	debug "Looking for corosync configuration file. This may take a while..."
	for f in `find / $depth -type f -name corosync.conf`; do
	best_file=$f
	break
	done
	fi
	debug "Located corosync config file: $best_file"
	echo "$best_file"
	;;
	openais)
	# TODO: Technically it could be anywhere :-/
	cf="/etc/ais/openais.conf"
	if [ -f $cf ]; then
	echo "$cf"
	fi
	;;
	heartbeat)
	cf="/etc/ha.d/ha.cf"
	if [ -f $cf ]; then
	echo "$cf"
	fi
	;;
	*)
	warning "Unknown cluster type: $1"
	;;
	esac
	}

	#
	# check for the major prereq for a) parameter parsing and b)
	# parsing logs
	#
	t=`get_time "12:00"`
	if [ "$t" = "" ]; then
	fatal "please install the perl Date::Parse module (perl-DateTime-Format-DateParse on Fedora/Red Hat)"
	fi

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 7:34 PM (4 h, 22 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1665582
Default Alt Text: (393 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions