No OneTemporary
Actions

Size

436 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/fencing/admin.c b/fencing/admin.c
	index c580e60aa1..d78195f09b 100644
	--- a/fencing/admin.c
	+++ b/fencing/admin.c
	@@ -1,379 +1,379 @@
	/*
	* Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/cluster/internal.h>

	#include <crm/stonith-ng.h>
	#include <crm/cib.h>
	#include <crm/pengine/status.h>

	#include <crm/common/xml.h>


	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	{"help", 0, 0, '?', "\tThis text"},
	{"version", 0, 0, '$', "\tVersion information" },
	{"verbose", 0, 0, 'V', "\tIncrease debug output"},
	{"quiet", 0, 0, 'q', "\tPrint only essential output"},

	{"-spacer-", 0, 0, '-', "Commands:"},
	{"list", 1, 0, 'l', "List devices that can terminate the specified host"},
	{"list-registered", 0, 0, 'L', "List all registered devices"},
	{"list-installed", 0, 0, 'I', "List all installed devices"},

	{"-spacer-", 0, 0, '-', ""},
	{"metadata", 0, 0, 'M', "Check the device's metadata"},
	{"query", 1, 0, 'Q', "Check the device's status"},
	{"fence", 1, 0, 'F', "Fence the named host"},
	{"unfence", 1, 0, 'U', "Unfence the named host"},
	{"reboot", 1, 0, 'B', "Reboot the named host"},
	{"confirm", 1, 0, 'C', "Confirm the named host is now safely down"},
	{"history", 1, 0, 'H', "Retrieve last fencing operation"},

	{"-spacer-", 0, 0, '-', ""},
	{"register", 1, 0, 'R', "Register the named stonith device. Requires: --agent, optional: --option"},
	{"deregister", 1, 0, 'D', "De-register the named stonith device"},

	{"register-level", 1, 0, 'r', "Register a stonith level for the named host. Requires: --index, one or more --device entries"},
	{"deregister-level", 1, 0, 'd', "De-register a stonith level for the named host. Requires: --index"},

	{"-spacer-", 0, 0, '-', ""},
	{"-spacer-", 0, 0, '-', "Options and modifiers:"},
	{"agent", 1, 0, 'a', "The agent (eg. fence_xvm) to instantiate when calling with --register"},
	{"env-option", 1, 0, 'e'},
	{"option", 1, 0, 'o'},

	{"device", 1, 0, 'v', "A device to associate with a given host and stonith level"},
	{"index", 1, 0, 'i', "The stonith level (1-9)"},

	{"timeout", 1, 0, 't', "Operation timeout in seconds"},

	{"list-all", 0, 0, 'L', "legacy alias for --list-registered"},

	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	int st_opts = st_opt_sync_call;

	int
	main(int argc, char ** argv)
	{
	int flag;
	int rc = 0;
	int quiet = 0;
	int verbose = 0;
	int argerr = 0;
	int timeout = 120;
	int option_index = 0;
	int fence_level = 0;

	char name[512];
	char value[512];
	const char *agent = NULL;
	const char *device = NULL;
	const char *target = NULL;

	char action = 0;
	stonith_t *st = NULL;
	stonith_key_value_t *params = NULL;
	stonith_key_value_t *devices = NULL;
	stonith_key_value_t *dIter = NULL;

	crm_log_init(NULL, LOG_NOTICE, FALSE, FALSE, argc, argv, FALSE);
	crm_set_options(NULL, "mode [options]", long_options,
	"Provides access to the stonith-ng API.\n"
	"\nAllows the administrator to add/remove/list devices, check device and host status and fence hosts\n");

	while (1) {
	flag = crm_get_option(argc, argv, &option_index);
	if (flag == -1)
	break;

	switch(flag) {
	case 'V':
	verbose = 1;
	crm_bump_log_level();
	break;
	case '$':
	case '?':
	crm_help(flag, EX_OK);
	break;
	case 'L':
	case 'I':
	action = flag;
	break;
	case 'q':
	quiet = 1;
	break;
	case 'Q':
	case 'R':
	case 'D':
	action = flag;
	device = optarg;
	break;
	case 'a':
	agent = optarg;
	break;
	case 'l':
	target = optarg;
	action = 'L';
	break;
	case 'M':
	action = flag;
	break;
	case 't':
	timeout = crm_atoi(optarg, NULL);
	break;
	case 'B':
	case 'F':
	case 'U':
	case 'C':
	case 'H':
	case 'r':
	case 'd':
	target = optarg;
	action = flag;
	break;
	case 'i':
	fence_level = crm_atoi(optarg, NULL);
	break;
	case 'v':
	devices = stonith_key_value_add(devices, NULL, optarg);
	break;
	case 'o':
	crm_info("Scanning: -o %s", optarg);
	rc = sscanf(optarg, "%[^=]=%[^=]", name, value);
	if(rc != 2) {
	crm_err("Invalid option: -o %s", optarg);
	++argerr;
	} else {
	crm_info("Got: '%s'='%s'", name, value);
	params = stonith_key_value_add(params, name, value);
	}
	break;
	case 'e':
	{
	char *key = crm_concat("OCF_RESKEY", optarg, '_');
	const char *env = getenv(key);

	if(env == NULL) {
	crm_err("Invalid option: -e %s", optarg);
	++argerr;
	} else {
	crm_info("Got: '%s'='%s'", optarg, env);
	params = stonith_key_value_add( params, optarg, env);
	}
	}
	break;
	default:
	++argerr;
	break;
	}
	}

	if (optind > argc) {
	++argerr;
	}

	if (argerr) {
	crm_help('?', EX_USAGE);
	}

	crm_debug("Create");
	st = stonith_api_new();

	if(action != 'M' && action != 'I') {
	rc = st->cmds->connect(st, crm_system_name, NULL);
	crm_debug("Connect: %d", rc);

	if(rc < 0) {
	goto done;
	}
	}

	switch(action) {
	case 'I':
	- rc = st->cmds->list(st, st_opt_sync_call, NULL, &devices, timeout);
	+ rc = st->cmds->list_agents(st, st_opt_sync_call, NULL, &devices, timeout);
	for(dIter = devices; dIter; dIter = dIter->next ) {
	fprintf( stdout, " %s\n", dIter->value );
	}
	if(rc == 0) {
	fprintf(stderr, "No devices found\n");

	} else if(rc > 0) {
	fprintf(stderr, "%d devices found\n", rc);
	rc = 0;
	}
	stonith_key_value_freeall(devices, 1, 1);
	break;
	case 'L':
	rc = st->cmds->query(st, st_opts, target, &devices, timeout);
	for(dIter = devices; dIter; dIter = dIter->next ) {
	fprintf( stdout, " %s\n", dIter->value );
	}
	if(rc == 0) {
	fprintf(stderr, "No devices found\n");
	} else if(rc > 0) {
	fprintf(stderr, "%d devices found\n", rc);
	rc = 0;
	}
	stonith_key_value_freeall(devices, 1, 1);
	break;
	case 'Q':
	- rc = st->cmds->call(st, st_opts, device, "monitor", NULL, timeout);
	+ rc = st->cmds->monitor(st, st_opts, device, timeout);
	if(rc < 0) {
	- rc = st->cmds->call(st, st_opts, device, "list", NULL, timeout);
	+ rc = st->cmds->list(st, st_opts, device, NULL, timeout);
	}
	break;
	case 'R':
	rc = st->cmds->register_device(st, st_opts, device, "stonith-ng",
	agent, params);
	break;
	case 'D':
	rc = st->cmds->remove_device(st, st_opts, device);
	break;
	case 'r':
	rc = st->cmds->register_level(st, st_opts, target, fence_level, devices);
	break;
	case 'd':
	rc = st->cmds->remove_level(st, st_opts, target, fence_level);
	break;
	case 'M':
	if (agent == NULL) {
	printf("Please specify an agent to query using -a,--agent [value]\n");
	return -1;
	} else {
	char *buffer = NULL;
	rc = st->cmds->metadata(st, st_opt_sync_call, agent, NULL, &buffer, timeout);
	if(rc == pcmk_ok) {
	printf("%s\n", buffer);
	}
	free(buffer);
	}
	break;
	case 'C':
	rc = st->cmds->confirm(st, st_opts, target);
	break;
	case 'B':
	rc = st->cmds->fence(st, st_opts, target, "reboot", timeout);
	break;
	case 'F':
	rc = st->cmds->fence(st, st_opts, target, "off", timeout);
	break;
	case 'U':
	rc = st->cmds->fence(st, st_opts, target, "on", timeout);
	break;
	case 'H':
	{
	stonith_history_t history, hp, *latest = NULL;
	rc = st->cmds->history(st, st_opts, target, &history, timeout);
	for(hp = history; hp; hp = hp->next) {
	char *action_s = NULL;
	time_t complete = hp->completed;

	if(hp->state == st_done) {
	latest = hp;
	}

	if(quiet \|\| !verbose) {
	continue;
	} else if(hp->action == NULL) {
	action_s = strdup("unknown");
	} else if(hp->action[0] != 'r') {
	action_s = crm_concat("turn", hp->action, ' ');
	} else {
	action_s = strdup(hp->action);
	}

	if(hp->state == st_failed) {
	printf("%s failed to %s node %s on behalf of %s at %s\n",
	hp->delegate?hp->delegate:"We", action_s, hp->target, hp->origin,
	ctime(&complete));

	} else if(hp->state == st_done && hp->delegate) {
	printf("%s was able to %s node %s on behalf of %s at %s\n",
	hp->delegate, action_s, hp->target, hp->origin, ctime(&complete));

	} else if(hp->state == st_done) {
	printf("We were able to %s node %s on behalf of %s at %s\n",
	action_s, hp->target, hp->origin, ctime(&complete));
	} else {
	printf("%s wishes to %s node %s - %d %d\n",
	hp->origin, action_s, hp->target, hp->state, hp->completed);
	}

	free(action_s);
	}

	if(latest) {
	if(quiet) {
	printf("%d\n", latest->completed);
	} else {
	char *action_s = NULL;
	time_t complete = latest->completed;
	if(latest->action == NULL) {
	action_s = strdup("unknown");
	} else if(latest->action[0] != 'r') {
	action_s = crm_concat("turn", latest->action, ' ');
	} else {
	action_s = strdup(latest->action);
	}

	printf("%s was able to %s node %s on behalf of %s at %s\n",
	latest->delegate?latest->delegate:"We", action_s, latest->target,
	latest->origin, ctime(&complete));

	free(action_s);
	}
	}
	break;
	} /* closing bracket for -H case */
	} /* closing bracket for switch case */

	done:
	if(rc < 0) {
	printf("Command failed: %s\n", pcmk_strerror(rc));
	}

	stonith_key_value_freeall(params, 1, 1);
	st->cmds->disconnect(st);
	crm_debug("Disconnect: %d", rc);

	crm_debug("Destroy");
	stonith_api_delete(st);

	return rc;
	}
	diff --git a/fencing/commands.c b/fencing/commands.c
	index 1a7d56e342..bb4e15d0e4 100644
	--- a/fencing/commands.c
	+++ b/fencing/commands.c
	@@ -1,1310 +1,1312 @@
	/*
	* Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <ctype.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/mainloop.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>

	#include <internal.h>

	GHashTable *device_list = NULL;
	GHashTable *topology = NULL;

	static int active_children = 0;

	static gboolean stonith_device_dispatch(gpointer user_data);
	static void st_child_done(GPid pid, gint status, gpointer user_data);

	static void free_async_command(async_command_t *cmd)
	{
	g_list_free(cmd->device_list);
	free(cmd->device);
	free(cmd->action);
	free(cmd->victim);
	free(cmd->remote);
	free(cmd->client);
	free(cmd->origin);
	free(cmd->op);
	free(cmd);
	}

	static async_command_t create_async_command(xmlNode msg)
	{
	async_command_t *cmd = NULL;
	xmlNode *op = get_xpath_object("//@"F_STONITH_ACTION, msg, LOG_ERR);
	const char *action = crm_element_value(op, F_STONITH_ACTION);

	CRM_CHECK(action != NULL, crm_log_xml_warn(msg, "NoAction"); return NULL);

	crm_log_xml_trace(msg, "Command");
	cmd = calloc(1, sizeof(async_command_t));
	crm_element_value_int(msg, F_STONITH_CALLID, &(cmd->id));
	crm_element_value_int(msg, F_STONITH_CALLOPTS, &(cmd->options));
	crm_element_value_int(msg, F_STONITH_TIMEOUT, &(cmd->timeout));

	cmd->origin = crm_element_value_copy(msg, F_ORIG);
	cmd->remote = crm_element_value_copy(msg, F_STONITH_REMOTE);
	cmd->client = crm_element_value_copy(msg, F_STONITH_CLIENTID);
	cmd->op = crm_element_value_copy(msg, F_STONITH_OPERATION);
	cmd->action = strdup(action);
	cmd->victim = crm_element_value_copy(op, F_STONITH_TARGET);
	cmd->mode = crm_element_value_copy(op, F_STONITH_MODE);
	cmd->device = crm_element_value_copy(op, F_STONITH_DEVICE);
	cmd->done = st_child_done;

	CRM_CHECK(cmd->op != NULL, crm_log_xml_warn(msg, "NoOp"); free_async_command(cmd); return NULL);
	CRM_CHECK(cmd->client != NULL, crm_log_xml_warn(msg, "NoClient"));
	return cmd;
	}

	static int stonith_manual_ack(xmlNode msg, remote_fencing_op_t op)
	{
	async_command_t *cmd = create_async_command(msg);
	xmlNode *dev = get_xpath_object("//@"F_STONITH_TARGET, msg, LOG_ERR);

	if(cmd == NULL) {
	return -EINVAL;
	}

	cmd->device = strdup("manual_ack");
	cmd->remote = strdup(op->id);

	crm_notice("Injecting manual confirmation that %s is safely off/down",
	crm_element_value(dev, F_STONITH_TARGET));

	st_child_done(0, 0, cmd);
	return pcmk_ok;
	}

	static gboolean stonith_device_execute(stonith_device_t *device)
	{
	int rc = 0;
	int exec_rc = 0;
	async_command_t *cmd = NULL;
	CRM_CHECK(device != NULL, return FALSE);

	if(device->active_pid) {
	crm_trace("%s is still active with pid %u", device->id, device->active_pid);
	return TRUE;
	}

	if(device->pending_ops) {
	GList *first = device->pending_ops;
	device->pending_ops = g_list_remove_link(device->pending_ops, first);
	cmd = first->data;
	g_list_free_1(first);
	}

	if(cmd == NULL) {
	crm_trace("Nothing further to do for %s", device->id);
	return TRUE;
	}

	cmd->device = strdup(device->id);
	exec_rc = run_stonith_agent(device->agent, cmd->action, cmd->victim,
	device->params, device->aliases, &rc, NULL, cmd);

	if(exec_rc > 0) {
	crm_debug("Operation %s%s%s on %s now running with pid=%d, timeout=%dms",
	cmd->action, cmd->victim?" for node ":"", cmd->victim?cmd->victim:"",
	device->id, exec_rc, cmd->timeout);
	device->active_pid = exec_rc;

	} else {
	crm_warn("Operation %s%s%s on %s failed (%d/%d)",
	cmd->action, cmd->victim?" for node ":"", cmd->victim?cmd->victim:"",
	device->id, exec_rc, rc);
	st_child_done(0, rc<0?rc:exec_rc, cmd);
	}
	return TRUE;
	}

	static gboolean stonith_device_dispatch(gpointer user_data)
	{
	return stonith_device_execute(user_data);
	}

	static void schedule_stonith_command(async_command_t cmd, stonith_device_t device)
	{
	CRM_CHECK(cmd != NULL, return);
	CRM_CHECK(device != NULL, return);

	crm_debug("Scheduling %s on %s for %s (timeout=%dms)", cmd->action, device->id,
	cmd->remote?cmd->remote:cmd->client, cmd->timeout);
	device->pending_ops = g_list_append(device->pending_ops, cmd);
	mainloop_set_trigger(device->work);
	}

	void free_device(gpointer data)
	{
	GListPtr gIter = NULL;
	stonith_device_t *device = data;

	g_hash_table_destroy(device->params);
	g_hash_table_destroy(device->aliases);

	for(gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) {
	async_command_t *cmd = gIter->data;

	crm_warn("Removal of device '%s' purged operation %s", device->id, cmd->action);
	st_child_done(0, -ENODEV, cmd);
	free_async_command(cmd);
	}
	g_list_free(device->pending_ops);

	g_list_free_full(device->targets, free);
	free(device->namespace);
	free(device->agent);
	free(device->id);
	free(device);
	}

	static GHashTable build_port_aliases(const char hostmap, GListPtr *targets)
	{
	char *name = NULL;
	int last = 0, lpc = 0, max = 0, added = 0;
	GHashTable *aliases = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	if(hostmap == NULL) {
	return aliases;
	}

	max = strlen(hostmap);
	for(; lpc <= max; lpc++) {
	switch(hostmap[lpc]) {
	/* Assignment chars */
	case '=':
	case ':':
	if(lpc > last) {
	free(name);
	name = calloc(1, 1 + lpc - last);
	memcpy(name, hostmap + last, lpc - last);
	}
	last = lpc + 1;
	break;

	/* Delimeter chars */
	/* case ',': Potentially used to specify multiple ports */
	case 0:
	case ';':
	case ' ':
	case '\t':
	if(name) {
	char *value = NULL;
	value = calloc(1, 1 + lpc - last);
	memcpy(value, hostmap + last, lpc - last);

	crm_debug("Adding alias '%s'='%s'", name, value);
	g_hash_table_replace(aliases, name, value);
	if(targets) {
	targets = g_list_append(targets, strdup(value));
	}
	value=NULL;
	name=NULL;
	added++;

	} else if(lpc > last) {
	crm_debug("Parse error at offset %d near '%s'", lpc-last, hostmap+last);
	}

	last = lpc + 1;
	break;
	}

	if(hostmap[lpc] == 0) {
	break;
	}
	}

	if(added == 0) {
	crm_info("No host mappings detected in '%s'", hostmap);
	}

	free(name);
	return aliases;
	}

	static void parse_host_line(const char line, GListPtr output)
	{
	int lpc = 0;
	int max = 0;
	int last = 0;

	if(line) {
	max = strlen(line);
	} else {
	return;
	}

	/* Check for any complaints about additional parameters that the device doesn't understand */
	if(strstr(line, "invalid") \|\| strstr(line, "variable")) {
	crm_debug("Skipping: %s", line);
	return;
	}

	crm_trace("Processing: %s", line);
	/* Skip initial whitespace */
	for(lpc = 0; lpc <= max && isspace(line[lpc]); lpc++) {
	last = lpc+1;
	}

	/* Now the actual content */
	for(lpc = 0; lpc <= max; lpc++) {
	gboolean a_space = isspace(line[lpc]);
	if(a_space && lpc < max && isspace(line[lpc+1])) {
	/* fast-forward to the end of the spaces */

	} else if(a_space \|\| line[lpc] == ',' \|\| line[lpc] == 0) {
	int rc = 1;
	char *entry = NULL;

	if(lpc != last) {
	entry = calloc(1, 1 + lpc - last);
	rc = sscanf(line+last, "%[a-zA-Z0-9_-.]", entry);
	}

	if(entry == NULL) {
	/* Skip */
	} else if(rc != 1) {
	crm_warn("Could not parse (%d %d): %s", last, lpc, line+last);
	} else if(safe_str_neq(entry, "on") && safe_str_neq(entry, "off")) {
	crm_trace("Adding '%s'", entry);
	output = g_list_append(output, entry);
	entry = NULL;
	}

	free(entry);
	last = lpc + 1;
	}
	}
	}

	static GListPtr parse_host_list(const char *hosts)
	{
	int lpc = 0;
	int max = 0;
	int last = 0;
	GListPtr output = NULL;

	if(hosts == NULL) {
	return output;
	}

	max = strlen(hosts);
	for(lpc = 0; lpc <= max; lpc++) {
	if(hosts[lpc] == '\n' \|\| hosts[lpc] == 0) {
	char *line = NULL;

	line = calloc(1, 2 + lpc - last);
	snprintf(line, 1 + lpc - last, "%s", hosts+last);
	parse_host_line(line, &output);
	free(line);

	last = lpc + 1;
	}
	}

	return output;
	}

	static stonith_device_t build_device_from_xml(xmlNode msg)
	{
	xmlNode *dev = get_xpath_object("//"F_STONITH_DEVICE, msg, LOG_ERR);
	stonith_device_t *device = NULL;

	device = calloc(1, sizeof(stonith_device_t));
	device->id = crm_element_value_copy(dev, XML_ATTR_ID);
	device->agent = crm_element_value_copy(dev, "agent");
	device->namespace = crm_element_value_copy(dev, "namespace");
	device->params = xml2list(dev);
	device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device);
	/* TODO: Hook up priority */

	return device;
	}

	int stonith_device_register(xmlNode msg, const char *desc)
	{
	const char *value = NULL;
	stonith_device_t *device = build_device_from_xml(msg);

	value = g_hash_table_lookup(device->params, STONITH_ATTR_HOSTLIST);
	if(value) {
	device->targets = parse_host_list(value);
	}

	value = g_hash_table_lookup(device->params, STONITH_ATTR_HOSTMAP);
	device->aliases = build_port_aliases(value, &(device->targets));

	g_hash_table_replace(device_list, device->id, device);

	crm_notice("Added '%s' to the device list (%d active devices)", device->id, g_hash_table_size(device_list));
	if(desc) {
	*desc = device->id;
	}
	return pcmk_ok;
	}

	static int stonith_device_remove(xmlNode msg, const char *desc)
	{
	xmlNode *dev = get_xpath_object("//"F_STONITH_DEVICE, msg, LOG_ERR);
	const char *id = crm_element_value(dev, XML_ATTR_ID);
	if(g_hash_table_remove(device_list, id)) {
	crm_info("Removed '%s' from the device list (%d active devices)",
	id, g_hash_table_size(device_list));
	} else {
	crm_info("Device '%s' not found (%d active devices)",
	id, g_hash_table_size(device_list));
	}
	if(desc) {
	*desc = id;
	}
	return pcmk_ok;
	}

	static int count_active_levels(stonith_topology_t *tp)
	{
	int lpc = 0;
	int count = 0;
	for(lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
	if(tp->levels[lpc] != NULL) {
	count++;
	}
	}
	return count;
	}

	void free_topology_entry(gpointer data)
	{
	stonith_topology_t *tp = data;

	int lpc = 0;
	for(lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
	if(tp->levels[lpc] != NULL) {
	g_list_free_full(tp->levels[lpc], free);
	}
	}
	free(tp->node);
	free(tp);
	}

	int stonith_level_register(xmlNode msg, char *desc)
	{
	int id = 0;
	int rc = pcmk_ok;
	xmlNode *child = NULL;

	xmlNode *level = get_xpath_object("//"F_STONITH_LEVEL, msg, LOG_ERR);
	const char *node = crm_element_value(level, F_STONITH_TARGET);
	stonith_topology_t *tp = g_hash_table_lookup(topology, node);

	crm_element_value_int(level, XML_ATTR_ID, &id);
	if(desc) {
	*desc = g_strdup_printf("%s[%d]", node, id);
	}
	if(id <= 0 \|\| id >= ST_LEVEL_MAX) {
	return -EINVAL;
	}

	if(tp == NULL) {
	tp = calloc(1, sizeof(stonith_topology_t));
	tp->node = strdup(node);
	g_hash_table_replace(topology, tp->node, tp);
	crm_trace("Added %s to the topology (%d active entries)", node, g_hash_table_size(topology));
	}

	if(tp->levels[id] != NULL) {
	crm_info("Adding to the existing %s[%d] topology entry (%d active entries)", node, id, count_active_levels(tp));
	}

	for (child = __xml_first_child(level); child != NULL; child = __xml_next(child)) {
	const char *device = ID(child);
	crm_trace("Adding device '%s' for %s (%d)", device, node, id);
	tp->levels[id] = g_list_append(tp->levels[id], strdup(device));
	}

	crm_info("Node %s has %d active fencing levels", node, count_active_levels(tp));
	return rc;
	}

	int stonith_level_remove(xmlNode msg, char *desc)
	{
	int id = 0;
	xmlNode *level = get_xpath_object("//"F_STONITH_LEVEL, msg, LOG_ERR);
	const char *node = crm_element_value(level, F_STONITH_TARGET);
	stonith_topology_t *tp = g_hash_table_lookup(topology, node);

	if(desc) {
	*desc = g_strdup_printf("%s[%d]", node, id);
	}
	crm_element_value_int(level, XML_ATTR_ID, &id);

	if(tp == NULL) {
	crm_info("Node %s not found (%d active entries)",
	node, g_hash_table_size(topology));
	return pcmk_ok;

	} else if(id < 0 \|\| id >= ST_LEVEL_MAX) {
	return -EINVAL;
	}

	if(id == 0 && g_hash_table_remove(topology, node)) {
	crm_info("Removed all %s related entries from the topology (%d active entries)",
	node, g_hash_table_size(topology));

	} else if(id > 0 && tp->levels[id] != NULL) {
	g_list_free_full(tp->levels[id], free);
	tp->levels[id] = NULL;

	crm_info("Removed entry '%d' from %s's topology (%d active entries remaining)",
	id, node, count_active_levels(tp));
	}
	return pcmk_ok;
	}

	static gboolean string_in_list(GListPtr list, const char *item)
	{
	int lpc = 0;
	int max = g_list_length(list);
	for(lpc = 0; lpc < max; lpc ++) {
	const char *value = g_list_nth_data(list, lpc);
	if(safe_str_eq(item, value)) {
	return TRUE;
	}
	}
	return FALSE;
	}

	static int stonith_device_action(xmlNode msg, char *output)
	{
	int rc = pcmk_ok;
	xmlNode *dev = get_xpath_object("//"F_STONITH_DEVICE, msg, LOG_ERR);
	const char *id = crm_element_value(dev, F_STONITH_DEVICE);

	async_command_t *cmd = NULL;
	stonith_device_t *device = NULL;

	if(id) {
	crm_trace("Looking for '%s'", id);
	device = g_hash_table_lookup(device_list, id);
	}

	if(device) {
	cmd = create_async_command(msg);
	if(cmd == NULL) {
	free_device(device);
	return -EPROTO;
	}

	schedule_stonith_command(cmd, device);
	rc = -EINPROGRESS;

	} else {
	crm_info("Device %s not found", id?id:"<none>");
	rc = -ENODEV;
	}
	return rc;
	}

	static gboolean can_fence_host_with_device(stonith_device_t dev, const char host)
	{
	gboolean can = FALSE;
	const char *alias = host;
	const char *check_type = NULL;

	if(dev == NULL) {
	return FALSE;

	} else if(host == NULL) {
	return TRUE;
	}

	if(g_hash_table_lookup(dev->aliases, host)) {
	alias = g_hash_table_lookup(dev->aliases, host);
	}

	check_type = g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTCHECK);

	if(check_type == NULL) {

	if(g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTLIST)) {
	check_type = "static-list";
	} else if(g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTMAP)) {
	check_type = "static-list";
	} else {
	check_type = "dynamic-list";
	}
	}

	if(safe_str_eq(check_type, "none")) {
	can = TRUE;

	} else if(safe_str_eq(check_type, "static-list")) {

	/* Presence in the hostmap is sufficient
	* Only use if all hosts on which the device can be active can always fence all listed hosts
	*/

	if(string_in_list(dev->targets, host)) {
	can = TRUE;
	} else if(g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTMAP)
	&& g_hash_table_lookup(dev->aliases, host)) {
	can = TRUE;
	}

	} else if(safe_str_eq(check_type, "dynamic-list")) {
	time_t now = time(NULL);

	/* Host/alias must be in the list output to be eligable to be fenced
	*
	* Will cause problems if down'd nodes aren't listed or (for virtual nodes)
	* if the guest is still listed despite being moved to another machine
	*/

	if(dev->targets_age < 0) {
	crm_trace("Port list queries disabled for %s", dev->id);

	} else if(dev->targets == NULL \|\| dev->targets_age + 60 < now) {
	char *output = NULL;
	int rc = pcmk_ok;
	int exec_rc = pcmk_ok;

	/* Check for the target's presence in the output of the 'list' command */
	g_list_free_full(dev->targets, free);
	dev->targets = NULL;

	while(dev->active_pid != 0 && kill(dev->active_pid, 0) == 0) {
	/* This is a hack
	* The proper approach would be to do asynchronous replies
	*/
	crm_trace("Waiting for %u to exit for %s", dev->active_pid, dev->id);
	sleep(1);
	}

	exec_rc = run_stonith_agent(dev->agent, "list", NULL, dev->params, NULL, &rc, &output, NULL);
	if(rc != 0 && dev->active_pid == 0) {
	/* This device probably only supports a single
	* connection, which appears to already be in use,
	* likely involved in a montior or (less likely)
	* metadata operation.
	*
	* Avoid disabling port list queries in the hope that
	* the op would succeed next time
	*/
	crm_info("Couldn't query ports for %s. Call failed with rc=%d and active_pid=%d: %s",
	dev->agent, rc, dev->active_pid, output);

	} else if(exec_rc < 0 \|\| rc != 0) {
	crm_notice("Disabling port list queries for %s (%d/%d): %s",
	dev->id, exec_rc, rc, output);
	dev->targets_age = -1;

	/* Fall back to status */
	g_hash_table_replace(dev->params, strdup(STONITH_ATTR_HOSTCHECK), strdup("status"));

	} else {
	crm_info("Refreshing port list for %s", dev->id);
	dev->targets = parse_host_list(output);
	dev->targets_age = now;
	}

	free(output);
	}

	if(string_in_list(dev->targets, alias)) {
	can = TRUE;
	}

	} else if(safe_str_eq(check_type, "status")) {
	int rc = 0;
	int exec_rc = 0;

	/* Run the status operation for the device/target combination
	* Will cause problems if the device doesn't return 2 for down'd nodes or
	* (for virtual nodes) if the device doesn't return 1 for guests that
	* have been moved to another host
	*/

	exec_rc = run_stonith_agent(
	dev->agent, "status", host, dev->params, dev->aliases, &rc, NULL, NULL);

	if(exec_rc != 0) {
	crm_err("Could not invoke %s: rc=%d", dev->id, exec_rc);

	} else if(rc == 1 /* unkown */) {
	crm_trace("Host %s is not known by %s", host, dev->id);

	} else if(rc == 0 /* active / \|\| rc == 2 / inactive */) {
	can = TRUE;

	} else {
	crm_notice("Unkown result when testing if %s can fence %s: rc=%d", dev->id, host, rc);
	}

	} else {
	crm_err("Unknown check type: %s", check_type);
	}

	if(safe_str_eq(host, alias)) {
	crm_info("%s can%s fence %s: %s", dev->id, can?"":" not", host, check_type);
	} else {
	crm_info("%s can%s fence %s (aka. '%s'): %s", dev->id, can?"":" not", host, alias, check_type);
	}
	return can;
	}


	struct device_search_s
	{
	const char *host;
	GListPtr capable;
	};

	static void search_devices(
	gpointer key, gpointer value, gpointer user_data)
	{
	stonith_device_t *dev = value;
	struct device_search_s *search = user_data;
	if(can_fence_host_with_device(dev, search->host)) {
	search->capable = g_list_append(search->capable, value);
	}
	}

	static int stonith_query(xmlNode msg, xmlNode *list)
	{
	struct device_search_s search;
	int available_devices = 0;
	xmlNode *dev = get_xpath_object("//@"F_STONITH_TARGET, msg, LOG_DEBUG_3);

	search.host = NULL;
	search.capable = NULL;

	if(dev) {
	const char *device = crm_element_value(dev, F_STONITH_DEVICE);
	search.host = crm_element_value(dev, F_STONITH_TARGET);
	if(device && safe_str_eq(device, "manual_ack")) {
	/* No query necessary */
	if(list) {
	*list = NULL;
	}
	return pcmk_ok;
	}
	}

	crm_log_xml_debug(msg, "Query");

	g_hash_table_foreach(device_list, search_devices, &search);
	available_devices = g_list_length(search.capable);
	if(search.host) {
	crm_debug("Found %d matching devices for '%s'",
	available_devices, search.host);
	} else {
	crm_debug("%d devices installed", available_devices);
	}

	/* Pack the results into data */
	if(list) {
	GListPtr lpc = NULL;
	*list = create_xml_node(NULL, __FUNCTION__);
	crm_xml_add(*list, F_STONITH_TARGET, search.host);
	crm_xml_add_int(*list, "st-available-devices", available_devices);
	for(lpc = search.capable; lpc != NULL; lpc = lpc->next) {
	stonith_device_t device = (stonith_device_t)lpc->data;
	dev = create_xml_node(*list, F_STONITH_DEVICE);
	crm_xml_add(dev, XML_ATTR_ID, device->id);
	crm_xml_add(dev, "namespace", device->namespace);
	crm_xml_add(dev, "agent", device->agent);
	if(search.host == NULL) {
	xmlNode *attrs = create_xml_node(dev, XML_TAG_ATTRS);
	g_hash_table_foreach(device->params, hash2field, attrs);
	}
	}
	}

	g_list_free(search.capable);

	return available_devices;
	}

	static void log_operation(async_command_t cmd, int rc, int pid, const char next, const char *output)
	{
	if(rc == 0) {
	next = NULL;
	}

	if(cmd->victim != NULL) {
	do_crm_log(rc==0?LOG_NOTICE:LOG_ERR,
	"Operation '%s' [%d] (call %d from %s) for host '%s' with device '%s' returned: %d%s%s",
	cmd->action, pid, cmd->id, cmd->client, cmd->victim, cmd->device, rc,
	next?". Trying: ":"", next?next:"");
	} else {
	do_crm_log_unlikely(rc==0?LOG_DEBUG:LOG_NOTICE,
	"Operation '%s' [%d] for device '%s' returned: %d%s%s",
	cmd->action, pid, cmd->device, rc, next?". Trying: ":"", next?next:"");
	}

	if(output) {
	/* Logging the whole string confuses syslog when the string is xml */
	char *local_copy = strdup(output);
	int lpc = 0, last = 0, more = strlen(local_copy);
	for(lpc = 0; lpc < more; lpc++) {
	if(local_copy[lpc] == '\n' \|\| local_copy[lpc] == 0) {
	local_copy[lpc] = 0;
	do_crm_log(rc==0?LOG_INFO:LOG_WARNING, "%s: %s",
	cmd->device, local_copy+last);
	last = lpc+1;
	}
	}
	crm_debug("%s: %s (total %d bytes)", cmd->device, local_copy+last, more);
	free(local_copy);
	}
	}

	#define READ_MAX 500
	static void st_child_done(GPid pid, gint status, gpointer user_data)
	{
	int rc = -pcmk_err_generic;

	int len = 0;
	int more = 0;
	gboolean bcast = TRUE;

	char *output = NULL;
	xmlNode *data = NULL;
	xmlNode *reply = NULL;

	stonith_device_t *device = NULL;
	async_command_t *cmd = user_data;

	CRM_CHECK(cmd != NULL, return);

	if(cmd->timer_sigterm > 0) {
	g_source_remove(cmd->timer_sigterm);
	}
	if(cmd->timer_sigkill > 0) {
	g_source_remove(cmd->timer_sigkill);
	}

	if(WIFSIGNALED(status)) {
	int signo = WTERMSIG(status);

	if(signo == SIGTERM \|\| signo == SIGKILL) {
	rc = -ETIME;
	} else {
	rc = -ECONNABORTED;
	}
	crm_notice("Child process %d performing action '%s' with '%s' terminated with signal %d",
	pid, cmd->action, cmd->device, signo);

	} else if(WIFEXITED(status)) {
	rc = WEXITSTATUS(status);
	crm_debug("Child process %d performing action '%s' with '%s' exited with rc %d",
	pid, cmd->action, cmd->device, rc);
	}

	active_children--;

	/* The device is ready to do something else now */
	device = g_hash_table_lookup(device_list, cmd->device);
	if(device) {
	device->active_pid = 0;
	mainloop_set_trigger(device->work);
	}

	do {
	char buffer[READ_MAX];

	errno = 0;
	if(cmd->stdout > 0) {
	memset(&buffer, 0, READ_MAX);
	more = read(cmd->stdout, buffer, READ_MAX-1);
	crm_trace("Got %d more bytes: %s", more, buffer);
	}

	if(more > 0) {
	output = realloc(output, len + more + 1);
	sprintf(output+len, "%s", buffer);
	len += more;
	}

	} while (more == (READ_MAX-1) \|\| (more < 0 && errno == EINTR));

	if(cmd->stdout) {
	close(cmd->stdout);
	cmd->stdout = 0;
	}

	crm_trace("Operation on %s completed with rc=%d (%d remaining)",
	cmd->device, rc, g_list_length(cmd->device_next));

	if(rc != 0 && cmd->device_next) {
	stonith_device_t *dev = cmd->device_next->data;

	log_operation(cmd, rc, pid, dev->id, output);

	cmd->device_next = cmd->device_next->next;
	schedule_stonith_command(cmd, dev);
	goto done;
	}

	if(rc > 0) {
	rc = -pcmk_err_generic;
	}

	reply = stonith_construct_async_reply(cmd, output, data, rc);
	if(safe_str_eq(cmd->action, "metadata")) {
	/* Too verbose to log */
	bcast = FALSE;
	free(output); output = NULL;
	crm_trace("Directed reply: %s op", cmd->action);

	- } else if(crm_str_eq(cmd->action, "monitor", TRUE)) {
	+ } else if(crm_str_eq(cmd->action, "monitor", TRUE) \|\|
	+ crm_str_eq(cmd->action, "list", TRUE) \|\|
	+ crm_str_eq(cmd->action, "status", TRUE)) {
	crm_trace("Directed reply: %s op", cmd->action);
	bcast = FALSE;

	} else if(safe_str_eq(cmd->mode, "slave")) {
	crm_trace("Directed reply: Complex op with %s", cmd->device);
	bcast = FALSE;
	}

	log_operation(cmd, rc, pid, NULL, output);
	crm_log_xml_trace(reply, "Reply");

	if(bcast && !stand_alone) {
	/* Send reply as T_STONITH_NOTIFY so everyone does notifications
	* Potentially limit to unsucessful operations to the originator?
	*/
	crm_trace("Broadcast reply");
	crm_xml_add(reply, F_STONITH_OPERATION, T_STONITH_NOTIFY);
	send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE);

	} else if(cmd->origin) {
	crm_trace("Directed reply to %s", cmd->origin);
	send_cluster_message(cmd->origin, crm_msg_stonith_ng, reply, FALSE);

	} else {
	crm_trace("Directed local %ssync reply to %s", (cmd->options & st_opt_sync_call)?"":"a-", cmd->client);
	do_local_reply(reply, cmd->client, cmd->options & st_opt_sync_call, FALSE);
	}

	if(stand_alone) {
	/* Do notification with a clean data object */
	xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE);
	crm_xml_add_int(notify_data, F_STONITH_RC, rc);
	crm_xml_add(notify_data, F_STONITH_TARGET, cmd->victim);
	crm_xml_add(notify_data, F_STONITH_OPERATION, cmd->op);
	crm_xml_add(notify_data, F_STONITH_DELEGATE, cmd->device);
	crm_xml_add(notify_data, F_STONITH_REMOTE, cmd->remote);
	crm_xml_add(notify_data, F_STONITH_ORIGIN, cmd->client);

	do_stonith_notify(0, T_STONITH_NOTIFY_FENCE, rc, notify_data, NULL);
	}

	free_async_command(cmd);
	done:

	free(output);
	free_xml(reply);
	free_xml(data);
	}

	static gint sort_device_priority(gconstpointer a, gconstpointer b)
	{
	const stonith_device_t *dev_a = a;
	const stonith_device_t *dev_b = a;
	if(dev_a->priority > dev_b->priority) {
	return -1;
	} else if(dev_a->priority < dev_b->priority) {
	return 1;
	}
	return 0;
	}

	static int stonith_fence(xmlNode *msg)
	{
	int options = 0;
	const char *device_id = NULL;
	stonith_device_t *device = NULL;
	async_command_t *cmd = create_async_command(msg);
	xmlNode *dev = get_xpath_object("//@"F_STONITH_TARGET, msg, LOG_ERR);

	if(cmd == NULL) {
	return -EPROTO;
	}

	device_id = crm_element_value(dev, F_STONITH_DEVICE);
	if(device_id) {
	device = g_hash_table_lookup(device_list, device_id);
	if(device == NULL) {
	crm_err("Requested device '%s' is not available", device_id);
	}

	} else {
	struct device_search_s search;

	search.capable = NULL;
	search.host = crm_element_value(dev, F_STONITH_TARGET);

	crm_element_value_int(msg, F_STONITH_CALLOPTS, &options);
	if(options & st_opt_cs_nodeid) {
	int nodeid = crm_atoi(search.host, NULL);
	crm_node_t *node = crm_get_peer(nodeid, NULL);
	if(node) {
	search.host = node->uname;
	}
	}

	g_hash_table_foreach(device_list, search_devices, &search);
	crm_info("Found %d matching devices for '%s'", g_list_length(search.capable), search.host);

	if(g_list_length(search.capable) > 0) {
	/* Order based on priority */
	search.capable = g_list_sort(search.capable, sort_device_priority);

	device = search.capable->data;

	if(g_list_length(search.capable) > 1) {
	cmd->device_list = search.capable;
	cmd->device_next = cmd->device_list->next;
	} else {
	g_list_free(search.capable);
	}
	}
	}

	if(device) {
	cmd->device = device->id;
	schedule_stonith_command(cmd, device);
	return -EINPROGRESS;
	}

	free_async_command(cmd);
	return -EHOSTUNREACH;
	}

	xmlNode stonith_construct_reply(xmlNode request, char output, xmlNode data, int rc)
	{
	int lpc = 0;
	xmlNode *reply = NULL;

	const char *name = NULL;
	const char *value = NULL;
	const char *names[] = {
	F_STONITH_OPERATION,
	F_STONITH_CALLID,
	F_STONITH_CLIENTID,
	F_STONITH_REMOTE,
	F_STONITH_CALLOPTS
	};

	crm_trace("Creating a basic reply");
	reply = create_xml_node(NULL, T_STONITH_REPLY);

	crm_xml_add(reply, "st_origin", __FUNCTION__);
	crm_xml_add(reply, F_TYPE, T_STONITH_NG);
	crm_xml_add(reply, "st_output", output);
	crm_xml_add_int(reply, F_STONITH_RC, rc);

	CRM_CHECK(request != NULL, crm_warn("Can't create a sane reply"); return reply);
	for(lpc = 0; lpc < DIMOF(names); lpc++) {
	name = names[lpc];
	value = crm_element_value(request, name);
	crm_xml_add(reply, name, value);
	}

	if(data != NULL) {
	crm_trace("Attaching reply output");
	add_message_xml(reply, F_STONITH_CALLDATA, data);
	}
	return reply;
	}

	xmlNode stonith_construct_async_reply(async_command_t cmd, char output, xmlNode data, int rc)
	{
	xmlNode *reply = NULL;

	crm_trace("Creating a basic reply");
	reply = create_xml_node(NULL, T_STONITH_REPLY);

	crm_xml_add(reply, "st_origin", __FUNCTION__);
	crm_xml_add(reply, F_TYPE, T_STONITH_NG);

	crm_xml_add(reply, F_STONITH_OPERATION, cmd->op);
	crm_xml_add(reply, F_STONITH_DEVICE, cmd->device);
	crm_xml_add(reply, F_STONITH_REMOTE, cmd->remote);
	crm_xml_add(reply, F_STONITH_CLIENTID, cmd->client);
	crm_xml_add(reply, F_STONITH_TARGET, cmd->victim);
	crm_xml_add(reply, F_STONITH_ACTION, cmd->op);
	crm_xml_add_int(reply, F_STONITH_CALLID, cmd->id);
	crm_xml_add_int(reply, F_STONITH_CALLOPTS, cmd->options);

	crm_xml_add_int(reply, F_STONITH_RC, rc);
	crm_xml_add(reply, "st_output", output);

	if(data != NULL) {
	crm_info("Attaching reply output");
	add_message_xml(reply, F_STONITH_CALLDATA, data);
	}
	return reply;
	}

	void
	stonith_command(stonith_client_t client, xmlNode request, const char *remote)
	{
	int call_options = 0;
	int rc = -EOPNOTSUPP;

	gboolean is_reply = FALSE;
	gboolean always_reply = FALSE;

	xmlNode *reply = NULL;
	xmlNode *data = NULL;

	char *output = NULL;
	const char *op = crm_element_value(request, F_STONITH_OPERATION);
	const char *client_id = crm_element_value(request, F_STONITH_CLIENTID);

	crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
	if(get_xpath_object("//"T_STONITH_REPLY, request, LOG_DEBUG_3)) {
	is_reply = TRUE;
	}

	crm_debug("Processing %s%s from %s (%16x)", op, is_reply?" reply":"",
	client?client->name:remote, call_options);

	if(crm_str_eq(op, CRM_OP_REGISTER, TRUE)) {
	xmlNode *reply = create_xml_node(NULL, "reply");
	crm_xml_add(reply, F_STONITH_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(reply, F_STONITH_CLIENTID, client->id);
	crm_ipcs_send(client->channel, reply, FALSE);
	free_xml(reply);
	return;

	} else if(crm_str_eq(op, STONITH_OP_EXEC, TRUE)) {
	rc = stonith_device_action(request, &output);

	} else if(is_reply && crm_str_eq(op, STONITH_OP_QUERY, TRUE)) {
	process_remote_stonith_query(request);
	return;

	} else if(crm_str_eq(op, STONITH_OP_QUERY, TRUE)) {
	create_remote_stonith_op(client_id, request, TRUE); /* Record it for the future notification */
	rc = stonith_query(request, &data);
	always_reply = TRUE;
	if(!data) {
	return;
	}

	} else if(is_reply && crm_str_eq(op, T_STONITH_NOTIFY, TRUE)) {
	process_remote_stonith_exec(request);
	return;

	} else if(is_reply && crm_str_eq(op, STONITH_OP_FENCE, TRUE)) {
	/* Reply to a complex fencing op */
	process_remote_stonith_exec(request);
	return;

	} else if(crm_str_eq(op, T_STONITH_NOTIFY, TRUE)) {
	const char *flag_name = NULL;

	flag_name = crm_element_value(request, F_STONITH_NOTIFY_ACTIVATE);
	if(flag_name) {
	crm_debug("Setting %s callbacks for %s (%s): ON",
	flag_name, client->name, client->id);
	client->flags \|= get_stonith_flag(flag_name);
	}

	flag_name = crm_element_value(request, F_STONITH_NOTIFY_DEACTIVATE);
	if(flag_name) {
	crm_debug("Setting %s callbacks for %s (%s): off",
	flag_name, client->name, client->id);
	client->flags \|= get_stonith_flag(flag_name);
	}

	crm_ipcs_send_ack(client->channel, "ack", __FUNCTION__, __LINE__);
	return;

	/* } else if(is_reply && crm_str_eq(op, STONITH_OP_FENCE, TRUE)) { */
	/* process_remote_stonith_exec(request); */
	/* return; */

	} else if(is_reply == FALSE && crm_str_eq(op, STONITH_OP_RELAY, TRUE)) {
	if(initiate_remote_stonith_op(NULL, request, FALSE) != NULL) {
	rc = -EINPROGRESS;
	}

	} else if(is_reply == FALSE && crm_str_eq(op, STONITH_OP_FENCE, TRUE)) {

	if(remote \|\| stand_alone) {
	rc = stonith_fence(request);

	} else if(call_options & st_opt_manual_ack) {
	remote_fencing_op_t *rop = initiate_remote_stonith_op(client, request, TRUE);
	rc = stonith_manual_ack(request, rop);

	} else {
	const char *alternate_host = NULL;
	xmlNode *dev = get_xpath_object("//@"F_STONITH_TARGET, request, LOG_TRACE);
	const char *target = crm_element_value_copy(dev, F_STONITH_TARGET);

	if((call_options & st_opt_sync_call) == 0) {
	crm_ipcs_send_ack(client->channel, "ack", __FUNCTION__, __LINE__);
	}

	if(g_hash_table_lookup(topology, target) && safe_str_eq(target, stonith_our_uname)) {
	GHashTableIter gIter;
	crm_node_t *entry = NULL;
	int membership = crm_proc_plugin \| crm_proc_heartbeat \| crm_proc_cpg;

	g_hash_table_iter_init(&gIter, crm_peer_cache);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	crm_trace("Checking for %s.%d != %s",
	entry->uname, entry->id, target);
	if(entry->uname
	&& (entry->processes & membership)
	&& safe_str_neq(entry->uname, target)) {
	alternate_host = entry->uname;
	break;
	}
	}
	if(alternate_host == NULL) {
	crm_err("No alternate host available to handle complex self fencing request");
	}
	}

	if(alternate_host) {
	crm_notice("Forwarding complex self fencing request to peer %s", alternate_host);
	crm_xml_add(request, F_STONITH_OPERATION, STONITH_OP_RELAY);
	crm_xml_add(request, F_STONITH_CLIENTID, client->id);
	send_cluster_message(alternate_host, crm_msg_stonith_ng, request, FALSE);
	rc = -EINPROGRESS;

	} else if(initiate_remote_stonith_op(client, request, FALSE) != NULL) {
	rc = -EINPROGRESS;
	}
	}

	} else if (crm_str_eq(op, STONITH_OP_FENCE_HISTORY, TRUE)) {
	rc = stonith_fence_history(request, &data);
	always_reply = TRUE;

	} else if(crm_str_eq(op, CRM_OP_REGISTER, TRUE)) {
	return;

	} else if(crm_str_eq(op, STONITH_OP_DEVICE_ADD, TRUE)) {
	const char *id = NULL;
	xmlNode *notify_data = create_xml_node(NULL, op);
	rc = stonith_device_register(request, &id);

	crm_xml_add(notify_data, F_STONITH_DEVICE, id);
	crm_xml_add_int(notify_data, F_STONITH_ACTIVE, g_hash_table_size(device_list));

	do_stonith_notify(call_options, op, rc, notify_data, NULL);
	free_xml(notify_data);

	} else if(crm_str_eq(op, STONITH_OP_DEVICE_DEL, TRUE)) {
	const char *id = NULL;
	xmlNode *notify_data = create_xml_node(NULL, op);
	rc = stonith_device_remove(request, &id);

	crm_xml_add(notify_data, F_STONITH_DEVICE, id);
	crm_xml_add_int(notify_data, F_STONITH_ACTIVE, g_hash_table_size(device_list));

	do_stonith_notify(call_options, op, rc, notify_data, NULL);
	free_xml(notify_data);

	} else if(crm_str_eq(op, STONITH_OP_LEVEL_ADD, TRUE)) {
	char *id = NULL;
	xmlNode *notify_data = create_xml_node(NULL, op);
	rc = stonith_level_register(request, &id);

	crm_xml_add(notify_data, F_STONITH_DEVICE, id);
	crm_xml_add_int(notify_data, F_STONITH_ACTIVE, g_hash_table_size(topology));

	do_stonith_notify(call_options, op, rc, notify_data, NULL);
	free_xml(notify_data);

	} else if(crm_str_eq(op, STONITH_OP_LEVEL_DEL, TRUE)) {
	char *id = NULL;
	xmlNode *notify_data = create_xml_node(NULL, op);
	rc = stonith_level_remove(request, &id);

	crm_xml_add(notify_data, F_STONITH_DEVICE, id);
	crm_xml_add_int(notify_data, F_STONITH_ACTIVE, g_hash_table_size(topology));

	do_stonith_notify(call_options, op, rc, notify_data, NULL);
	free_xml(notify_data);

	} else if(crm_str_eq(op, STONITH_OP_CONFIRM, TRUE)) {
	async_command_t *cmd = create_async_command(request);
	xmlNode *reply = stonith_construct_async_reply(cmd, NULL, NULL, 0);

	crm_xml_add(reply, F_STONITH_OPERATION, T_STONITH_NOTIFY);
	crm_notice("Broadcasting manual fencing confirmation for node %s", cmd->victim);
	send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE);

	free_async_command(cmd);
	free_xml(reply);

	} else {
	crm_err("Unknown %s%s from %s", op, is_reply?" reply":"",
	client?client->name:remote);
	crm_log_xml_warn(request, "UnknownOp");
	}

	do_crm_log_unlikely(rc>0?LOG_DEBUG:LOG_INFO,"Processed %s%s from %s: %s (%d)", op, is_reply?" reply":"",
	client?client->name:remote, rc>0?"":pcmk_strerror(rc), rc);

	if(is_reply \|\| rc == -EINPROGRESS) {
	/* Nothing (yet) */

	} else if(remote) {
	reply = stonith_construct_reply(request, output, data, rc);
	send_cluster_message(remote, crm_msg_stonith_ng, reply, FALSE);
	free_xml(reply);

	} else if(rc <= pcmk_ok \|\| always_reply) {
	reply = stonith_construct_reply(request, output, data, rc);
	do_local_reply(reply, client_id, call_options & st_opt_sync_call, remote!=NULL);
	free_xml(reply);
	}

	free(output);
	free_xml(data);
	}
	diff --git a/fencing/fence_false b/fencing/fence_false
	index 6fdc5a2dcd..af9409e893 100644
	--- a/fencing/fence_false
	+++ b/fencing/fence_false
	@@ -1,70 +1,75 @@
	#!/usr/bin/python

	# The Following Agent Has Been Tested On:
	#
	# Virsh 0.3.3 on RHEL 5.2 with xen-3.0.3-51
	#

	import sys, time
	sys.path.append("/usr/share/fence")
	from fencing import *

	#BEGIN_VERSION_GENERATION
	RELEASE_VERSION="3.1.6"
	BUILD_DATE="(built Mon Oct 24 12:14:08 UTC 2011)"
	REDHAT_COPYRIGHT="Copyright (C) Red Hat, Inc. 2004-2010 All rights reserved."
	#END_VERSION_GENERATION
	plug_status="on"

	def get_outlets_status(conn, options):
	- result={}
	+ result={}

	- if (not (options["-n"])):
	- fail_usage("Failed: You have to enter existing machine!")
	- else:
	- port=options["-n"]
	- result[port]=[plug_status, "fake"]
	+ # This fake agent has no port data to list, so we have to make
	+ # something up for the list action.
	+ if options.has_key("-o") and options["-o"] == "list":
	+ result["fake_port_1"]=[plug_status, "fake"]
	+ result["fake_port_2"]=[plug_status, "fake"]
	+ elif (options.has_key("-n") == 0):
	+ fail_usage("Failed: You have to enter existing machine!")
	+ else:
	+ port=options["-n"]
	+ result[port]=[plug_status, "fake"]

	- return result
	+ return result

	def get_power_status(conn, options):
	- outlets=get_outlets_status(conn,options)
	+ outlets=get_outlets_status(conn,options)

	- if (not (options["-n"] in outlets)):
	- fail_usage("Failed: You have to enter existing machine!")
	- else:
	- return outlets[options["-n"]][0]
	+ if len(outlets) == 0 or options.has_key("-n") == 0:
	+ fail_usage("Failed: You have to enter existing machine!")
	+ else:
	+ return outlets[options["-n"]][0]

	def set_power_status(conn, options):
	- global plug_status
	- plug_status = "unknown"
	+ global plug_status
	+ plug_status = "unknown"

	def main():
	- device_opt = [ "help", "version", "agent", "quiet", "verbose", "debug", "action", "port",
	- "no_password", "power_wait", "power_timeout", ]
	+ device_opt = [ "help", "version", "agent", "quiet", "verbose", "debug", "action", "port",
	+ "no_password", "power_wait", "power_timeout", ]

	- atexit.register(atexit_handler)
	+ atexit.register(atexit_handler)

	- pinput = process_input(device_opt)
	+ pinput = process_input(device_opt)

	- # Fake options to keep the library happy
	- #pinput["-p"] = "none"
	- pinput["-a"] = "localhost"
	- pinput["-C"] = ","
	+ # Fake options to keep the library happy
	+ #pinput["-p"] = "none"
	+ pinput["-a"] = "localhost"
	+ pinput["-C"] = ","

	- options = check_input(device_opt, pinput)
	- if options["-o"] == "monitor":
	- sys.exit(0)
	+ options = check_input(device_opt, pinput)
	+ if options.has_key("-o") and (options["-o"] == "monitor"):
	+ sys.exit(0)

	- ## Defaults for fence agent
	- docs = { }
	- docs["shortdesc"] = "Fake failed fence agent"
	- docs["longdesc"] = "fence_false is a fake Fencing agent which always reports failure without doing anything."
	- show_docs(options, docs)
	+ ## Defaults for fence agent
	+ docs = { }
	+ docs["shortdesc"] = "Fake fence agent"
	+ docs["longdesc"] = "fence_true is a fake Fencing agent which always reports success without doing anything."
	+ show_docs(options, docs)

	- ## Operate the fencing device
	- result = fence_action(None, options, set_power_status, get_power_status, get_outlets_status)
	- sys.exit(result)
	+ ## Operate the fencing device
	+ result = fence_action(None, options, set_power_status, get_power_status, get_outlets_status)
	+ sys.exit(result)

	if __name__ == "__main__":
	- main()
	+ main()
	diff --git a/fencing/fence_true b/fencing/fence_true
	index 73b650e14e..afbd6e1581 100644
	--- a/fencing/fence_true
	+++ b/fencing/fence_true
	@@ -1,70 +1,75 @@
	#!/usr/bin/python

	# The Following Agent Has Been Tested On:
	#
	# Virsh 0.3.3 on RHEL 5.2 with xen-3.0.3-51
	#

	import sys, time
	sys.path.append("/usr/share/fence")
	from fencing import *

	#BEGIN_VERSION_GENERATION
	RELEASE_VERSION="3.1.6"
	BUILD_DATE="(built Mon Oct 24 12:14:08 UTC 2011)"
	REDHAT_COPYRIGHT="Copyright (C) Red Hat, Inc. 2004-2010 All rights reserved."
	#END_VERSION_GENERATION
	plug_status="on"

	def get_outlets_status(conn, options):
	- result={}
	+ result={}

	- if (not (options["-n"])):
	- fail_usage("Failed: You have to enter existing machine!")
	- else:
	- port=options["-n"]
	- result[port]=[plug_status, "fake"]
	+ # This fake agent has no port data to list, so we have to make
	+ # something up for the list action.
	+ if options.has_key("-o") and options["-o"] == "list":
	+ result["fake_port_1"]=[plug_status, "fake"]
	+ result["fake_port_2"]=[plug_status, "fake"]
	+ elif (options.has_key("-n") == 0):
	+ fail_usage("Failed: You have to enter existing machine!")
	+ else:
	+ port=options["-n"]
	+ result[port]=[plug_status, "fake"]

	- return result
	+ return result

	def get_power_status(conn, options):
	- outlets=get_outlets_status(conn,options)
	+ outlets=get_outlets_status(conn,options)

	- if (not (options["-n"] in outlets)):
	- fail_usage("Failed: You have to enter existing machine!")
	- else:
	- return outlets[options["-n"]][0]
	+ if len(outlets) == 0 or options.has_key("-n") == 0:
	+ fail_usage("Failed: You have to enter existing machine!")
	+ else:
	+ return outlets[options["-n"]][0]

	def set_power_status(conn, options):
	- global plug_status
	- plug_status = "off"
	+ global plug_status
	+ plug_status = "off"

	def main():
	- device_opt = [ "help", "version", "agent", "quiet", "verbose", "debug", "action", "port",
	- "no_password", "power_wait", "power_timeout", ]
	+ device_opt = [ "help", "version", "agent", "quiet", "verbose", "debug", "action", "port",
	+ "no_password", "power_wait", "power_timeout", ]

	- atexit.register(atexit_handler)
	+ atexit.register(atexit_handler)

	- pinput = process_input(device_opt)
	+ pinput = process_input(device_opt)

	- # Fake options to keep the library happy
	- #pinput["-p"] = "none"
	- pinput["-a"] = "localhost"
	- pinput["-C"] = ","
	+ # Fake options to keep the library happy
	+ #pinput["-p"] = "none"
	+ pinput["-a"] = "localhost"
	+ pinput["-C"] = ","

	- options = check_input(device_opt, pinput)
	- if options["-o"] == "monitor":
	- sys.exit(0)
	+ options = check_input(device_opt, pinput)
	+ if options.has_key("-o") and (options["-o"] == "monitor"):
	+ sys.exit(0)

	- ## Defaults for fence agent
	- docs = { }
	- docs["shortdesc"] = "Fake fence agent"
	- docs["longdesc"] = "fence_true is a fake Fencing agent which always reports success without doing anything."
	- show_docs(options, docs)
	+ ## Defaults for fence agent
	+ docs = { }
	+ docs["shortdesc"] = "Fake fence agent"
	+ docs["longdesc"] = "fence_true is a fake Fencing agent which always reports success without doing anything."
	+ show_docs(options, docs)

	- ## Operate the fencing device
	- result = fence_action(None, options, set_power_status, get_power_status, get_outlets_status)
	- sys.exit(result)
	+ ## Operate the fencing device
	+ result = fence_action(None, options, set_power_status, get_power_status, get_outlets_status)
	+ sys.exit(result)

	if __name__ == "__main__":
	- main()
	+ main()
	diff --git a/fencing/main.c b/fencing/main.c
	index 087bd01160..502bfe1a57 100644
	--- a/fencing/main.c
	+++ b/fencing/main.c
	@@ -1,863 +1,864 @@
	/*
	* Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/cluster/internal.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>

	#include <crm/common/mainloop.h>

	#include <crm/cib/internal.h>

	#include <internal.h>

	#include <standalone_config.h>

	char *stonith_our_uname = NULL;

	GMainLoop *mainloop = NULL;
	GHashTable *client_list = NULL;

	gboolean stand_alone = FALSE;
	gboolean stonith_shutdown_flag = FALSE;

	qb_ipcs_service_t *ipcs = NULL;

	#if SUPPORT_HEARTBEAT
	ll_cluster_t *hb_conn = NULL;
	#endif

	static void stonith_shutdown(int nsig);
	static void stonith_cleanup(void);

	static int32_t
	st_ipc_accept(qb_ipcs_connection_t *c, uid_t uid, gid_t gid)
	{
	crm_trace("Connecting %p for uid=%d gid=%d", c, uid, gid);
	if(stonith_shutdown_flag) {
	- crm_info("Ignoring new client [%d] during shutdown", crm_ipcs_client_pid(c));
	- return -EPERM;
	+ crm_info("Ignoring new client [%d] during shutdown", crm_ipcs_client_pid(c));
	+ return -EPERM;
	}
	return 0;
	}

	static void
	st_ipc_created(qb_ipcs_connection_t *c)
	{
	stonith_client_t *new_client = NULL;

	#if 0
	struct qb_ipcs_stats srv_stats;

	qb_ipcs_stats_get(s1, &srv_stats, QB_FALSE);
	qb_log(LOG_INFO, "Connection created (active:%d, closed:%d)",
	srv_stats.active_connections,
	srv_stats.closed_connections);
	#endif

	new_client = calloc(1, sizeof(stonith_client_t));
	new_client->channel = c;
	new_client->channel_name = strdup("ipc");
	-
	+
	CRM_CHECK(new_client->id == NULL, free(new_client->id));
	new_client->id = crm_generate_uuid();
	crm_trace("Created channel %p for client %s", c, new_client->id);
	-
	+
	/* make sure we can find ourselves later for sync calls
	* redirected to the master instance
	*/
	g_hash_table_insert(client_list, new_client->id, new_client);
	qb_ipcs_context_set(c, new_client);
	CRM_ASSERT(qb_ipcs_context_get(c) != NULL);
	}

	/* Exit code means? */
	static int32_t
	st_ipc_dispatch(qb_ipcs_connection_t c, void data, size_t size)
	{
	xmlNode *request = NULL;
	- stonith_client_t client = (stonith_client_t)qb_ipcs_context_get(c);
	+ stonith_client_t client = (stonith_client_t)qb_ipcs_context_get(c);

	request = crm_ipcs_recv(c, data, size);
	if (request == NULL) {
	return 0;
	}

	CRM_CHECK(client != NULL, goto cleanup);
	-
	+
	if(client->name == NULL) {
	const char *value = crm_element_value(request, F_STONITH_CLIENTNAME);
	if(value == NULL) {
	client->name = crm_itoa(crm_ipcs_client_pid(c));
	} else {
	client->name = strdup(value);
	}
	}

	CRM_CHECK(client->id != NULL, crm_err("Invalid client: %p/%s", client, client->name); goto cleanup);
	-
	+
	crm_xml_add(request, F_STONITH_CLIENTID, client->id);
	crm_xml_add(request, F_STONITH_CLIENTNAME, client->name);
	-
	+
	crm_log_xml_trace(request, "Client[inbound]");
	stonith_command(client, request, NULL);

	cleanup:
	if(client == NULL \|\| client->id == NULL) {
	crm_log_xml_notice(request, "Invalid client");
	}

	free_xml(request);
	return 0;
	}

	/* Error code means? */
	static int32_t
	-st_ipc_closed(qb_ipcs_connection_t *c)
	+st_ipc_closed(qb_ipcs_connection_t *c)
	{
	stonith_client_t client = (stonith_client_t)qb_ipcs_context_get(c);

	#if 0
	qb_ipcs_stats_get(s1, &srv_stats, QB_FALSE);
	qb_ipcs_connection_stats_get(c, &stats, QB_FALSE);
	qb_log(LOG_INFO, "Connection to pid:%d destroyed (active:%d, closed:%d)",
	stats.client_pid,
	srv_stats.active_connections,
	srv_stats.closed_connections);

	qb_log(LOG_DEBUG, " Requests %"PRIu64"", stats.requests);
	qb_log(LOG_DEBUG, " Responses %"PRIu64"", stats.responses);
	qb_log(LOG_DEBUG, " Events %"PRIu64"", stats.events);
	qb_log(LOG_DEBUG, " Send retries %"PRIu64"", stats.send_retries);
	qb_log(LOG_DEBUG, " Recv retries %"PRIu64"", stats.recv_retries);
	qb_log(LOG_DEBUG, " FC state %d", stats.flow_control_state);
	qb_log(LOG_DEBUG, " FC count %"PRIu64"", stats.flow_control_count);
	#endif

	if (client == NULL) {
	- crm_err("No client");
	- return 0;
	+ crm_err("No client");
	+ return 0;
	}
	-
	+
	crm_trace("Cleaning up after client disconnect: %p/%s/%s", client, crm_str(client->name), client->id);
	if(client->id != NULL) {
	g_hash_table_remove(client_list, client->id);
	}

	/* 0 means: yes, go ahead and destroy the connection */
	return 0;
	}

	static void
	-st_ipc_destroy(qb_ipcs_connection_t *c)
	+st_ipc_destroy(qb_ipcs_connection_t *c)
	{
	stonith_client_t client = (stonith_client_t)qb_ipcs_context_get(c);

	/* Make sure the connection is fully cleaned up */
	st_ipc_closed(c);

	if(client == NULL) {
	- crm_trace("Nothing to destroy");
	- return;
	+ crm_trace("Nothing to destroy");
	+ return;
	}

	crm_trace("Destroying %s (%p)", client->name, client);
	-
	+
	free(client->name);
	free(client->id);
	free(client);
	crm_trace("Done");

	return;
	}

	static void
	stonith_peer_callback(xmlNode * msg, void* private_data)
	{
	const char *remote = crm_element_value(msg, F_ORIG);
	crm_log_xml_trace(msg, "Peer[inbound]");
	stonith_command(NULL, msg, remote);
	}

	#if SUPPORT_HEARTBEAT
	static void
	stonith_peer_hb_callback(HA_Message * msg, void* private_data)
	{
	xmlNode *xml = convert_ha_message(NULL, msg, __FUNCTION__);
	stonith_peer_callback(xml, private_data);
	free_xml(xml);
	}
	static void
	stonith_peer_hb_destroy(gpointer user_data)
	{
	if(stonith_shutdown_flag) {
	- crm_info("Heartbeat disconnection complete... exiting");
	+ crm_info("Heartbeat disconnection complete... exiting");
	} else {
	- crm_err("Heartbeat connection lost! Exiting.");
	+ crm_err("Heartbeat connection lost! Exiting.");
	}
	stonith_shutdown(0);
	}
	#endif


	-#if SUPPORT_COROSYNC
	+#if SUPPORT_COROSYNC
	static gboolean stonith_peer_ais_callback(
	- AIS_Message wrapper, char data, int sender)
	+ AIS_Message wrapper, char data, int sender)
	{
	xmlNode *xml = NULL;

	if(wrapper->header.id == crm_class_cluster) {
	- xml = string2xml(data);
	- if(xml == NULL) {
	- goto bail;
	- }
	- crm_xml_add(xml, F_ORIG, wrapper->sender.uname);
	- crm_xml_add_int(xml, F_SEQ, wrapper->id);
	- stonith_peer_callback(xml, NULL);
	+ xml = string2xml(data);
	+ if(xml == NULL) {
	+ goto bail;
	+ }
	+ crm_xml_add(xml, F_ORIG, wrapper->sender.uname);
	+ crm_xml_add_int(xml, F_SEQ, wrapper->id);
	+ stonith_peer_callback(xml, NULL);
	}

	free_xml(xml);
	return TRUE;

	bail:
	crm_err("Invalid XML: '%.120s'", data);
	return TRUE;

	}

	static void
	stonith_peer_ais_destroy(gpointer user_data)
	{
	crm_err("AIS connection terminated");
	stonith_shutdown(0);
	}
	#endif

	void do_local_reply(xmlNode notify_src, const char client_id,
	- gboolean sync_reply, gboolean from_peer)
	+ gboolean sync_reply, gboolean from_peer)
	{
	/* send callback to originating child */
	stonith_client_t *client_obj = NULL;
	int local_rc = pcmk_ok;

	crm_trace("Sending response");

	if(client_id != NULL) {
	- client_obj = g_hash_table_lookup(client_list, client_id);
	+ client_obj = g_hash_table_lookup(client_list, client_id);
	} else {
	- crm_trace("No client to sent the response to."
	- " F_STONITH_CLIENTID not set.");
	+ crm_trace("No client to sent the response to."
	+ " F_STONITH_CLIENTID not set.");
	}
	-
	+
	crm_trace("Sending callback to request originator");
	if(client_obj == NULL) {
	- local_rc = -1;
	-
	+ local_rc = -1;
	+
	} else {
	- crm_trace("Sending %ssync response to %s %s",
	- sync_reply?"":"an a-",
	- client_obj->name,
	- from_peer?"(originator of delegated request)":"");
	-
	- local_rc = crm_ipcs_send(client_obj->channel, notify_src, !sync_reply);
	- }
	-
	+ crm_trace("Sending %ssync response to %s %s",
	+ sync_reply?"":"an a-",
	+ client_obj->name,
	+ from_peer?"(originator of delegated request)":"");
	+
	+ local_rc = crm_ipcs_send(client_obj->channel, notify_src, !sync_reply);
	+ }
	+
	if(local_rc < pcmk_ok && client_obj != NULL) {
	- crm_warn("%sSync reply to %s failed: %s",
	- sync_reply?"":"A-",
	- client_obj?client_obj->name:"<unknown>", pcmk_strerror(local_rc));
	+ crm_warn("%sSync reply to %s failed: %s",
	+ sync_reply?"":"A-",
	+ client_obj?client_obj->name:"<unknown>", pcmk_strerror(local_rc));
	}
	}

	-long long get_stonith_flag(const char *name)
	+long long get_stonith_flag(const char *name)
	{
	if(safe_str_eq(name, T_STONITH_NOTIFY_FENCE)) {
	- return 0x01;
	-
	+ return 0x01;
	+
	} else if(safe_str_eq(name, STONITH_OP_DEVICE_ADD)) {
	- return 0x04;
	+ return 0x04;

	} else if(safe_str_eq(name, STONITH_OP_DEVICE_DEL)) {
	- return 0x10;
	+ return 0x10;
	}
	return 0;
	}

	static void
	stonith_notify_client(gpointer key, gpointer value, gpointer user_data)
	{

	xmlNode *update_msg = user_data;
	stonith_client_t *client = value;
	const char *type = NULL;

	CRM_CHECK(client != NULL, return);
	CRM_CHECK(update_msg != NULL, return);

	type = crm_element_value(update_msg, F_SUBTYPE);
	CRM_CHECK(type != NULL, crm_log_xml_err(update_msg, "notify"); return);

	if(client->channel == NULL) {
	- crm_trace("Skipping client with NULL channel");
	- return;
	+ crm_trace("Skipping client with NULL channel");
	+ return;

	} else if(client->name == NULL) {
	- crm_trace("Skipping unnammed client / comamnd channel");
	- return;
	+ crm_trace("Skipping unnammed client / comamnd channel");
	+ return;
	}

	if(client->flags & get_stonith_flag(type)) {
	- crm_trace("Sending %s-notification to client %s/%s", type, client->name, client->id);
	+ crm_trace("Sending %s-notification to client %s/%s", type, client->name, client->id);
	if(crm_ipcs_send(client->channel, update_msg, ipcs_send_event\|ipcs_send_error) <= 0) {
	- crm_warn("%s-Notification of client %s/%s failed",
	- type, client->name, client->id);
	- }
	+ crm_warn("%s-Notification of client %s/%s failed",
	+ type, client->name, client->id);
	+ }
	}
	}

	void
	do_stonith_notify(
	int options, const char type, int result, xmlNode data,
	const char *remote)
	{
	/* TODO: Standardize the contents of data */
	xmlNode *update_msg = create_xml_node(NULL, "notify");

	CRM_CHECK(type != NULL, ;);
	-
	+
	crm_xml_add(update_msg, F_TYPE, T_STONITH_NOTIFY);
	crm_xml_add(update_msg, F_SUBTYPE, type);
	crm_xml_add(update_msg, F_STONITH_OPERATION, type);
	crm_xml_add_int(update_msg, F_STONITH_RC, result);
	-
	+
	if(data != NULL) {
	- add_message_xml(update_msg, F_STONITH_CALLDATA, data);
	+ add_message_xml(update_msg, F_STONITH_CALLDATA, data);
	}

	crm_trace("Notifying clients");
	g_hash_table_foreach(client_list, stonith_notify_client, update_msg);
	free_xml(update_msg);
	crm_trace("Notify complete");
	}

	-static stonith_key_value_t parse_device_list(const char devices)
	+static stonith_key_value_t parse_device_list(const char devices)
	{
	int lpc = 0;
	int max = 0;
	int last = 0;
	stonith_key_value_t *output = NULL;

	if(devices == NULL) {
	- return output;
	+ return output;
	}

	max = strlen(devices);
	for(lpc = 0; lpc <= max; lpc++) {
	if(devices[lpc] == ',' \|\| devices[lpc] == 0) {
	- char *line = NULL;
	+ char *line = NULL;

	line = calloc(1, 2 + lpc - last);
	snprintf(line, 1 + lpc - last, "%s", devices+last);
	output = stonith_key_value_add(output, NULL, line);
	free(line);

	last = lpc + 1;
	}
	}

	return output;
	}

	-static void topology_remove_helper(const char *node, int level)
	+static void topology_remove_helper(const char *node, int level)
	{
	int rc;
	char *desc = NULL;
	xmlNode *data = create_xml_node(NULL, F_STONITH_LEVEL);
	xmlNode *notify_data = create_xml_node(NULL, STONITH_OP_LEVEL_DEL);

	crm_xml_add(data, "origin", __FUNCTION__);
	crm_xml_add_int(data, XML_ATTR_ID, level);
	crm_xml_add(data, F_STONITH_TARGET, node);

	rc = stonith_level_remove(data, &desc);

	crm_xml_add(notify_data, F_STONITH_DEVICE, desc);
	crm_xml_add_int(notify_data, F_STONITH_ACTIVE, g_hash_table_size(topology));
	-
	+
	do_stonith_notify(0, STONITH_OP_LEVEL_DEL, rc, notify_data, NULL);

	free_xml(notify_data);
	free_xml(data);
	free(desc);
	}

	static void topology_register_helper(const char node, int level, stonith_key_value_t device_list)
	{
	int rc;
	char *desc = NULL;
	xmlNode *notify_data = create_xml_node(NULL, STONITH_OP_LEVEL_ADD);
	xmlNode *data = create_level_registration_xml(node, level, device_list);

	rc = stonith_level_register(data, &desc);

	crm_xml_add(notify_data, F_STONITH_DEVICE, desc);
	crm_xml_add_int(notify_data, F_STONITH_ACTIVE, g_hash_table_size(topology));
	-
	+
	do_stonith_notify(0, STONITH_OP_LEVEL_ADD, rc, notify_data, NULL);

	free_xml(notify_data);
	free_xml(data);
	free(desc);
	}

	static void remove_fencing_topology(xmlXPathObjectPtr xpathObj)
	{
	int max = 0, lpc = 0;

	if(xpathObj && xpathObj->nodesetval) {
	max = xpathObj->nodesetval->nodeNr;
	}

	for(lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);
	CRM_CHECK(match != NULL, continue);

	if(crm_element_value(match, XML_DIFF_MARKER)) {
	/* Deletion */
	int index = 0;
	const char *target = crm_element_value(match, XML_ATTR_STONITH_TARGET);

	crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index);
	if(target == NULL) {
	crm_err("Invalid fencing target in element %s", ID(match));

	} else if(index <= 0) {
	crm_err("Invalid level for %s in element %s", target, ID(match));

	} else {
	topology_remove_helper(target, index);
	}
	/* } else { Deal with modifications during the 'addition' stage */
	}
	}
	}


	static void register_fencing_topology(xmlXPathObjectPtr xpathObj, gboolean force)
	{
	int max = 0, lpc = 0;

	if(xpathObj && xpathObj->nodesetval) {
	max = xpathObj->nodesetval->nodeNr;
	}

	for(lpc = 0; lpc < max; lpc++) {
	int index = 0;
	const char *target;
	const char *dev_list;
	stonith_key_value_t *devices = NULL;
	xmlNode *match = getXpathResult(xpathObj, lpc);
	CRM_CHECK(match != NULL, continue);

	crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index);
	target = crm_element_value(match, XML_ATTR_STONITH_TARGET);
	dev_list = crm_element_value(match, XML_ATTR_STONITH_DEVICES);
	devices = parse_device_list(dev_list);

	crm_trace("Updating %s[%d] (%s) to %s", target, index, ID(match), dev_list);

	if(target == NULL) {
	crm_err("Invalid fencing target in element %s", ID(match));

	} else if(index <= 0) {
	crm_err("Invalid level for %s in element %s", target, ID(match));

	} else if(force == FALSE && crm_element_value(match, XML_DIFF_MARKER)) {
	/* Addition */
	topology_register_helper(target, index, devices);

	} else { /* Modification */
	/* Remove then re-add */
	topology_remove_helper(target, index);
	topology_register_helper(target, index, devices);
	}

	stonith_key_value_freeall(devices, 1, 1);
	}
	}

	/* Fencing
	<diff crm_feature_set="3.0.6">
	<diff-removed>
	<fencing-topology>
	<fencing-level id="f-p1.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="removed:top"/>
	<fencing-level id="f-p1.2" target="pcmk-1" index="2" devices="power" __crm_diff_marker__="removed:top"/>
	<fencing-level devices="disk,network" id="f-p2.1"/>
	</fencing-topology>
	</diff-removed>
	<diff-added>
	<fencing-topology>
	<fencing-level id="f-p.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="added:top"/>
	<fencing-level id="f-p2.1" target="pcmk-2" index="1" devices="disk,something"/>
	<fencing-level id="f-p3.1" target="pcmk-2" index="2" devices="power" __crm_diff_marker__="added:top"/>
	</fencing-topology>
	</diff-added>
	</diff>
	*/

	static void
	fencing_topology_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	xmlXPathObjectPtr xpathObj = NULL;
	const char *xpath = "//" XML_TAG_FENCING_LEVEL;

	crm_trace("Pushing in stonith topology");

	/* Grab everything */
	xpathObj = xpath_search(msg, xpath);

	register_fencing_topology(xpathObj, TRUE);

	if(xpathObj) {
	- xmlXPathFreeObject(xpathObj);
	+ xmlXPathFreeObject(xpathObj);
	}
	}

	static void
	update_fencing_topology(const char event, xmlNode msg)
	{
	const char *xpath;
	xmlXPathObjectPtr xpathObj = NULL;

	/* Process deletions (only) */
	xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_TAG_FENCING_LEVEL;
	xpathObj = xpath_search(msg, xpath);

	remove_fencing_topology(xpathObj);

	if(xpathObj) {
	- xmlXPathFreeObject(xpathObj);
	+ xmlXPathFreeObject(xpathObj);
	}

	/* Process additions and changes */
	xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_TAG_FENCING_LEVEL;
	xpathObj = xpath_search(msg, xpath);

	register_fencing_topology(xpathObj, FALSE);

	if(xpathObj) {
	- xmlXPathFreeObject(xpathObj);
	+ xmlXPathFreeObject(xpathObj);
	}
	}

	static void
	stonith_shutdown(int nsig)
	{
	stonith_shutdown_flag = TRUE;
	crm_info("Terminating with %d clients", g_hash_table_size(client_list));
	if(mainloop != NULL && g_main_is_running(mainloop)) {
	g_main_quit(mainloop);
	} else {
	stonith_cleanup();
	exit(EX_OK);
	}
	}

	cib_t *cib = NULL;

	static void
	stonith_cleanup(void)
	{
	if(cib) {
	cib->cmds->signoff(cib);
	}

	qb_ipcs_destroy(ipcs);
	- crm_peer_destroy();
	+ crm_peer_destroy();
	g_hash_table_destroy(client_list);
	free(stonith_our_uname);
	#if HAVE_LIBXML2
	crm_xml_cleanup();
	#endif
	}

	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	{"stand-alone", 0, 0, 's'},
	{"verbose", 0, 0, 'V'},
	{"version", 0, 0, '$'},
	{"help", 0, 0, '?'},
	-
	+
	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	static void
	setup_cib(void)
	{
	static void *cib_library = NULL;
	static cib_t (cib_new_fn)(void) = NULL;
	static const char (cib_err_fn)(int) = NULL;

	int rc, retries = 0;

	if(cib_library == NULL) {
	cib_library = dlopen(CIB_LIBRARY, RTLD_LAZY);
	}
	if(cib_library && cib_new_fn == NULL) {
	cib_new_fn = dlsym(cib_library, "cib_new");
	}
	if(cib_library && cib_err_fn == NULL) {
	cib_err_fn = dlsym(cib_library, "pcmk_strerror");
	}
	if(cib_new_fn != NULL) {
	cib = (*cib_new_fn)();
	}
	-
	+
	if(cib == NULL) {
	crm_err("No connection to the CIB");
	return;
	}

	do {
	sleep(retries);
	rc = cib->cmds->signon(cib, CRM_SYSTEM_CRMD, cib_command);
	} while(rc == -ENOTCONN && ++retries < 5);
	-
	+
	if (rc != pcmk_ok) {
	crm_err("Could not connect to the CIB service: %s", (*cib_err_fn)(rc));
	-
	+
	} else if (pcmk_ok != cib->cmds->add_notify_callback(
	cib, T_CIB_DIFF_NOTIFY, update_fencing_topology)) {
	crm_err("Could not set CIB notification callback");
	-
	+
	} else {
	rc = cib->cmds->query(cib, NULL, NULL, cib_scope_local);
	add_cib_op_callback(cib, rc, FALSE, NULL, fencing_topology_callback);
	crm_notice("Watching for stonith topology changes");
	- }
	+ }
	}

	-struct qb_ipcs_service_handlers ipc_callbacks =
	+struct qb_ipcs_service_handlers ipc_callbacks =
	{
	.connection_accept = st_ipc_accept,
	.connection_created = st_ipc_created,
	.msg_process = st_ipc_dispatch,
	.connection_closed = st_ipc_closed,
	.connection_destroyed = st_ipc_destroy
	};

	int
	main(int argc, char ** argv)
	{
	int flag;
	int rc = 0;
	int lpc = 0;
	int argerr = 0;
	int option_index = 0;
	const char *actions[] = { "reboot", "poweroff", "list", "monitor", "status" };

	crm_log_init("stonith-ng", LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
	crm_set_options(NULL, "mode [options]", long_options,
	- "Provides a summary of cluster's current state."
	- "\n\nOutputs varying levels of detail in a number of different formats.\n");
	+ "Provides a summary of cluster's current state."
	+ "\n\nOutputs varying levels of detail in a number of different formats.\n");

	while (1) {
	- flag = crm_get_option(argc, argv, &option_index);
	- if (flag == -1)
	- break;
	-
	- switch(flag) {
	- case 'V':
	- crm_bump_log_level();
	- break;
	- case 's':
	- stand_alone = TRUE;
	- break;
	- case '$':
	- case '?':
	- crm_help(flag, EX_OK);
	- break;
	- default:
	- ++argerr;
	- break;
	- }
	+ flag = crm_get_option(argc, argv, &option_index);
	+ if (flag == -1) {
	+ break;
	+ }
	+
	+ switch(flag) {
	+ case 'V':
	+ crm_bump_log_level();
	+ break;
	+ case 's':
	+ stand_alone = TRUE;
	+ break;
	+ case '$':
	+ case '?':
	+ crm_help(flag, EX_OK);
	+ break;
	+ default:
	+ ++argerr;
	+ break;
	+ }
	}

	if(argc - optind == 1 && safe_str_eq("metadata", argv[optind])) {
	- printf("<?xml version=\"1.0\"?><!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n");
	- printf("<resource-agent name=\"stonithd\">\n");
	- printf(" <version>1.0</version>\n");
	- printf(" <longdesc lang=\"en\">This is a fake resource that details the instance attributes handled by stonithd.</longdesc>\n");
	- printf(" <shortdesc lang=\"en\">Options available for all stonith resources</shortdesc>\n");
	- printf(" <parameters>\n");
	-
	- printf(" <parameter name=\"stonith-timeout\" unique=\"0\">\n");
	- printf(" <shortdesc lang=\"en\">How long to wait for the STONITH action to complete.</shortdesc>\n");
	- printf(" <longdesc lang=\"en\">Overrides the stonith-timeout cluster property</longdesc>\n");
	- printf(" <content type=\"time\" default=\"60s\"/>\n");
	- printf(" </parameter>\n");
	-
	- printf(" <parameter name=\"priority\" unique=\"0\">\n");
	- printf(" <shortdesc lang=\"en\">The priority of the stonith resource. The lower the number, the higher the priority.</shortdesc>\n");
	- printf(" <content type=\"integer\" default=\"0\"/>\n");
	- printf(" </parameter>\n");
	-
	- printf(" <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTARG);
	- printf(" <shortdesc lang=\"en\">Advanced use only: An alternate parameter to supply instead of 'port'</shortdesc>\n");
	- printf(" <longdesc lang=\"en\">Some devices do not support the standard 'port' parameter or may provide additional ones.\n"
	- "Use this to specify an alternate, device-specific, parameter that should indicate the machine to be fenced.\n"
	- "A value of 'none' can be used to tell the cluster not to supply any additional parameters.\n"
	- " </longdesc>\n");
	- printf(" <content type=\"string\" default=\"port\"/>\n");
	- printf(" </parameter>\n");
	-
	- printf(" <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTMAP);
	- printf(" <shortdesc lang=\"en\">A mapping of host names to ports numbers for devices that do not support host names.</shortdesc>\n");
	- printf(" <longdesc lang=\"en\">Eg. node1:1;node2:2,3 would tell the cluster to use port 1 for node1 and ports 2 and 3 for node2</longdesc>\n");
	- printf(" <content type=\"string\" default=\"\"/>\n");
	- printf(" </parameter>\n");
	-
	- printf(" <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTLIST);
	- printf(" <shortdesc lang=\"en\">A list of machines controlled by this device (Optional unless %s=static-list).</shortdesc>\n", STONITH_ATTR_HOSTCHECK);
	- printf(" <content type=\"string\" default=\"\"/>\n");
	- printf(" </parameter>\n");
	-
	- printf(" <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTCHECK);
	- printf(" <shortdesc lang=\"en\">How to determin which machines are controlled by the device.</shortdesc>\n");
	- printf(" <longdesc lang=\"en\">Allowed values: dynamic-list (query the device), static-list (check the %s attribute), none (assume every device can fence every machine)</longdesc>\n", STONITH_ATTR_HOSTLIST);
	- printf(" <content type=\"string\" default=\"dynamic-list\"/>\n");
	- printf(" </parameter>\n");
	-
	- for(lpc = 0; lpc < DIMOF(actions); lpc++) {
	- printf(" <parameter name=\"pcmk_%s_action\" unique=\"0\">\n", actions[lpc]);
	- printf(" <shortdesc lang=\"en\">Advanced use only: An alternate command to run instead of '%s'</shortdesc>\n", actions[lpc]);
	- printf(" <longdesc lang=\"en\">Some devices do not support the standard commands or may provide additional ones.\n"
	- "Use this to specify an alternate, device-specific, command that implements the '%s' action.</longdesc>\n", actions[lpc]);
	- printf(" <content type=\"string\" default=\"%s\"/>\n", actions[lpc]);
	- printf(" </parameter>\n");
	- }
	-
	- printf(" </parameters>\n");
	- printf("</resource-agent>\n");
	- return 0;
	+ printf("<?xml version=\"1.0\"?><!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n");
	+ printf("<resource-agent name=\"stonithd\">\n");
	+ printf(" <version>1.0</version>\n");
	+ printf(" <longdesc lang=\"en\">This is a fake resource that details the instance attributes handled by stonithd.</longdesc>\n");
	+ printf(" <shortdesc lang=\"en\">Options available for all stonith resources</shortdesc>\n");
	+ printf(" <parameters>\n");
	+
	+ printf(" <parameter name=\"stonith-timeout\" unique=\"0\">\n");
	+ printf(" <shortdesc lang=\"en\">How long to wait for the STONITH action to complete.</shortdesc>\n");
	+ printf(" <longdesc lang=\"en\">Overrides the stonith-timeout cluster property</longdesc>\n");
	+ printf(" <content type=\"time\" default=\"60s\"/>\n");
	+ printf(" </parameter>\n");
	+
	+ printf(" <parameter name=\"priority\" unique=\"0\">\n");
	+ printf(" <shortdesc lang=\"en\">The priority of the stonith resource. The lower the number, the higher the priority.</shortdesc>\n");
	+ printf(" <content type=\"integer\" default=\"0\"/>\n");
	+ printf(" </parameter>\n");
	+
	+ printf(" <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTARG);
	+ printf(" <shortdesc lang=\"en\">Advanced use only: An alternate parameter to supply instead of 'port'</shortdesc>\n");
	+ printf(" <longdesc lang=\"en\">Some devices do not support the standard 'port' parameter or may provide additional ones.\n"
	+ "Use this to specify an alternate, device-specific, parameter that should indicate the machine to be fenced.\n"
	+ "A value of 'none' can be used to tell the cluster not to supply any additional parameters.\n"
	+ " </longdesc>\n");
	+ printf(" <content type=\"string\" default=\"port\"/>\n");
	+ printf(" </parameter>\n");
	+
	+ printf(" <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTMAP);
	+ printf(" <shortdesc lang=\"en\">A mapping of host names to ports numbers for devices that do not support host names.</shortdesc>\n");
	+ printf(" <longdesc lang=\"en\">Eg. node1:1;node2:2,3 would tell the cluster to use port 1 for node1 and ports 2 and 3 for node2</longdesc>\n");
	+ printf(" <content type=\"string\" default=\"\"/>\n");
	+ printf(" </parameter>\n");
	+
	+ printf(" <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTLIST);
	+ printf(" <shortdesc lang=\"en\">A list of machines controlled by this device (Optional unless %s=static-list).</shortdesc>\n", STONITH_ATTR_HOSTCHECK);
	+ printf(" <content type=\"string\" default=\"\"/>\n");
	+ printf(" </parameter>\n");
	+
	+ printf(" <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTCHECK);
	+ printf(" <shortdesc lang=\"en\">How to determin which machines are controlled by the device.</shortdesc>\n");
	+ printf(" <longdesc lang=\"en\">Allowed values: dynamic-list (query the device), static-list (check the %s attribute), none (assume every device can fence every machine)</longdesc>\n", STONITH_ATTR_HOSTLIST);
	+ printf(" <content type=\"string\" default=\"dynamic-list\"/>\n");
	+ printf(" </parameter>\n");
	+
	+ for(lpc = 0; lpc < DIMOF(actions); lpc++) {
	+ printf(" <parameter name=\"pcmk_%s_action\" unique=\"0\">\n", actions[lpc]);
	+ printf(" <shortdesc lang=\"en\">Advanced use only: An alternate command to run instead of '%s'</shortdesc>\n", actions[lpc]);
	+ printf(" <longdesc lang=\"en\">Some devices do not support the standard commands or may provide additional ones.\n"
	+ "Use this to specify an alternate, device-specific, command that implements the '%s' action.</longdesc>\n", actions[lpc]);
	+ printf(" <content type=\"string\" default=\"%s\"/>\n", actions[lpc]);
	+ printf(" </parameter>\n");
	+ }
	+
	+ printf(" </parameters>\n");
	+ printf("</resource-agent>\n");
	+ return 0;
	}

	if (optind != argc) {
	- ++argerr;
	+ ++argerr;
	}
	-
	+
	if (argerr) {
	- crm_help('?', EX_USAGE);
	+ crm_help('?', EX_USAGE);
	}

	mainloop_add_signal(SIGTERM, stonith_shutdown);

	crm_peer_init();
	client_list = g_hash_table_new(crm_str_hash, g_str_equal);
	-
	+
	if(stand_alone == FALSE) {
	- void *dispatch = NULL;
	- void *destroy = NULL;
	+ void *dispatch = NULL;
	+ void *destroy = NULL;

	#if SUPPORT_HEARTBEAT
	- dispatch = stonith_peer_hb_callback;
	- destroy = stonith_peer_hb_destroy;
	+ dispatch = stonith_peer_hb_callback;
	+ destroy = stonith_peer_hb_destroy;
	#endif

	- if(is_openais_cluster()) {
	+ if(is_openais_cluster()) {
	#if SUPPORT_COROSYNC
	- destroy = stonith_peer_ais_destroy;
	- dispatch = stonith_peer_ais_callback;
	+ destroy = stonith_peer_ais_destroy;
	+ dispatch = stonith_peer_ais_callback;
	#endif
	- }
	-
	- if(crm_cluster_connect(&stonith_our_uname, NULL, dispatch, destroy,
	+ }
	+
	+ if(crm_cluster_connect(&stonith_our_uname, NULL, dispatch, destroy,
	#if SUPPORT_HEARTBEAT
	- &hb_conn
	+ &hb_conn
	#else
	- NULL
	+ NULL
	#endif
	- ) == FALSE){
	- crm_crit("Cannot sign in to the cluster... terminating");
	- exit(100);
	- }
	+ ) == FALSE) {
	+ crm_crit("Cannot sign in to the cluster... terminating");
	+ exit(100);
	+ }

	setup_cib();

	} else {
	- stonith_our_uname = strdup("localhost");
	+ stonith_our_uname = strdup("localhost");
	}

	device_list = g_hash_table_new_full(
	crm_str_hash, g_str_equal, NULL, free_device);

	topology = g_hash_table_new_full(
	crm_str_hash, g_str_equal, NULL, free_topology_entry);

	ipcs = mainloop_add_ipc_server("stonith-ng", QB_IPC_NATIVE, &ipc_callbacks);

	#if SUPPORT_STONITH_CONFIG
	if (((stand_alone == TRUE)) && !(standalone_cfg_read_file(STONITH_NG_CONF_FILE))) {
	standalone_cfg_commit();
	}
	#endif

	if(ipcs != NULL) {
	- /* Create the mainloop and run it... */
	- mainloop = g_main_new(FALSE);
	- crm_info("Starting %s mainloop", crm_system_name);
	+ /* Create the mainloop and run it... */
	+ mainloop = g_main_new(FALSE);
	+ crm_info("Starting %s mainloop", crm_system_name);

	- g_main_run(mainloop);
	+ g_main_run(mainloop);

	} else {
	- crm_err("Couldnt start all communication channels, exiting.");
	+ crm_err("Couldnt start all communication channels, exiting.");
	}
	-
	+
	stonith_cleanup();

	#if SUPPORT_HEARTBEAT
	if(hb_conn) {
	- hb_conn->llc_ops->delete(hb_conn);
	+ hb_conn->llc_ops->delete(hb_conn);
	}
	#endif
	-
	+
	crm_info("Done");
	qb_log_fini();

	return rc;
	}

	diff --git a/fencing/test.c b/fencing/test.c
	index c82fed8bcb..cb26bd56da 100644
	--- a/fencing/test.c
	+++ b/fencing/test.c
	@@ -1,207 +1,209 @@
	/*
	* Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/cluster/internal.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>


	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	{"verbose", 0, 0, 'V'},
	{"version", 0, 0, '$'},
	{"help", 0, 0, '?'},
	{"passive", 0, 0, 'p'},
	-
	+
	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	int st_opts = st_opt_sync_call;

	static void st_callback(stonith_t st, stonith_event_t e)
	{
	if(st->state == stonith_disconnected) {
	exit(1);
	}

	crm_notice("Operation %s requested by %s %s for peer %s. %s reported: %s (ref=%s)",
	e->operation, e->origin, e->result == pcmk_ok?"completed":"failed",
	e->target, e->executioner ? e->executioner : "<none>",
	- pcmk_strerror(e->result), e->id);
	+ pcmk_strerror(e->result), e->id);
	}

	static void
	st_global_callback(stonith_t * stonith, const xmlNode * msg, int call_id, int rc,
	xmlNode * output, void *userdata)
	{
	crm_log_xml_notice((xmlNode*)msg, "Event");
	}


	int
	main(int argc, char ** argv)
	{
	int argerr = 0;
	int flag;
	int option_index = 0;
	int rc = 0;

	+ char *tmp = NULL;
	struct pollfd pollfd;
	stonith_t *st = NULL;

	stonith_key_value_t *params = NULL;
	gboolean passive_mode = FALSE;
	-
	+
	crm_log_cli_init("stonith-test");
	crm_set_options(NULL, "mode [options]", long_options,
	- "Provides a summary of cluster's current state."
	- "\n\nOutputs varying levels of detail in a number of different formats.\n");
	+ "Provides a summary of cluster's current state."
	+ "\n\nOutputs varying levels of detail in a number of different formats.\n");

	params = stonith_key_value_add(params, "pcmk_host_map", "some-host=pcmk-7 pcmk-3=3,4");

	while (1) {
	- flag = crm_get_option(argc, argv, &option_index);
	- if (flag == -1)
	- break;
	-
	- switch(flag) {
	- case 'V':
	- crm_bump_log_level();
	- break;
	- case '$':
	- case '?':
	- crm_help(flag, EX_OK);
	- break;
	- case 'p':
	- passive_mode = TRUE;
	- break;
	- default:
	- ++argerr;
	- break;
	- }
	+ flag = crm_get_option(argc, argv, &option_index);
	+ if (flag == -1) {
	+ break;
	+ }
	+
	+ switch(flag) {
	+ case 'V':
	+ crm_bump_log_level();
	+ break;
	+ case '$':
	+ case '?':
	+ crm_help(flag, EX_OK);
	+ break;
	+ case 'p':
	+ passive_mode = TRUE;
	+ break;
	+ default:
	+ ++argerr;
	+ break;
	+ }
	}

	if (optind > argc) {
	- ++argerr;
	+ ++argerr;
	}
	-
	+
	if (argerr) {
	- crm_help('?', EX_USAGE);
	+ crm_help('?', EX_USAGE);
	}

	crm_debug("Create");
	st = stonith_api_new();

	rc = st->cmds->connect(st, crm_system_name, &pollfd.fd);
	crm_debug("Connect: %d", rc);

	rc = st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT, st_callback);
	-
	+
	if(passive_mode) {
	- rc = st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, st_callback);
	- rc = st->cmds->register_notification(st, STONITH_OP_DEVICE_ADD, st_callback);
	- rc = st->cmds->register_notification(st, STONITH_OP_DEVICE_DEL, st_callback);
	+ rc = st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, st_callback);
	+ rc = st->cmds->register_notification(st, STONITH_OP_DEVICE_ADD, st_callback);
	+ rc = st->cmds->register_notification(st, STONITH_OP_DEVICE_DEL, st_callback);

	st->cmds->register_callback(st, 0, 120, FALSE, NULL, "st_global_callback", st_global_callback);

	- crm_info("Looking for notification");
	+ crm_info("Looking for notification");
	pollfd.events = POLLIN;
	while(true) {
	- rc = poll( &pollfd, 1, 600 * 1000 ); /* wait 10 minutes, -1 forever */
	+ rc = poll( &pollfd, 1, 600 * 1000 ); /* wait 10 minutes, -1 forever */
	if (rc > 0 )
	- stonith_dispatch( st );
	- else
	- break;
	- }
	+ stonith_dispatch( st );
	+ else
	+ break;
	+ }

	} else {
	- rc = st->cmds->register_device(st, st_opts, "test-id", "stonith-ng", "fence_xvm", params);
	- crm_debug("Register: %d", rc);
	-
	- rc = st->cmds->call(st, st_opts, "test-id", "list", NULL, 10);
	- crm_debug("List: %d", rc);
	-
	- rc = st->cmds->call(st, st_opts, "test-id", "monitor", NULL, 10);
	- crm_debug("Monitor: %d", rc);
	-
	- rc = st->cmds->call(st, st_opts, "test-id", "status", "pcmk-2", 10);
	- crm_debug("Status pcmk-2: %d", rc);
	-
	- rc = st->cmds->call(st, st_opts, "test-id", "status", "pcmk-1", 10);
	- crm_debug("Status pcmk-1: %d", rc);
	-
	- rc = st->cmds->fence(st, st_opts, "unknown-host", "off", 60);
	- crm_debug("Fence unknown-host: %d", rc);
	-
	- rc = st->cmds->call(st, st_opts, "test-id", "status", "pcmk-1", 10);
	- crm_debug("Status pcmk-1: %d", rc);
	-
	- rc = st->cmds->fence(st, st_opts, "pcmk-1", "off", 60);
	- crm_debug("Fence pcmk-1: %d", rc);
	-
	- rc = st->cmds->call(st, st_opts, "test-id", "status", "pcmk-1", 10);
	- crm_debug("Status pcmk-1: %d", rc);
	-
	- rc = st->cmds->fence(st, st_opts, "pcmk-1", "on", 10);
	- crm_debug("Unfence pcmk-1: %d", rc);
	-
	- rc = st->cmds->call(st, st_opts, "test-id", "status", "pcmk-1", 10);
	- crm_debug("Status pcmk-1: %d", rc);
	-
	- rc = st->cmds->fence(st, st_opts, "some-host", "off", 10);
	- crm_debug("Fence alias: %d", rc);
	-
	- rc = st->cmds->call(st, st_opts, "test-id", "status", "some-host", 10);
	- crm_debug("Status alias: %d", rc);
	-
	- rc = st->cmds->fence(st, st_opts, "pcmk-1", "on", 10);
	- crm_debug("Unfence pcmk-1: %d", rc);
	-
	- rc = st->cmds->remove_device(st, st_opts, "test-id");
	- crm_debug("Remove test-id: %d", rc);
	- }
	-
	+ rc = st->cmds->register_device(st, st_opts, "test-id", "stonith-ng", "fence_xvm", params);
	+ crm_debug("Register: %d", rc);
	+
	+ rc = st->cmds->list(st, st_opts, "test-id", &tmp, 10);
	+ crm_debug("List: %d output: %s\n", rc, tmp ? tmp : "<none>");
	+
	+ rc = st->cmds->monitor(st, st_opts, "test-id", 10);
	+ crm_debug("Monitor: %d", rc);
	+
	+ rc = st->cmds->status(st, st_opts, "test-id", "pcmk-2", 10);
	+ crm_debug("Status pcmk-2: %d", rc);
	+
	+ rc = st->cmds->status(st, st_opts, "test-id", "pcmk-1", 10);
	+ crm_debug("Status pcmk-1: %d", rc);
	+
	+ rc = st->cmds->fence(st, st_opts, "unknown-host", "off", 60);
	+ crm_debug("Fence unknown-host: %d", rc);
	+
	+ rc = st->cmds->status(st, st_opts, "test-id", "pcmk-1", 10);
	+ crm_debug("Status pcmk-1: %d", rc);
	+
	+ rc = st->cmds->fence(st, st_opts, "pcmk-1", "off", 60);
	+ crm_debug("Fence pcmk-1: %d", rc);
	+
	+ rc = st->cmds->status(st, st_opts, "test-id", "pcmk-1", 10);
	+ crm_debug("Status pcmk-1: %d", rc);
	+
	+ rc = st->cmds->fence(st, st_opts, "pcmk-1", "on", 10);
	+ crm_debug("Unfence pcmk-1: %d", rc);
	+
	+ rc = st->cmds->status(st, st_opts, "test-id", "pcmk-1", 10);
	+ crm_debug("Status pcmk-1: %d", rc);
	+
	+ rc = st->cmds->fence(st, st_opts, "some-host", "off", 10);
	+ crm_debug("Fence alias: %d", rc);
	+
	+ rc = st->cmds->status(st, st_opts, "test-id", "some-host", 10);
	+ crm_debug("Status alias: %d", rc);
	+
	+ rc = st->cmds->fence(st, st_opts, "pcmk-1", "on", 10);
	+ crm_debug("Unfence pcmk-1: %d", rc);
	+
	+ rc = st->cmds->remove_device(st, st_opts, "test-id");
	+ crm_debug("Remove test-id: %d", rc);
	+ }
	+
	stonith_key_value_freeall(params, 1, 1);

	rc = st->cmds->disconnect(st);
	crm_debug("Disconnect: %d", rc);

	crm_debug("Destroy");
	stonith_api_delete(st);
	-
	+
	return rc;
	}
	diff --git a/include/crm/fencing/internal.h b/include/crm/fencing/internal.h
	index 795c1a7a96..677f851517 100644
	--- a/include/crm/fencing/internal.h
	+++ b/include/crm/fencing/internal.h
	@@ -1,119 +1,119 @@
	/*
	* Copyright (C) 2011 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/
	#ifndef STONITH_NG_INTERNAL__H
	# define STONITH_NG_INTERNAL__H

	# include <crm/common/ipc.h>
	# include <crm/common/xml.h>

	typedef struct async_command_s {

	int id;
	int pid;
	int stdout;
	int options;
	int timeout;

	char *op;
	char *origin;
	char *client;
	char *remote;

	char *victim;
	char *action;
	char *device;
	char *mode;

	GListPtr device_list;
	GListPtr device_next;

	void (*done)(GPid pid, gint status, gpointer user_data);
	guint timer_sigterm;
	guint timer_sigkill;

	} async_command_t;

	int run_stonith_agent(const char agent, const char action, const char *victim,
	GHashTable * dev_hash, GHashTable * port_map, int *agent_result,
	char *output, async_command_t track);

	gboolean is_redhat_agent(const char *agent);

	xmlNode create_level_registration_xml(const char node, int level,
	stonith_key_value_t * device_list);

	xmlNode create_device_registration_xml(const char id, const char namespace, const char agent,
	stonith_key_value_t * params);

	#define ST_LEVEL_MAX 10

	-#define F_STONITH_CLIENTID "st_clientid"
	-#define F_STONITH_CALLOPTS "st_callopt"
	-#define F_STONITH_CALLID "st_callid"
	-#define F_STONITH_CALLDATA "st_calldata"
	-#define F_STONITH_OPERATION "st_op"
	-#define F_STONITH_TARGET "st_target"
	-#define F_STONITH_REMOTE "st_remote_op"
	-#define F_STONITH_RC "st_rc"
	-#define F_STONITH_TIMEOUT "st_timeout"
	-#define F_STONITH_CALLBACK_TOKEN "st_async_id"
	-#define F_STONITH_CLIENTNAME "st_clientname"
	-#define F_STONITH_NOTIFY_TYPE "st_notify_type"
	-#define F_STONITH_NOTIFY_ACTIVATE "st_notify_activate"
	-#define F_STONITH_NOTIFY_DEACTIVATE "st_notify_deactivate"
	-#define F_STONITH_DELEGATE "st_delegate"
	-#define F_STONITH_ORIGIN "st_origin"
	-#define F_STONITH_HISTORY_LIST "st_history"
	-#define F_STONITH_DATE "st_date"
	-#define F_STONITH_STATE "st_state"
	-#define F_STONITH_LEVEL "st_level"
	-#define F_STONITH_OWNER "st_owner"
	-#define F_STONITH_ACTIVE "st_active"
	-
	-#define F_STONITH_DEVICE "st_device_id"
	-#define F_STONITH_ACTION "st_device_action"
	-#define F_STONITH_MODE "st_mode"
	-
	-#define T_STONITH_NG "stonith-ng"
	-#define T_STONITH_REPLY "st-reply"
	-#define T_STONITH_NOTIFY "st_notify"
	-
	-#define STONITH_ATTR_ARGMAP "pcmk_arg_map"
	-#define STONITH_ATTR_HOSTARG "pcmk_host_argument"
	-#define STONITH_ATTR_HOSTMAP "pcmk_host_map"
	-#define STONITH_ATTR_HOSTLIST "pcmk_host_list"
	-#define STONITH_ATTR_HOSTCHECK "pcmk_host_check"
	-
	-#define STONITH_ATTR_ACTION_OP "option" /* To be replaced by 'action' at some point */
	-
	-#define STONITH_OP_EXEC "st_execute"
	-#define STONITH_OP_QUERY "st_query"
	-#define STONITH_OP_FENCE "st_fence"
	-#define STONITH_OP_RELAY "st_relay"
	-#define STONITH_OP_CONFIRM "st_confirm"
	-#define STONITH_OP_DEVICE_ADD "st_device_register"
	-#define STONITH_OP_DEVICE_DEL "st_device_remove"
	+#define F_STONITH_CLIENTID "st_clientid"
	+#define F_STONITH_CALLOPTS "st_callopt"
	+#define F_STONITH_CALLID "st_callid"
	+#define F_STONITH_CALLDATA "st_calldata"
	+#define F_STONITH_OPERATION "st_op"
	+#define F_STONITH_TARGET "st_target"
	+#define F_STONITH_REMOTE "st_remote_op"
	+#define F_STONITH_RC "st_rc"
	+#define F_STONITH_TIMEOUT "st_timeout"
	+#define F_STONITH_CALLBACK_TOKEN "st_async_id"
	+#define F_STONITH_CLIENTNAME "st_clientname"
	+#define F_STONITH_NOTIFY_TYPE "st_notify_type"
	+#define F_STONITH_NOTIFY_ACTIVATE "st_notify_activate"
	+#define F_STONITH_NOTIFY_DEACTIVATE "st_notify_deactivate"
	+#define F_STONITH_DELEGATE "st_delegate"
	+#define F_STONITH_ORIGIN "st_origin"
	+#define F_STONITH_HISTORY_LIST "st_history"
	+#define F_STONITH_DATE "st_date"
	+#define F_STONITH_STATE "st_state"
	+#define F_STONITH_LEVEL "st_level"
	+#define F_STONITH_OWNER "st_owner"
	+#define F_STONITH_ACTIVE "st_active"
	+
	+#define F_STONITH_DEVICE "st_device_id"
	+#define F_STONITH_ACTION "st_device_action"
	+#define F_STONITH_MODE "st_mode"
	+
	+#define T_STONITH_NG "stonith-ng"
	+#define T_STONITH_REPLY "st-reply"
	+#define T_STONITH_NOTIFY "st_notify"
	+
	+#define STONITH_ATTR_ARGMAP "pcmk_arg_map"
	+#define STONITH_ATTR_HOSTARG "pcmk_host_argument"
	+#define STONITH_ATTR_HOSTMAP "pcmk_host_map"
	+#define STONITH_ATTR_HOSTLIST "pcmk_host_list"
	+#define STONITH_ATTR_HOSTCHECK "pcmk_host_check"
	+
	+#define STONITH_ATTR_ACTION_OP "option" /* To be replaced by 'action' at some point */
	+
	+#define STONITH_OP_EXEC "st_execute"
	+#define STONITH_OP_QUERY "st_query"
	+#define STONITH_OP_FENCE "st_fence"
	+#define STONITH_OP_RELAY "st_relay"
	+#define STONITH_OP_CONFIRM "st_confirm"
	+#define STONITH_OP_DEVICE_ADD "st_device_register"
	+#define STONITH_OP_DEVICE_DEL "st_device_remove"
	#define STONITH_OP_DEVICE_METADATA "st_device_metadata"
	#define STONITH_OP_FENCE_HISTORY "st_fence_history"
	-#define STONITH_OP_LEVEL_ADD "st_level_add"
	-#define STONITH_OP_LEVEL_DEL "st_level_remove"
	+#define STONITH_OP_LEVEL_ADD "st_level_add"
	+#define STONITH_OP_LEVEL_DEL "st_level_remove"

	-#define stonith_channel "st_command"
	+#define stonith_channel "st_command"
	#define stonith_channel_callback "st_callback"

	#endif
	diff --git a/include/crm/stonith-ng.h b/include/crm/stonith-ng.h
	index e0fefc2f6c..d1e0d6b992 100644
	--- a/include/crm/stonith-ng.h
	+++ b/include/crm/stonith-ng.h
	@@ -1,256 +1,381 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/
	#ifndef STONITH_NG__H
	# define STONITH_NG__H

	# include <dlfcn.h>
	# include <stdbool.h>

	/* TO-DO: Work out how to drop this requirement */
	# include <libxml/tree.h>

	#define T_STONITH_NOTIFY_DISCONNECT "st_notify_disconnect"
	#define T_STONITH_NOTIFY_FENCE "st_notify_fence"

	/* INDENT-OFF */
	enum stonith_state {
	stonith_connected_command,
	stonith_connected_query,
	stonith_disconnected,
	};

	enum stonith_call_options {
	st_opt_none = 0x00000000,
	st_opt_verbose = 0x00000001,
	st_opt_allow_suicide = 0x00000002,

	- st_opt_manual_ack = 0x00000008,
	+ st_opt_manual_ack = 0x00000008,
	st_opt_discard_reply = 0x00000010,
	- st_opt_all_replies = 0x00000020,
	- st_opt_topology = 0x00000040,
	+ st_opt_all_replies = 0x00000020,
	+ st_opt_topology = 0x00000040,
	st_opt_scope_local = 0x00000100,
	st_opt_cs_nodeid = 0x00000200,
	st_opt_sync_call = 0x00001000,
	};

	#define stonith_default_options = stonith_none

	-
	-enum op_state
	+enum op_state
	{
	st_query,
	st_exec,
	st_done,
	st_failed,
	};

	typedef struct stonith_key_value_s {
	- char *key;
	- char *value;
	+ char *key;
	+ char *value;
	struct stonith_key_value_s *next;
	} stonith_key_value_t;

	typedef struct stonith_history_s {
	- char *target;
	- char *action;
	- char *origin;
	- char *delegate;
	- int completed;
	- int state;
	-
	- struct stonith_history_s *next;
	+ char *target;
	+ char *action;
	+ char *origin;
	+ char *delegate;
	+ int completed;
	+ int state;
	+
	+ struct stonith_history_s *next;
	} stonith_history_t;

	typedef struct stonith_s stonith_t;

	-typedef struct stonith_event_s
	+typedef struct stonith_event_s
	{
	- char *id;
	- char *type;
	- char *message;
	- char *operation;
	-
	- int result;
	- char *origin;
	- char *target;
	- char *executioner;
	-
	- char *device;
	-
	+ char *id;
	+ char *type;
	+ char *message;
	+ char *operation;
	+
	+ int result;
	+ char *origin;
	+ char *target;
	+ char *executioner;
	+
	+ char *device;
	+
	} stonith_event_t;

	typedef struct stonith_api_operations_s
	{
	- int (free) (stonith_t st);
	- int (connect) (stonith_t st, const char name, int stonith_fd);
	- int (disconnect)(stonith_t st);
	-
	- int (*remove_device)(
	- stonith_t st, int options, const char name);
	- int (*register_device)(
	- stonith_t st, int options, const char id,
	- const char namespace, const char agent, stonith_key_value_t *params);
	-
	- int (*remove_level)(
	- stonith_t st, int options, const char node, int level);
	- int (*register_level)(
	- stonith_t st, int options, const char node, int level, stonith_key_value_t *device_list);
	-
	- int (metadata)(stonith_t st, int options,
	- const char device, const char namespace, char **output, int timeout);
	- int (list)(stonith_t stonith, int call_options, const char *namespace,
	- stonith_key_value_t **devices, int timeout);
	-
	- int (call)(stonith_t st, int options, const char *id,
	- const char action, const char port, int timeout);
	-
	- int (query)(stonith_t st, int options, const char *node,
	+ /*!
	+ * \brief Destroy the stonith api structure.
	+ */
	+ int (free) (stonith_t st);
	+
	+ /*!
	+ * \brief Connect to the local stonith daemon.
	+ *
	+ * \retval 0, success
	+ * \retval negative error code on failure
	+ */
	+ int (connect) (stonith_t st, const char name, int stonith_fd);
	+
	+ /*!
	+ * \brief Disconnect from the local stonith daemon.
	+ *
	+ * \retval 0, success
	+ * \retval negative error code on failure
	+ */
	+ int (disconnect)(stonith_t st);
	+
	+ /*!
	+ * \brief Remove a registered stonith device with the local stonith daemon.
	+ *
	+ * \note Synchronous, guaranteed to occur in daemon before function returns.
	+ *
	+ * \retval 0, success
	+ * \retval negative error code on failure
	+ */
	+ int (*remove_device)(
	+ stonith_t st, int options, const char name);
	+
	+ /*!
	+ * \brief Register a stonith device with the local stonith daemon.
	+ *
	+ * \note Synchronous, guaranteed to occur in daemon before function returns.
	+ *
	+ * \retval 0, success
	+ * \retval negative error code on failure
	+ */
	+ int (*register_device)(
	+ stonith_t st, int options, const char id,
	+ const char namespace, const char agent, stonith_key_value_t *params);
	+
	+ /*!
	+ * \brief Remove a fencing level for a specific node.
	+ *
	+ * \note This feature is not available when stonith is in standalone mode.
	+ *
	+ * \retval 0, success
	+ * \retval negative error code on failure
	+ */
	+ int (*remove_level)(
	+ stonith_t st, int options, const char node, int level);
	+
	+ /*!
	+ * \brief Register a fencing level containing the fencing devices to be used
	+ * at that level for a specific node.
	+ *
	+ * \note This feature is not available when stonith is in standalone mode.
	+ *
	+ * \retval 0, success
	+ * \retval negative error code on failure
	+ */
	+ int (*register_level)(
	+ stonith_t st, int options, const char node, int level, stonith_key_value_t *device_list);
	+
	+ /*!
	+ * \brief Get the metadata documentation for a resource.
	+ *
	+ * \note Value is returned in output. Output must be freed when set.
	+ *
	+ * \retval 0 success
	+ * \retval negative error code on failure
	+ */
	+ int (metadata)(stonith_t st, int options,
	+ const char device, const char namespace, char **output, int timeout);
	+
	+ /*!
	+ * \brief Retrieve a list of installed stonith agents
	+ *
	+ * \note if namespace is not provided, all known agents will be returned
	+ * \note list must be freed using stonith_key_value_freeall()
	+ * \note call_options parameter is not used, it is reserved for future use.
	+ *
	+ * \retval num items in list on success
	+ * \retval negative error code on failure
	+ */
	+ int (list_agents)(stonith_t stonith, int call_options, const char *namespace,
	+ stonith_key_value_t **devices, int timeout);
	+
	+ /*!
	+ * \brief Retrieve string listing hosts and port assignments from a local stonith device.
	+ *
	+ * \retval 0 on success
	+ * \retval negative error code on failure
	+ */
	+ int (list)(stonith_t st, int options, const char id, char *list_output, int timeout);
	+
	+ /*!
	+ * \brief Check to see if a local stonith device is reachable
	+ *
	+ * \retval 0 on success
	+ * \retval negative error code on failure
	+ */
	+ int (monitor)(stonith_t st, int options, const char *id, int timeout);
	+
	+ /*!
	+ * \brief Check to see if a local stonith device's port is reachable
	+ *
	+ * \retval 0 on success
	+ * \retval negative error code on failure
	+ */
	+ int (status)(stonith_t st, int options, const char id, const char port, int timeout);
	+
	+ /*!
	+ * \brief Retrieve a list of registered stonith devices.
	+ *
	+ * \note If node is provided, only devices that can fence the node id
	+ * will be returned.
	+ *
	+ * \retval num items in list on success
	+ * \retval negative error code on failure
	+ */
	+ int (query)(stonith_t st, int options, const char *node,
	stonith_key_value_t **devices, int timeout);
	- int (fence)(stonith_t st, int options, const char node, const char action,
	+
	+ /*!
	+ * \brief Issue a fencing action against a node.
	+ *
	+ * \note Possible actions are, 'on', 'off', and 'reboot'.
	+ *
	+ * \retval 0 success
	+ * \retval negative error code on failure.
	+ */
	+ int (fence)(stonith_t st, int options, const char node, const char action,
	int timeout);
	- int (confirm)(stonith_t st, int options, const char *node);
	- int (history)(stonith_t st, int options, const char node, stonith_history_t *output, int timeout);
	-
	- int (*register_notification)(
	- stonith_t st, const char event,
	- void (notify)(stonith_t st, stonith_event_t *e));
	- int (remove_notification)(stonith_t st, const char *event);
	-
	- int (*register_callback)(
	- stonith_t *st, int call_id, int timeout, bool only_success,
	- void userdata, const char callback_name,
	- void (callback)(stonith_t st, const xmlNode msg, int call, int rc, xmlNode output, void *userdata));
	- int (remove_callback)(stonith_t st, int call_id, bool all_callbacks);
	-
	+
	+ /*!
	+ * \brief Manually confirm that a node is down.
	+ *
	+ * \retval 0 success
	+ * \retval negative error code on failure.
	+ */
	+ int (confirm)(stonith_t st, int options, const char *node);
	+
	+ /*!
	+ * \brief Retrieve a list of fencing operations that have occurred for a specific node.
	+ *
	+ * \note History is not available in standalone mode.
	+ *
	+ * \retval 0 success
	+ * \retval negative error code on failure.
	+ */
	+ int (history)(stonith_t st, int options, const char node, stonith_history_t *output, int timeout);
	+
	+ int (*register_notification)(
	+ stonith_t st, const char event,
	+ void (notify)(stonith_t st, stonith_event_t *e));
	+ int (remove_notification)(stonith_t st, const char *event);
	+
	+ int (*register_callback)(
	+ stonith_t *st, int call_id, int timeout, bool only_success,
	+ void userdata, const char callback_name,
	+ void (callback)(stonith_t st, const xmlNode msg, int call, int rc, xmlNode output, void *userdata));
	+ int (remove_callback)(stonith_t st, int call_id, bool all_callbacks);
	+
	} stonith_api_operations_t;

	struct stonith_s
	{
	- enum stonith_state state;
	+ enum stonith_state state;
	+
	+ int call_id;
	+ int call_timeout;
	+ void *private;

	- int call_id;
	- int call_timeout;
	- void *private;
	-
	- stonith_api_operations_t *cmds;
	+ stonith_api_operations_t *cmds;
	};
	/* INDENT-ON */

	/* Core functions */
	stonith_t *stonith_api_new(void);
	void stonith_api_delete(stonith_t * st);

	void stonith_dump_pending_callbacks(stonith_t * st);

	const char get_stonith_provider(const char agent, const char *provider);

	bool stonith_dispatch(stonith_t * st);

	stonith_key_value_t stonith_key_value_add(stonith_key_value_t kvp, const char *key,
	const char *value);
	void stonith_key_value_freeall(stonith_key_value_t * kvp, int keys, int values);

	/* Basic helpers that allows nodes to be fenced and the history to be
	* queried without mainloop or the caller understanding the full API
	*
	* At least one of nodeid and uname are required
	*/
	int stonith_api_kick(int nodeid, const char *uname, int timeout, bool off);
	time_t stonith_api_time(int nodeid, const char *uname, bool in_progress);

	/*
	* Helpers for using the above functions without install-time dependancies
	*
	* Usage:
	* #include <crm/stonith-ng.h>
	*
	* To turn a node off by corosync nodeid:
	* stonith_api_kick_helper(nodeid, 120, 1);
	*
	* To check the last fence date/time (also by nodeid):
	* last = stonith_api_time_helper(nodeid, 0);
	*
	* To check if fencing is in progress:
	* if(stonith_api_time_helper(nodeid, 1) > 0) { ... }
	*
	* eg.

	#include <stdio.h>
	#include <time.h>
	#include <crm/stonith-ng.h>
	int
	main(int argc, char ** argv)
	{
	int rc = 0;
	int nodeid = 102;

	rc = stonith_api_time_helper(nodeid, 0);
	printf("%d last fenced at %s\n", nodeid, ctime(rc));

	rc = stonith_api_kick_helper(nodeid, 120, 1);
	printf("%d fence result: %d\n", nodeid, rc);

	rc = stonith_api_time_helper(nodeid, 0);
	printf("%d last fenced at %s\n", nodeid, ctime(rc));

	return 0;
	}

	*/

	# define STONITH_LIBRARY "libstonithd.so.1"

	static inline int
	stonith_api_kick_helper(int nodeid, int timeout, bool off)
	{
	static void *st_library = NULL;
	static int (st_kick_fn) (int nodeid, const char uname, int timeout, bool off) = NULL;

	if (st_library == NULL) {
	st_library = dlopen(STONITH_LIBRARY, RTLD_LAZY);
	}
	if (st_library && st_kick_fn == NULL) {
	st_kick_fn = dlsym(st_library, "stonith_api_kick");
	}
	if (st_kick_fn == NULL) {
	return -ELIBACC;
	}

	return (*st_kick_fn) (nodeid, NULL, timeout, off);
	}

	static inline time_t
	stonith_api_time_helper(int nodeid, bool in_progress)
	{
	static void *st_library = NULL;
	static time_t(st_time_fn) (int nodeid, const char uname, bool in_progress) = NULL;

	if (st_library == NULL) {
	st_library = dlopen(STONITH_LIBRARY, RTLD_LAZY);
	}
	if (st_library && st_time_fn == NULL) {
	st_time_fn = dlsym(st_library, "stonith_api_time");
	}
	if (st_time_fn == NULL) {
	return 0;
	}

	return (*st_time_fn) (nodeid, NULL, in_progress);
	}

	#endif
	diff --git a/lib/common/xml.c b/lib/common/xml.c
	index a850c27cae..4368f4b0a8 100644
	--- a/lib/common/xml.c
	+++ b/lib/common/xml.c
	@@ -1,2922 +1,2924 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <time.h>
	#include <string.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <ctype.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <libxml/xmlreader.h>

	#if HAVE_BZLIB_H
	# include <bzlib.h>
	#endif

	#if HAVE_LIBXML2
	# include <libxml/parser.h>
	# include <libxml/tree.h>
	# include <libxml/relaxng.h>
	#endif

	#if HAVE_LIBXSLT
	# include <libxslt/xslt.h>
	# include <libxslt/transform.h>
	#endif

	#define XML_BUFFER_SIZE 4096
	#define XML_PARSER_DEBUG 0
	#define BEST_EFFORT_STATUS 0

	void xml_log(int priority, const char * fmt, ...) G_GNUC_PRINTF(2,3);

	void xml_log(int priority, const char * fmt, ...)
	{
	va_list ap;
	va_start(ap, fmt);
	qb_log_from_external_source_va(__FUNCTION__, __FILE__, fmt, priority, __LINE__, 0, ap);
	va_end(ap);
	}

	typedef struct
	{
	xmlRelaxNGPtr rng;
	xmlRelaxNGValidCtxtPtr valid;
	xmlRelaxNGParserCtxtPtr parser;
	} relaxng_ctx_cache_t;

	struct schema_s
	{
	int type;
	const char *name;
	const char *location;
	const char *transform;
	int after_transform;
	void *cache;
	};

	struct schema_s known_schemas[] = {
	/* 0 */ { 0, NULL, NULL, NULL, 1 },
	/* 1 */ { 1, "pacemaker-0.6", "crm.dtd", "upgrade06.xsl", 4, NULL },
	/* 2 */ { 1, "transitional-0.6", "crm-transitional.dtd", "upgrade06.xsl", 4, NULL },
	/* 3 */ { 2, "pacemaker-0.7", "pacemaker-1.0.rng", NULL, 0, NULL },
	/* 4 */ { 2, "pacemaker-1.0", "pacemaker-1.0.rng", NULL, 6, NULL },
	/* 5 */ { 2, "pacemaker-1.1", "pacemaker-1.1.rng", NULL, 6, NULL },
	/* 6 */ { 2, "pacemaker-1.2", "pacemaker-1.2.rng", NULL, 0, NULL },
	/* 7 */ { 0, "none", NULL, NULL, 0, NULL },
	};

	static int all_schemas = DIMOF(known_schemas);
	static int max_schemas = DIMOF(known_schemas) - 2; /* skip back past 'none' */


	typedef struct
	{
	int found;
	const char *string;
	} filter_t;

	static filter_t filter[] = {
	{ 0, XML_ATTR_ORIGIN },
	{ 0, XML_CIB_ATTR_WRITTEN },
	{ 0, XML_ATTR_UPDATE_ORIG },
	{ 0, XML_ATTR_UPDATE_CLIENT },
	{ 0, XML_ATTR_UPDATE_USER },
	};

	static char get_schema_path(const char file)
	{
	static const char *base = NULL;
	if(base == NULL) {
	base = getenv("PCMK_schema_directory");
	}
	if(base == NULL \|\| strlen(base) == 0) {
	base = CRM_DTD_DIRECTORY;
	}
	return crm_concat(base, file, '/');
	}

	int print_spaces(char *buffer, int spaces, int max);

	int get_tag_name(const char *input, size_t offset, size_t max);
	int get_attr_name(const char *input, size_t offset, size_t max);
	int get_attr_value(const char *input, size_t offset, size_t max);
	gboolean can_prune_leaf(xmlNode *xml_node);

	void diff_filter_context(int context, int upper_bound, int lower_bound,
	xmlNode xml_node, xmlNode parent);
	int in_upper_context(int depth, int context, xmlNode *xml_node);
	int write_file(const char string, const char filename);
	xmlNode subtract_xml_object(xmlNode parent, xmlNode left, xmlNode right, gboolean full, const char *marker);
	int add_xml_object(xmlNode parent, xmlNode target, xmlNode *update, gboolean as_diff);

	static inline const char *
	crm_attr_value(xmlAttr *attr)
	{
	if(attr == NULL \|\| attr->children == NULL) {
	return NULL;
	}
	return (const char*)attr->children->content;
	}

	static inline xmlAttr *
	crm_first_attr(xmlNode *xml)
	{
	if(xml == NULL) {
	return NULL;
	}
	return xml->properties;
	}

	xmlNode *
	find_xml_node(xmlNode root, const char search_path, gboolean must_find)
	{
	xmlNode *a_child = NULL;
	const char *name = "NULL";
	if(root != NULL) {
	name = crm_element_name(root);
	}

	if(search_path == NULL) {
	crm_warn("Will never find <NULL>");
	return NULL;
	}

	for(a_child = __xml_first_child(root); a_child != NULL; a_child = __xml_next(a_child)) {
	if(crm_str_eq((const char *)a_child->name, search_path, TRUE)) {
	/* crm_trace("returning node (%s).", crm_element_name(a_child)); */
	return a_child;
	}
	}

	if(must_find) {
	crm_warn("Could not find %s in %s.", search_path, name);
	} else if(root != NULL) {
	crm_trace("Could not find %s in %s.", search_path, name);
	} else {
	crm_trace("Could not find %s in <NULL>.", search_path);
	}


	return NULL;
	}

	xmlNode*
	find_entity(xmlNode parent, const char node_name, const char *id)
	{
	xmlNode *a_child = NULL;
	for(a_child = __xml_first_child(parent); a_child != NULL; a_child = __xml_next(a_child)) {
	/* Uncertain if node_name == NULL check is strictly necessary here */
	if(node_name == NULL \|\| crm_str_eq((const char *)a_child->name, node_name, TRUE)) {
	if(id == NULL \|\| crm_str_eq(id, ID(a_child), TRUE)) {
	crm_trace("returning node (%s).",
	crm_element_name(a_child));
	return a_child;
	}
	}
	}

	crm_trace("node <%s id=%s> not found in %s.",
	node_name, id, crm_element_name(parent));
	return NULL;
	}

	void
	copy_in_properties(xmlNode* target, xmlNode *src)
	{
	if(src == NULL) {
	crm_warn("No node to copy properties from");

	} else if (target == NULL) {
	crm_err("No node to copy properties into");

	} else {
	xmlAttrPtr pIter = NULL;
	for(pIter = crm_first_attr(src); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	expand_plus_plus(target, p_name, p_value);
	}
	}

	return;
	}

	void fix_plus_plus_recursive(xmlNode* target)
	{
	/* TODO: Remove recursion and use xpath searches for value++ */
	xmlNode *child = NULL;
	xmlAttrPtr pIter = NULL;
	for(pIter = crm_first_attr(target); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	expand_plus_plus(target, p_name, p_value);
	}
	for(child = __xml_first_child(target); child != NULL; child = __xml_next(child)) {
	fix_plus_plus_recursive(child);
	}
	}


	void
	expand_plus_plus(xmlNode* target, const char name, const char value)
	{
	int offset = 1;
	int name_len = 0;
	int int_value = 0;
	int value_len = 0;

	const char *old_value = NULL;

	if(value == NULL \|\| name == NULL) {
	return;
	}

	old_value = crm_element_value(target, name);

	if(old_value == NULL) {
	/* if no previous value, set unexpanded */
	goto set_unexpanded;

	} else if(strstr(value, name) != value) {
	goto set_unexpanded;
	}

	name_len = strlen(name);
	value_len = strlen(value);
	if(value_len < (name_len + 2)
	\|\| value[name_len] != '+'
	\|\| (value[name_len+1] != '+' && value[name_len+1] != '=')) {
	goto set_unexpanded;
	}

	/* if we are expanding ourselves,
	* then no previous value was set and leave int_value as 0
	*/
	if(old_value != value) {
	int_value = char2score(old_value);
	}

	if(value[name_len+1] != '+') {
	const char *offset_s = value+(name_len+2);
	offset = char2score(offset_s);
	}
	int_value += offset;

	if(int_value > INFINITY) {
	int_value = INFINITY;
	}

	crm_xml_add_int(target, name, int_value);
	return;

	set_unexpanded:
	if(old_value == value) {
	/* the old value is already set, nothing to do */
	return;
	}
	crm_xml_add(target, name, value);
	return;
	}

	xmlDoc getDocPtr(xmlNode node)
	{
	xmlDoc *doc = NULL;
	CRM_CHECK(node != NULL, return NULL);

	doc = node->doc;
	if(doc == NULL) {
	doc = xmlNewDoc((const xmlChar*)"1.0");
	xmlDocSetRootElement(doc, node);
	xmlSetTreeDoc(node, doc);
	}
	return doc;
	}

	xmlNode*
	add_node_copy(xmlNode parent, xmlNode src_node)
	{
	xmlNode *child = NULL;
	xmlDoc *doc = getDocPtr(parent);
	CRM_CHECK(src_node != NULL, return NULL);

	child = xmlDocCopyNode(src_node, doc, 1);
	xmlAddChild(parent, child);
	return child;
	}


	int
	add_node_nocopy(xmlNode parent, const char name, xmlNode *child)
	{
	add_node_copy(parent, child);
	free_xml(child);
	return 1;
	}

	const char *
	crm_xml_add(xmlNode* node, const char name, const char value)
	{
	xmlAttr *attr = NULL;
	CRM_CHECK(node != NULL, return NULL);
	CRM_CHECK(name != NULL, return NULL);

	if(value == NULL) {
	return NULL;
	}

	#if XML_PARANOIA_CHECKS
	{
	const char *old_value = NULL;
	old_value = crm_element_value(node, name);

	/* Could be re-setting the same value */
	CRM_CHECK(old_value != value,
	crm_err("Cannot reset %s with crm_xml_add(%s)",
	name, value);
	return value);
	}
	#endif

	attr = xmlSetProp(node, (const xmlChar)name, (const xmlChar)value);
	CRM_CHECK(attr && attr->children && attr->children->content, return NULL);
	return (char *)attr->children->content;
	}

	const char *
	crm_xml_replace(xmlNode* node, const char name, const char value)
	{
	xmlAttr *attr = NULL;
	const char *old_value = NULL;
	CRM_CHECK(node != NULL, return NULL);
	CRM_CHECK(name != NULL && name[0] != 0, return NULL);

	old_value = crm_element_value(node, name);

	/* Could be re-setting the same value */
	CRM_CHECK(old_value != value, return value);

	if (old_value != NULL && value == NULL) {
	xml_remove_prop(node, name);
	return NULL;

	} else if(value == NULL) {
	return NULL;
	}

	attr = xmlSetProp(node, (const xmlChar)name, (const xmlChar)value);
	CRM_CHECK(attr && attr->children && attr->children->content, return NULL);
	return (char *)attr->children->content;
	}

	const char *
	crm_xml_add_int(xmlNode* node, const char *name, int value)
	{
	char *number = crm_itoa(value);
	const char *added = crm_xml_add(node, name, number);
	free(number);
	return added;
	}

	xmlNode*
	create_xml_node(xmlNode parent, const char name)
	{
	xmlDoc *doc = NULL;
	xmlNode *node = NULL;

	if (name == NULL \|\| name[0] == 0) {
	return NULL;
	}

	if(parent == NULL) {
	doc = xmlNewDoc((const xmlChar*)"1.0");
	node = xmlNewDocRawNode(doc, NULL, (const xmlChar*)name, NULL);
	xmlDocSetRootElement(doc, node);

	} else {
	doc = getDocPtr(parent);
	node = xmlNewDocRawNode(doc, NULL, (const xmlChar*)name, NULL);
	xmlAddChild(parent, node);
	}
	return node;
	}

	void
	free_xml_from_parent(xmlNode parent, xmlNode a_node)
	{
	CRM_CHECK(a_node != NULL, return);

	xmlUnlinkNode(a_node);
	xmlFreeNode(a_node);
	}

	xmlNode*
	copy_xml(xmlNode *src)
	{
	xmlDoc doc = xmlNewDoc((const xmlChar)"1.0");
	xmlNode *copy = xmlDocCopyNode(src, doc, 1);
	xmlDocSetRootElement(doc, copy);
	xmlSetTreeDoc(copy, doc);
	return copy;
	}


	static void crm_xml_err(void * ctx, const char * msg, ...) G_GNUC_PRINTF(2,3);

	int
	write_file(const char string, const char filename)
	{
	int rc = 0;
	FILE *file_output_strm = NULL;

	CRM_CHECK(filename != NULL, return -1);

	if (string == NULL) {
	crm_err("Cannot write NULL to %s", filename);
	return -1;
	}

	file_output_strm = fopen(filename, "w");
	if(file_output_strm == NULL) {
	crm_perror(LOG_ERR,"Cannot open %s for writing", filename);
	return -1;
	}

	rc = fprintf(file_output_strm, "%s", string);
	if(rc < 0) {
	crm_perror(LOG_ERR,"Cannot write output to %s", filename);
	}

	if(fflush(file_output_strm) != 0) {
	crm_perror(LOG_ERR,"fflush for %s failed:", filename);
	rc = -1;
	}

	if(fsync(fileno(file_output_strm)) < 0) {
	crm_perror(LOG_ERR,"fsync for %s failed:", filename);
	rc = -1;
	}

	fclose(file_output_strm);
	return rc;
	}

	static void crm_xml_err(void * ctx, const char * msg, ...)
	{
	int len = 0;
	va_list args;
	char *buf = NULL;
	static int buffer_len = 0;
	static char *buffer = NULL;

	va_start(args, msg);
	len = vasprintf(&buf, msg, args);

	if(strchr(buf, '\n')) {
	buf[len - 1] = 0;
	if(buffer) {
	crm_err("XML Error: %s%s", buffer, buf);
	free(buffer);
	} else {
	crm_err("XML Error: %s", buf);
	}
	buffer = NULL;
	buffer_len = 0;

	} else if(buffer == NULL) {
	buffer_len = len;
	buffer = buf;
	buf = NULL;

	} else {
	buffer = realloc(buffer, 1+buffer_len+len);
	memcpy(buffer+buffer_len, buf, len);
	buffer_len += len;
	buffer[buffer_len] = 0;
	}

	va_end(args);
	free(buf);
	}

	xmlNode*
	string2xml(const char *input)
	{
	xmlNode *xml = NULL;
	xmlDocPtr output = NULL;
	xmlParserCtxtPtr ctxt = NULL;
	xmlErrorPtr last_error = NULL;

	if(input == NULL) {
	crm_err("Can't parse NULL input");
	return NULL;
	}

	/* create a parser context */
	ctxt = xmlNewParserCtxt();
	CRM_CHECK(ctxt != NULL, return NULL);

	/* xmlCtxtUseOptions(ctxt, XML_PARSE_NOBLANKS\|XML_PARSE_RECOVER); */

	xmlCtxtResetLastError(ctxt);
	xmlSetGenericErrorFunc(ctxt, crm_xml_err);
	/* initGenericErrorDefaultFunc(crm_xml_err); */
	output = xmlCtxtReadDoc(ctxt, (const xmlChar*)input, NULL, NULL, XML_PARSE_NOBLANKS\|XML_PARSE_RECOVER);
	if(output) {
	xml = xmlDocGetRootElement(output);
	}
	last_error = xmlCtxtGetLastError(ctxt);
	if(last_error && last_error->code != XML_ERR_OK) {
	/* crm_abort(__FILE__,__PRETTY_FUNCTION__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
	/*
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
	*/
	crm_warn("Parsing failed (domain=%d, level=%d, code=%d): %s",
	last_error->domain, last_error->level,
	last_error->code, last_error->message);

	if(last_error->code != XML_ERR_DOCUMENT_END) {
	crm_err("Couldn't%s parse %d chars: %s", xml?" fully":"", (int)strlen(input), input);
	if(xml != NULL) {
	crm_log_xml_err(xml, "Partial");
	}

	} else {
	int len = strlen(input);
	crm_warn("String start: %.50s", input);
	crm_warn("String start+%d: %s", len-50, input+len-50);
	crm_abort(__FILE__,__PRETTY_FUNCTION__,__LINE__, "String parsing error", TRUE, TRUE);
	}
	}

	xmlFreeParserCtxt(ctxt);
	return xml;
	}

	xmlNode *
	stdin2xml(void)
	{
	size_t data_length = 0;
	size_t read_chars = 0;

	char *xml_buffer = NULL;
	xmlNode *xml_obj = NULL;

	do {
	xml_buffer = realloc(xml_buffer, XML_BUFFER_SIZE + data_length + 1);
	read_chars = fread(xml_buffer + data_length, 1, XML_BUFFER_SIZE, stdin);
	data_length += read_chars;
	} while (read_chars > 0);

	if(data_length == 0) {
	crm_warn("No XML supplied on stdin");
	free(xml_buffer);
	return NULL;
	}

	xml_buffer[data_length] = '\0';

	xml_obj = string2xml(xml_buffer);
	free(xml_buffer);

	crm_log_xml_trace(xml_obj, "Created fragment");
	return xml_obj;
	}

	static char *
	decompress_file(const char *filename)
	{
	char *buffer = NULL;
	#if HAVE_BZLIB_H
	int rc = 0;
	size_t length = 0, read_len = 0;

	BZFILE *bz_file = NULL;
	FILE *input = fopen(filename, "r");

	if(input == NULL) {
	crm_perror(LOG_ERR,"Could not open %s for reading", filename);
	return NULL;
	}

	bz_file = BZ2_bzReadOpen(&rc, input, 0, 0, NULL, 0);

	if ( rc != BZ_OK ) {
	BZ2_bzReadClose ( &rc, bz_file);
	return NULL;
	}

	rc = BZ_OK;
	while ( rc == BZ_OK ) {
	buffer = realloc(buffer, XML_BUFFER_SIZE + length + 1);
	read_len = BZ2_bzRead (
	&rc, bz_file, buffer + length, XML_BUFFER_SIZE);

	crm_trace("Read %ld bytes from file: %d",
	(long)read_len, rc);

	if ( rc == BZ_OK \|\| rc == BZ_STREAM_END) {
	length += read_len;
	}
	}

	buffer[length] = '\0';
	read_len = length;

	if ( rc != BZ_STREAM_END ) {
	crm_err("Couldnt read compressed xml from file");
	free(buffer);
	buffer = NULL;
	}

	BZ2_bzReadClose (&rc, bz_file);
	fclose(input);

	#else
	crm_err("Cannot read compressed files:"
	" bzlib was not available at compile time");
	#endif
	return buffer;
	}

	static void strip_text_nodes(xmlNode *xml)
	{
	- xmlNode *iter = NULL;
	- for(iter = xml->children; iter; iter = iter->next) {
	+ xmlNode *iter = xml->children;
	+
	+ while (iter) {
	+ xmlNode *next = iter->next;
	+
	switch(iter->type) {
	case XML_TEXT_NODE:
	/* Remove it */
	xmlUnlinkNode(iter);
	xmlFreeNode(iter);
	-
	- /* Start again since xml->children will have changed */
	- strip_text_nodes(xml);
	- return;
	+ break;

	case XML_ELEMENT_NODE:
	/* Search it */
	strip_text_nodes(iter);
	break;

	default:
	/* Leave it */
	break;
	}
	+
	+ iter = next;
	}
	}

	xmlNode *
	filename2xml(const char *filename)
	{
	xmlNode *xml = NULL;
	xmlDocPtr output = NULL;
	const char *match = NULL;
	xmlParserCtxtPtr ctxt = NULL;
	xmlErrorPtr last_error = NULL;
	static int xml_options = XML_PARSE_NOBLANKS\|XML_PARSE_RECOVER;

	/* create a parser context */
	ctxt = xmlNewParserCtxt();
	CRM_CHECK(ctxt != NULL, return NULL);

	/* xmlCtxtUseOptions(ctxt, XML_PARSE_NOBLANKS\|XML_PARSE_RECOVER); */

	xmlCtxtResetLastError(ctxt);
	xmlSetGenericErrorFunc(ctxt, crm_xml_err);
	/* initGenericErrorDefaultFunc(crm_xml_err); */

	if(filename) {
	match = strstr(filename, ".bz2");
	}

	if(filename == NULL) {
	/* STDIN_FILENO == fileno(stdin) */
	output = xmlCtxtReadFd(ctxt, STDIN_FILENO, "unknown.xml", NULL, xml_options);

	} else if(match == NULL \|\| match[4] != 0) {
	output = xmlCtxtReadFile(ctxt, filename, NULL, xml_options);

	} else {
	char *input = decompress_file(filename);
	output = xmlCtxtReadDoc(ctxt, (const xmlChar*)input, NULL, NULL, xml_options);
	free(input);
	}

	if(output && (xml = xmlDocGetRootElement(output))) {
	strip_text_nodes(xml);
	}

	last_error = xmlCtxtGetLastError(ctxt);
	if(last_error && last_error->code != XML_ERR_OK) {
	/* crm_abort(__FILE__,__PRETTY_FUNCTION__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
	/*
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
	*/
	crm_err("Parsing failed (domain=%d, level=%d, code=%d): %s",
	last_error->domain, last_error->level,
	last_error->code, last_error->message);

	if(last_error && last_error->code != XML_ERR_OK) {
	crm_err("Couldn't%s parse %s", xml?" fully":"", filename);
	if(xml != NULL) {
	crm_log_xml_err(xml, "Partial");
	}
	}
	}

	xmlFreeParserCtxt(ctxt);
	return xml;
	}

	int
	write_xml_file(xmlNode xml_node, const char filename, gboolean compress)
	{
	int res = 0;
	time_t now;
	char *buffer = NULL;
	char *now_str = NULL;
	unsigned int out = 0;
	FILE *file_output_strm = NULL;
	static mode_t cib_mode = S_IRUSR\|S_IWUSR;

	CRM_CHECK(filename != NULL, return -1);

	crm_trace("Writing XML out to %s", filename);
	if (xml_node == NULL) {
	crm_err("Cannot write NULL to %s", filename);
	return -1;
	}

	file_output_strm = fopen(filename, "w");
	if(file_output_strm == NULL) {
	crm_perror(LOG_ERR,"Cannot open %s for writing", filename);
	return -1;
	}

	/* establish the correct permissions */
	fchmod(fileno(file_output_strm), cib_mode);

	crm_log_xml_trace(xml_node, "Writing out");

	now = time(NULL);
	now_str = ctime(&now);
	now_str[24] = EOS; /* replace the newline */
	crm_xml_add(xml_node, XML_CIB_ATTR_WRITTEN, now_str);

	buffer = dump_xml_formatted(xml_node);
	CRM_CHECK(buffer != NULL && strlen(buffer) > 0,
	crm_log_xml_warn(xml_node, "dump:failed");
	goto bail);

	if(compress) {
	#if HAVE_BZLIB_H
	int rc = BZ_OK;
	unsigned int in = 0;
	BZFILE *bz_file = NULL;
	bz_file = BZ2_bzWriteOpen(&rc, file_output_strm, 5, 0, 30);
	if(rc != BZ_OK) {
	crm_err("bzWriteOpen failed: %d", rc);
	} else {
	BZ2_bzWrite(&rc,bz_file,buffer,strlen(buffer));
	if(rc != BZ_OK) {
	crm_err("bzWrite() failed: %d", rc);
	}
	}

	if(rc == BZ_OK) {
	BZ2_bzWriteClose(&rc, bz_file, 0, &in, &out);
	if(rc != BZ_OK) {
	crm_err("bzWriteClose() failed: %d",rc);
	out = -1;
	} else {
	crm_trace("%s: In: %d, out: %d", filename, in, out);
	}
	}
	#else
	crm_err("Cannot write compressed files:"
	" bzlib was not available at compile time");
	#endif
	}

	if(out <= 0) {
	res = fprintf(file_output_strm, "%s", buffer);
	if(res < 0) {
	crm_perror(LOG_ERR,"Cannot write output to %s", filename);
	goto bail;
	}
	}

	bail:

	if(fflush(file_output_strm) != 0) {
	crm_perror(LOG_ERR,"fflush for %s failed:", filename);
	res = -1;
	}

	if(fsync(fileno(file_output_strm)) < 0) {
	crm_perror(LOG_ERR,"fsync for %s failed:", filename);
	res = -1;
	}

	fclose(file_output_strm);

	crm_trace("Saved %d bytes to the Cib as XML", res);
	free(buffer);

	return res;
	}

	xmlNode *
	get_message_xml(xmlNode msg, const char field)
	{
	xmlNode *tmp = first_named_child(msg, field);
	return __xml_first_child(tmp);
	}

	gboolean
	add_message_xml(xmlNode msg, const char field, xmlNode *xml)
	{
	xmlNode *holder = create_xml_node(msg, field);
	add_node_copy(holder, xml);
	return TRUE;
	}

	static char *
	dump_xml(xmlNode *an_xml_node, gboolean formatted, gboolean for_digest)
	{
	int len = 0;
	char *buffer = NULL;
	xmlBuffer *xml_buffer = NULL;
	xmlDoc *doc = getDocPtr(an_xml_node);

	/* doc will only be NULL if an_xml_node is */
	CRM_CHECK(doc != NULL, return NULL);

	xml_buffer = xmlBufferCreate();
	CRM_ASSERT(xml_buffer != NULL);

	len = xmlNodeDump(xml_buffer, doc, an_xml_node, 0, formatted);

	if(len > 0) {
	/* The copying here isn't ideal, but it doesn't even register
	* in the perf numbers
	*/
	if(for_digest) {
	/* for compatability with the old result which is used for digests */
	len += 3;
	buffer = calloc(1, len);
	snprintf(buffer, len, " %s\n", (char *)xml_buffer->content);
	} else {
	buffer = strdup((char *)xml_buffer->content);
	}

	} else {
	crm_err("Conversion failed");
	}

	xmlBufferFree(xml_buffer);
	return buffer;
	}

	char *
	dump_xml_formatted(xmlNode *an_xml_node)
	{
	return dump_xml(an_xml_node, TRUE, FALSE);
	}

	char *
	dump_xml_unformatted(xmlNode *an_xml_node)
	{
	return dump_xml(an_xml_node, FALSE, FALSE);
	}

	#define update_buffer() do { \
	if(printed < 0) { \
	crm_perror(LOG_ERR,"snprintf failed"); \
	goto print; \
	} else if(printed >= (buffer_len - offset)) { \
	crm_err("Output truncated: available=%d, needed=%d", buffer_len - offset, printed); \
	offset += printed; \
	goto print; \
	} else if(offset >= buffer_len) { \
	crm_err("Buffer exceeded"); \
	offset += printed; \
	goto print; \
	} else { \
	offset += printed; \
	} \
	} while(0)

	int
	print_spaces(char *buffer, int depth, int max)
	{
	int lpc = 0;
	int spaces = 2*depth;
	max--;

	/* <= so that we always print 1 space - prevents problems with syslog */
	for(lpc = 0; lpc <= spaces && lpc < max; lpc++) {
	if(sprintf(buffer+lpc, "%c", ' ') < 1) {
	return -1;
	}
	}
	return lpc;
	}

	int
	log_data_element(
	int log_level, const char file, const char function, int line,
	const char prefix, xmlNode data, int depth, int options)
	{
	xmlNode *a_child = NULL;
	int child_result = 0;

	int offset = 0;
	int printed = 0;
	char *buffer = NULL;
	char *prefix_m = NULL;
	int buffer_len = 1000;

	xmlAttrPtr pIter = NULL;
	const char *name = NULL;
	const char *hidden = NULL;
	gboolean formatted = (options & 0x1);
	gboolean diff_plus = (options & 0x2);
	gboolean diff_minus = (options & 0x4);

	/* Since we use the same file and line, to avoid confusing libqb, we need to use the same format strings */
	if(data == NULL) {
	do_crm_log_alias(log_level, file, function, line, "%s%s", prefix, ": No data to dump as XML");
	return 0;
	}

	if(prefix == NULL) {
	prefix = "";
	}

	name = crm_element_name(data);
	CRM_ASSERT(name != NULL);

	/* crm_trace("Dumping %s", name); */
	buffer = calloc(1, buffer_len);

	if(formatted) {
	offset = print_spaces(buffer, depth, buffer_len - offset);
	if(diff_plus && crm_element_value(data, XML_DIFF_MARKER)) {
	prefix_m = strdup(prefix);
	prefix_m[1] = '+';
	prefix = prefix_m;

	} else if(diff_minus && crm_element_value(data, XML_DIFF_MARKER)) {
	prefix_m = strdup(prefix);
	prefix_m[1] = '-';
	prefix = prefix_m;
	}
	}

	printed = snprintf(buffer + offset, buffer_len - offset, "<%s", name);
	update_buffer();

	hidden = crm_element_value(data, "hidden");
	for(pIter = crm_first_attr(data); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	if(p_name == NULL \|\| safe_str_eq(F_XML_TAGNAME, p_name)) {
	continue;

	} else if((diff_plus \|\| diff_minus) && safe_str_eq(XML_DIFF_MARKER, p_name)) {
	continue;

	} else if(hidden != NULL
	&& p_name[0] != 0
	&& strstr(hidden, p_name) != NULL) {
	p_value = "*****";
	}

	/* crm_trace("Dumping <%s %s=\"%s\"...", */
	/* name, prop_name, prop_value); */
	printed = snprintf(buffer + offset, buffer_len - offset,
	" %s=\"%s\"", p_name, p_value);
	update_buffer();
	}

	printed = snprintf(buffer + offset, buffer_len - offset,
	" %s>", xml_has_children(data)?"":"/");
	update_buffer();

	print:
	do_crm_log_alias(log_level, file, function, line, "%s%s", prefix, buffer);

	if(xml_has_children(data) == FALSE) {
	free(prefix_m);
	free(buffer);
	return 0;
	}

	for(a_child = __xml_first_child(data); a_child != NULL; a_child = __xml_next(a_child)) {
	child_result = log_data_element(
	log_level, file, function, line, prefix, a_child, depth+1, options);
	}

	if(formatted) {
	offset = print_spaces(buffer, depth, buffer_len);
	}

	printed = snprintf(buffer + offset, buffer_len - offset, "</%s>", name);
	update_buffer();

	do_crm_log_alias(log_level, file, function, line, "%s%s", prefix, buffer);
	free(prefix_m);
	free(buffer);
	return 1;
	}

	gboolean
	xml_has_children(const xmlNode *xml_root)
	{
	if(xml_root != NULL && xml_root->children != NULL) {
	return TRUE;
	}
	return FALSE;
	}

	int
	crm_element_value_int(xmlNode data, const char name, int *dest)
	{
	const char *value = crm_element_value(data, name);
	CRM_CHECK(dest != NULL, return -1);
	if(value) {
	*dest = crm_int_helper(value, NULL);
	return 0;
	}
	return -1;
	}

	int
	crm_element_value_const_int(const xmlNode data, const char name, int *dest)
	{
	return crm_element_value_int((xmlNode*)data, name, dest);
	}

	const char *
	crm_element_value_const(const xmlNode data, const char name)
	{
	return crm_element_value((xmlNode*)data, name);
	}

	char *
	crm_element_value_copy(xmlNode data, const char name)
	{
	char *value_copy = NULL;
	const char *value = crm_element_value(data, name);
	if(value != NULL) {
	value_copy = strdup(value);
	}
	return value_copy;
	}

	void
	xml_remove_prop(xmlNode obj, const char name)
	{
	xmlUnsetProp(obj, (const xmlChar*)name);
	}

	void
	log_xml_diff(unsigned int log_level, xmlNode diff, const char function)
	{
	xmlNode *child = NULL;
	xmlNode *added = find_xml_node(diff, "diff-added", FALSE);
	xmlNode *removed = find_xml_node(diff, "diff-removed", FALSE);
	gboolean is_first = TRUE;

	static struct qb_log_callsite *diff_cs = NULL;

	if(diff_cs == NULL) {
	diff_cs = qb_log_callsite_get(function, __FILE__, "xml-diff", log_level, __LINE__, 0);
	}

	if (crm_is_callsite_active(diff_cs, log_level) == FALSE) {
	return;
	}

	for(child = __xml_first_child(removed); child != NULL; child = __xml_next(child)) {
	log_data_element(log_level, __FILE__, function, __LINE__, "- ", child, 0, 5);
	if(is_first) {
	is_first = FALSE;
	} else {
	do_crm_log(log_level, " --- ");
	}
	}

	is_first = TRUE;
	for(child = __xml_first_child(added); child != NULL; child = __xml_next(child)) {
	log_data_element(log_level, __FILE__, function, __LINE__, "+ ", child, 0, 3);
	if(is_first) {
	is_first = FALSE;
	} else {
	do_crm_log(log_level, " +++ ");
	}
	}
	}

	void
	purge_diff_markers(xmlNode *a_node)
	{
	xmlNode *child = NULL;
	CRM_CHECK(a_node != NULL, return);

	xml_remove_prop(a_node, XML_DIFF_MARKER);
	for(child = __xml_first_child(a_node); child != NULL; child = __xml_next(child)) {
	purge_diff_markers(child);
	}
	}

	gboolean
	apply_xml_diff(xmlNode old, xmlNode diff, xmlNode **new)
	{
	gboolean result = TRUE;
	int root_nodes_seen = 0;
	const char *digest = crm_element_value(diff, XML_ATTR_DIGEST);
	const char *version = crm_element_value(diff, XML_ATTR_CRM_VERSION);

	xmlNode *child_diff = NULL;
	xmlNode *added = find_xml_node(diff, "diff-added", FALSE);
	xmlNode *removed = find_xml_node(diff, "diff-removed", FALSE);

	CRM_CHECK(new != NULL, return FALSE);

	crm_trace("Substraction Phase");
	for(child_diff = __xml_first_child(removed); child_diff != NULL; child_diff = __xml_next(child_diff)) {
	CRM_CHECK(root_nodes_seen == 0, result = FALSE);
	if(root_nodes_seen == 0) {
	*new = subtract_xml_object(NULL, old, child_diff, FALSE, NULL);
	}
	root_nodes_seen++;
	}

	if(root_nodes_seen == 0) {
	*new = copy_xml(old);

	} else if(root_nodes_seen > 1) {
	crm_err("(-) Diffs cannot contain more than one change set..."
	" saw %d", root_nodes_seen);
	result = FALSE;
	}

	root_nodes_seen = 0;
	crm_trace("Addition Phase");
	if(result) {
	xmlNode *child_diff = NULL;
	for(child_diff = __xml_first_child(added); child_diff != NULL; child_diff = __xml_next(child_diff)) {
	CRM_CHECK(root_nodes_seen == 0, result = FALSE);
	if(root_nodes_seen == 0) {
	add_xml_object(NULL, *new, child_diff, TRUE);
	}
	root_nodes_seen++;
	}
	}

	if(root_nodes_seen > 1) {
	crm_err("(+) Diffs cannot contain more than one change set..."
	" saw %d", root_nodes_seen);
	result = FALSE;

	} else if(result && digest) {
	char *new_digest = NULL;
	purge_diff_markers(new); / Purge now so the diff is ok */
	new_digest = calculate_xml_versioned_digest(*new, FALSE, TRUE, version);
	if(safe_str_neq(new_digest, digest)) {
	crm_info("Digest mis-match: expected %s, calculated %s",
	digest, new_digest);
	crm_log_xml_trace(old, "diff:original");
	crm_log_xml_trace(diff, "diff:input");
	result = FALSE;
	} else {
	crm_trace("Digest matched: expected %s, calculated %s",
	digest, new_digest);
	}
	free(new_digest);

	#if XML_PARANOIA_CHECKS
	} else if(result) {
	int lpc = 0;
	xmlNode *intermediate = NULL;
	xmlNode *diff_of_diff = NULL;
	xmlNode *calc_added = NULL;
	xmlNode *calc_removed = NULL;

	const char *value = NULL;
	const char *name = NULL;
	const char *version_attrs[] = {
	XML_ATTR_NUMUPDATES,
	XML_ATTR_GENERATION,
	XML_ATTR_GENERATION_ADMIN
	};

	crm_trace("Verification Phase");
	intermediate = diff_xml_object(old, *new, FALSE);
	calc_added = find_xml_node(intermediate, "diff-added", FALSE);
	calc_removed = find_xml_node(intermediate, "diff-removed", FALSE);

	/* add any version details to the diff so they match */
	for(lpc = 0; lpc < DIMOF(version_attrs); lpc++) {
	name = version_attrs[lpc];

	value = crm_element_value(added, name);
	crm_xml_add(calc_added, name, value);

	value = crm_element_value(removed, name);
	crm_xml_add(calc_removed, name, value);
	}

	diff_of_diff = diff_xml_object(intermediate, diff, TRUE);
	if(diff_of_diff != NULL) {
	crm_info("Diff application failed!");
	crm_log_xml_debug(old, "diff:original");
	crm_log_xml_debug(diff, "diff:input");
	result = FALSE;
	} else {
	purge_diff_markers(*new);
	}

	free_xml(diff_of_diff);
	free_xml(intermediate);
	diff_of_diff = NULL;
	intermediate = NULL;
	#endif
	}

	return result;
	}


	xmlNode *
	diff_xml_object(xmlNode old, xmlNode new, gboolean suppress)
	{
	xmlNode *tmp1 = NULL;
	xmlNode *diff = create_xml_node(NULL, "diff");
	xmlNode *removed = create_xml_node(diff, "diff-removed");
	xmlNode *added = create_xml_node(diff, "diff-added");

	crm_xml_add(diff, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);

	tmp1 = subtract_xml_object(removed, old, new, FALSE, "removed:top");
	if(suppress && tmp1 != NULL && can_prune_leaf(tmp1)) {
	free_xml_from_parent(removed, tmp1);
	}

	tmp1 = subtract_xml_object(added, new, old, TRUE, "added:top");
	if(suppress && tmp1 != NULL && can_prune_leaf(tmp1)) {
	free_xml_from_parent(added, tmp1);
	}

	if(added->children == NULL && removed->children == NULL) {
	free_xml(diff);
	diff = NULL;
	}

	return diff;
	}

	gboolean
	can_prune_leaf(xmlNode *xml_node)
	{
	xmlNode *child = NULL;
	xmlAttrPtr pIter = NULL;
	gboolean can_prune = TRUE;

	for(pIter = crm_first_attr(xml_node); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	if(safe_str_eq(p_name, XML_ATTR_ID)) {
	continue;
	}
	can_prune = FALSE;
	}

	for(child = __xml_first_child(xml_node); child != NULL; child = __xml_next(child)) {
	if(can_prune_leaf(child)) {
	free_xml(child);
	} else {
	can_prune = FALSE;
	}
	}
	return can_prune;
	}


	void
	diff_filter_context(int context, int upper_bound, int lower_bound,
	xmlNode xml_node, xmlNode parent)
	{
	xmlNode *us = NULL;
	xmlNode *child = NULL;
	xmlAttrPtr pIter = NULL;
	xmlNode *new_parent = parent;
	const char *name = crm_element_name(xml_node);

	CRM_CHECK(xml_node != NULL && name != NULL, return);

	us = create_xml_node(parent, name);
	for(pIter = crm_first_attr(xml_node); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	lower_bound = context;
	crm_xml_add(us, p_name, p_value);
	}

	if(lower_bound >= 0 \|\| upper_bound >= 0) {
	crm_xml_add(us, XML_ATTR_ID, ID(xml_node));
	new_parent = us;

	} else {
	upper_bound = in_upper_context(0, context, xml_node);
	if(upper_bound >= 0) {
	crm_xml_add(us, XML_ATTR_ID, ID(xml_node));
	new_parent = us;
	} else {
	free_xml(us);
	us = NULL;
	}
	}

	for(child = __xml_first_child(us); child != NULL; child = __xml_next(child)) {
	diff_filter_context(context, upper_bound-1, lower_bound-1,
	child, new_parent);
	}
	}

	int
	in_upper_context(int depth, int context, xmlNode *xml_node)
	{
	if(context == 0) {
	return 0;
	}

	if(xml_node->properties) {
	return depth;

	} else if(depth < context) {
	xmlNode *child = NULL;
	for(child = __xml_first_child(xml_node); child != NULL; child = __xml_next(child)) {
	if(in_upper_context(depth+1, context, child)) {
	return depth;
	}
	}
	}
	return 0;
	}


	xmlNode *
	subtract_xml_object(xmlNode parent, xmlNode left, xmlNode right, gboolean full, const char marker)
	{
	gboolean skip = FALSE;
	gboolean differences = FALSE;
	xmlNode *diff = NULL;
	xmlNode *child_diff = NULL;
	xmlNode *right_child = NULL;
	xmlNode *left_child = NULL;
	xmlAttrPtr xIter = NULL;

	const char *id = NULL;
	const char *name = NULL;
	const char *value = NULL;
	const char *right_val = NULL;

	int lpc = 0;
	static int filter_len = DIMOF(filter);

	if(left == NULL) {
	return NULL;
	}

	id = ID(left);
	if(right == NULL) {
	xmlNode *deleted = NULL;

	crm_trace("Processing <%s id=%s> (complete copy)",
	crm_element_name(left), id);
	deleted = add_node_copy(parent, left);
	crm_xml_add(deleted, XML_DIFF_MARKER, marker);

	return deleted;
	}

	name = crm_element_name(left);
	CRM_CHECK(name != NULL, return NULL);

	/* Avoiding creating the full heirarchy would save even more work here */
	diff = create_xml_node(parent, name);

	/* Reset filter */
	for(lpc = 0; lpc < filter_len; lpc++){
	filter[lpc].found = FALSE;
	}

	/* changes to child objects */
	for(left_child = __xml_first_child(left); left_child != NULL; left_child = __xml_next(left_child)) {
	right_child = find_entity(
	right, crm_element_name(left_child), ID(left_child));
	child_diff = subtract_xml_object(diff, left_child, right_child, full, marker);
	if(child_diff != NULL) {
	differences = TRUE;
	}
	}

	if(differences == FALSE) {
	/* check for XML_DIFF_MARKER in a child */
	for(right_child = __xml_first_child(right); right_child != NULL; right_child = __xml_next(right_child)) {
	value = crm_element_value(right_child, XML_DIFF_MARKER);
	if(value != NULL && safe_str_eq(value, "removed:top")) {
	xmlAttrPtr pIter = NULL;

	crm_trace("Found the root of the deletion: %s", name);
	for(pIter = crm_first_attr(left); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);
	xmlSetProp(diff, (const xmlChar)p_name, (const xmlChar)p_value);
	}
	differences = TRUE;
	goto done;
	}
	}

	} else if(full) {
	xmlAttrPtr pIter = NULL;
	for(pIter = crm_first_attr(left); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);
	xmlSetProp(diff, (const xmlChar)p_name, (const xmlChar)p_value);
	}

	/* We already have everything we need... */
	goto done;

	} else if(id) {
	xmlSetProp(diff, (const xmlChar)XML_ATTR_ID, (const xmlChar)id);
	}

	/* changes to name/value pairs */
	for(xIter = crm_first_attr(left); xIter != NULL; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;

	if(crm_str_eq(prop_name, XML_ATTR_ID, TRUE)) {
	continue;
	}

	skip = FALSE;
	for(lpc = 0; skip == FALSE && lpc < filter_len; lpc++){
	if(filter[lpc].found == FALSE && crm_str_eq(prop_name, filter[lpc].string, TRUE)) {
	filter[lpc].found = TRUE;
	skip = TRUE;
	break;
	}
	}

	if(skip) { continue; }

	right_val = crm_element_value(right, prop_name);
	if(right_val == NULL) {
	/* new */
	differences = TRUE;
	if(full) {
	xmlAttrPtr pIter = NULL;
	for(pIter = crm_first_attr(left); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);
	xmlSetProp(diff, (const xmlChar)p_name, (const xmlChar)p_value);
	}
	break;

	} else {
	const char *left_value = crm_element_value(left, prop_name);
	xmlSetProp(diff, (const xmlChar)prop_name, (const xmlChar)value);
	crm_xml_add(diff, prop_name, left_value);
	}

	} else {
	/* Only now do we need the left value */
	const char *left_value = crm_element_value(left, prop_name);
	if(strcmp(left_value, right_val) == 0) {
	/* unchanged */

	} else {
	/* changed */
	differences = TRUE;
	if(full) {
	xmlAttrPtr pIter = NULL;
	for(pIter = crm_first_attr(left); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	xmlSetProp(diff, (const xmlChar)p_name, (const xmlChar)p_value);
	}
	break;

	} else {
	crm_xml_add(diff, prop_name, left_value);
	}
	}
	}
	}

	if(differences == FALSE) {
	free_xml_from_parent(parent, diff);
	crm_trace("\tNo changes to <%s id=%s>", crm_str(name), id);
	return NULL;

	} else if(full == FALSE && id) {
	crm_xml_add(diff, XML_ATTR_ID, id);
	}
	done:
	return diff;
	}

	int
	add_xml_object(xmlNode parent, xmlNode target, xmlNode *update, gboolean as_diff)
	{
	xmlNode *a_child = NULL;
	const char *object_id = NULL;
	const char *object_name = NULL;

	#if XML_PARSE_DEBUG
	crm_log_xml_trace("update:", update);
	crm_log_xml_trace("target:", target);
	#endif

	CRM_CHECK(update != NULL, return 0);

	object_name = crm_element_name(update);
	object_id = ID(update);

	CRM_CHECK(object_name != NULL, return 0);

	if(target == NULL && object_id == NULL) {
	/* placeholder object */
	target = find_xml_node(parent, object_name, FALSE);

	} else if(target == NULL) {
	target = find_entity(parent, object_name, object_id);
	}

	if(target == NULL) {
	target = create_xml_node(parent, object_name);
	CRM_CHECK(target != NULL, return 0);
	#if XML_PARSER_DEBUG
	crm_trace("Added <%s%s%s/>", crm_str(object_name),
	object_id?" id=":"", object_id?object_id:"");

	} else {
	crm_trace("Found node <%s%s%s/> to update",
	crm_str(object_name),
	object_id?" id=":"", object_id?object_id:"");
	#endif
	}

	if(as_diff == FALSE) {
	/* So that expand_plus_plus() gets called */
	copy_in_properties(target, update);

	} else {
	/* No need for expand_plus_plus(), just raw speed */
	xmlAttrPtr pIter = NULL;
	for(pIter = crm_first_attr(update); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);
	/* Remove it first so the ordering of the update is preserved */
	xmlUnsetProp(target, (const xmlChar*)p_name);
	xmlSetProp(target, (const xmlChar)p_name, (const xmlChar)p_value);
	}
	}

	for(a_child = __xml_first_child(update); a_child != NULL; a_child = __xml_next(a_child)) {
	#if XML_PARSER_DEBUG
	crm_trace("Updating child <%s id=%s>",
	crm_element_name(a_child), ID(a_child));
	#endif
	add_xml_object(target, NULL, a_child, as_diff);
	}

	#if XML_PARSER_DEBUG
	crm_trace("Finished with <%s id=%s>",
	crm_str(object_name), crm_str(object_id));
	#endif
	return 0;
	}

	gboolean
	update_xml_child(xmlNode child, xmlNode to_update)
	{
	gboolean can_update = TRUE;
	xmlNode *child_of_child = NULL;

	CRM_CHECK(child != NULL, return FALSE);
	CRM_CHECK(to_update != NULL, return FALSE);

	if(safe_str_neq(crm_element_name(to_update), crm_element_name(child))) {
	can_update = FALSE;

	} else if(safe_str_neq(ID(to_update), ID(child))) {
	can_update = FALSE;

	} else if(can_update) {
	#if XML_PARSER_DEBUG
	crm_log_xml_trace(child, "Update match found...");
	#endif
	add_xml_object(NULL, child, to_update, FALSE);
	}

	for(child_of_child = __xml_first_child(child); child_of_child != NULL; child_of_child = __xml_next(child_of_child)) {
	/* only update the first one */
	if(can_update) {
	break;
	}
	can_update = update_xml_child(child_of_child, to_update);
	}

	return can_update;
	}


	int
	find_xml_children(xmlNode *children, xmlNode root,
	const char tag, const char field, const char *value,
	gboolean search_matches)
	{
	int match_found = 0;

	CRM_CHECK(root != NULL, return FALSE);
	CRM_CHECK(children != NULL, return FALSE);

	if(tag != NULL && safe_str_neq(tag, crm_element_name(root))) {

	} else if(value != NULL
	&& safe_str_neq(value, crm_element_value(root, field))) {

	} else {
	if(*children == NULL) {
	*children = create_xml_node(NULL, __FUNCTION__);
	}
	add_node_copy(*children, root);
	match_found = 1;
	}

	if(search_matches \|\| match_found == 0) {
	xmlNode *child = NULL;
	for(child = __xml_first_child(root); child != NULL; child = __xml_next(child)) {
	match_found += find_xml_children(
	children, child, tag, field, value,
	search_matches);
	}
	}

	return match_found;
	}

	gboolean
	replace_xml_child(xmlNode parent, xmlNode child, xmlNode *update, gboolean delete_only)
	{
	gboolean can_delete = FALSE;
	xmlNode *child_of_child = NULL;

	const char *up_id = NULL;
	const char *child_id = NULL;
	const char *right_val = NULL;

	CRM_CHECK(child != NULL, return FALSE);
	CRM_CHECK(update != NULL, return FALSE);

	up_id = ID(update);
	child_id = ID(child);

	if(up_id == NULL \|\| safe_str_eq(child_id, up_id)) {
	can_delete = TRUE;
	}
	if(safe_str_neq(crm_element_name(update), crm_element_name(child))) {
	can_delete = FALSE;
	}
	if(can_delete && delete_only) {
	xmlAttrPtr pIter = NULL;
	for(pIter = crm_first_attr(update); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	right_val = crm_element_value(child, p_name);
	if(safe_str_neq(p_value, right_val)) {
	can_delete = FALSE;
	}
	}
	}

	if(can_delete && parent != NULL) {
	crm_log_xml_trace(child, "Delete match found...");
	if(delete_only \|\| update == NULL) {
	free_xml_from_parent(NULL, child);

	} else {
	xmlNode *tmp = copy_xml(update);
	xmlDoc *doc = tmp->doc;
	xmlNode *old = xmlReplaceNode(child, tmp);
	free_xml_from_parent(NULL, old);
	xmlDocSetRootElement(doc, NULL);
	xmlFreeDoc(doc);
	}
	child = NULL;
	return TRUE;

	} else if(can_delete) {
	crm_log_xml_debug(child, "Cannot delete the search root");
	can_delete = FALSE;
	}

	child_of_child = __xml_first_child(child);
	while(child_of_child) {
	xmlNode *next = __xml_next(child_of_child);
	can_delete = replace_xml_child(child, child_of_child, update, delete_only);

	/* only delete the first one */
	if(can_delete) {
	child_of_child = NULL;
	} else {
	child_of_child = next;
	}
	}

	return can_delete;
	}

	void
	hash2nvpair(gpointer key, gpointer value, gpointer user_data)
	{
	const char *name = key;
	const char *s_value = value;

	xmlNode *xml_node = user_data;
	xmlNode *xml_child = create_xml_node(xml_node, XML_CIB_TAG_NVPAIR);

	crm_xml_add(xml_child, XML_ATTR_ID, name);
	crm_xml_add(xml_child, XML_NVPAIR_ATTR_NAME, name);
	crm_xml_add(xml_child, XML_NVPAIR_ATTR_VALUE, s_value);

	crm_trace("dumped: name=%s value=%s", name, s_value);
	}

	void
	hash2smartfield(gpointer key, gpointer value, gpointer user_data)
	{
	const char *name = key;
	const char *s_value = value;

	xmlNode *xml_node = user_data;

	if(isdigit(name[0])) {
	xmlNode *tmp = create_xml_node(xml_node, XML_TAG_PARAM);
	crm_xml_add(tmp, XML_NVPAIR_ATTR_NAME, name);
	crm_xml_add(tmp, XML_NVPAIR_ATTR_VALUE, s_value);

	} else if(crm_element_value(xml_node, name) == NULL) {
	crm_xml_add(xml_node, name, s_value);
	crm_trace("dumped: %s=%s", name, s_value);

	} else {
	crm_trace("duplicate: %s=%s", name, s_value);
	}
	}

	void
	hash2field(gpointer key, gpointer value, gpointer user_data)
	{
	const char *name = key;
	const char *s_value = value;

	xmlNode *xml_node = user_data;

	if(crm_element_value(xml_node, name) == NULL) {
	crm_xml_add(xml_node, name, s_value);
	crm_trace("dumped: %s=%s", name, s_value);

	} else {
	crm_trace("duplicate: %s=%s", name, s_value);
	}
	}

	void
	hash2metafield(gpointer key, gpointer value, gpointer user_data)
	{
	char *crm_name = NULL;

	if(key == NULL \|\| value == NULL) {
	return;
	} else if(((char*)key)[0] == '#') {
	return;
	} else if(strstr(key, ":")) {
	return;
	}

	crm_name = crm_meta_name(key);
	hash2field(crm_name, value, user_data);
	free(crm_name);
	}


	GHashTable *
	xml2list(xmlNode *parent)
	{
	xmlNode *child = NULL;
	xmlAttrPtr pIter = NULL;
	xmlNode *nvpair_list = NULL;
	GHashTable *nvpair_hash = g_hash_table_new_full(
	crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	CRM_CHECK(parent != NULL, return nvpair_hash);

	nvpair_list = find_xml_node(parent, XML_TAG_ATTRS, FALSE);
	if(nvpair_list == NULL) {
	crm_trace("No attributes in %s",
	crm_element_name(parent));
	crm_log_xml_trace(
	parent,"No attributes for resource op");
	}

	crm_log_xml_trace(nvpair_list, "Unpacking");

	for(pIter = crm_first_attr(nvpair_list); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	crm_trace("Added %s=%s", p_name, p_value);

	g_hash_table_insert(nvpair_hash, strdup(p_name), strdup(p_value));
	}

	for(child = __xml_first_child(nvpair_list); child != NULL; child = __xml_next(child)) {
	if(crm_str_eq((const char *)child->name, XML_TAG_PARAM, TRUE)) {
	const char *key = crm_element_value(child, XML_NVPAIR_ATTR_NAME);
	const char *value = crm_element_value(child, XML_NVPAIR_ATTR_VALUE);
	crm_trace("Added %s=%s", key, value);
	if(key != NULL && value != NULL) {
	g_hash_table_insert(nvpair_hash, strdup(key), strdup(value));
	}
	}
	}

	return nvpair_hash;
	}


	typedef struct name_value_s
	{
	const char *name;
	const void *value;
	} name_value_t;

	static gint
	sort_pairs(gconstpointer a, gconstpointer b)
	{
	int rc = 0;
	const name_value_t *pair_a = a;
	const name_value_t *pair_b = b;

	CRM_ASSERT(a != NULL);
	CRM_ASSERT(pair_a->name != NULL);

	CRM_ASSERT(b != NULL);
	CRM_ASSERT(pair_b->name != NULL);

	rc = strcmp(pair_a->name, pair_b->name);
	if(rc < 0) {
	return -1;
	} else if(rc > 0) {
	return 1;
	}
	return 0;
	}

	static void
	dump_pair(gpointer data, gpointer user_data)
	{
	name_value_t *pair = data;
	xmlNode *parent = user_data;
	crm_xml_add(parent, pair->name, pair->value);
	}

	xmlNode *
	sorted_xml(xmlNode input, xmlNode parent, gboolean recursive)
	{
	xmlNode *child = NULL;
	GListPtr sorted = NULL;
	GListPtr unsorted = NULL;
	name_value_t *pair = NULL;
	xmlNode *result = NULL;
	const char *name = NULL;
	xmlAttrPtr pIter = NULL;

	CRM_CHECK(input != NULL, return NULL);

	name = crm_element_name(input);
	CRM_CHECK(name != NULL, return NULL);

	result = create_xml_node(parent, name);

	for(pIter = crm_first_attr(input); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	pair = calloc(1, sizeof(name_value_t));
	pair->name = p_name;
	pair->value = p_value;
	unsorted = g_list_prepend(unsorted, pair);
	pair = NULL;
	}

	sorted = g_list_sort(unsorted, sort_pairs);
	g_list_foreach(sorted, dump_pair, result);
	g_list_free_full(sorted, free);

	for(child = __xml_first_child(input); child != NULL; child = __xml_next(child)) {
	if(recursive) {
	sorted_xml(child, result, recursive);
	} else {
	add_node_copy(result, child);
	}
	}

	return result;
	}

	static void
	filter_xml(xmlNode data, filter_t filter, int filter_len, gboolean recursive)
	{
	int lpc = 0;
	xmlNode *child = NULL;

	for(lpc = 0; lpc < filter_len; lpc++) {
	xml_remove_prop(data, filter[lpc].string);
	}

	if(recursive == FALSE \|\| filter_len == 0) {
	return;
	}

	for(child = __xml_first_child(data); child != NULL; child = __xml_next(child)) {
	filter_xml(child, filter, filter_len, recursive);
	}
	}

	/* "c048eae664dba840e1d2060f00299e9d" */
	static char *
	calculate_xml_digest_v1(xmlNode *input, gboolean sort, gboolean do_filter)
	{
	char *digest = NULL;
	xmlNode *copy = NULL;
	char *buffer = NULL;

	if(sort \|\| do_filter) {
	copy = sorted_xml(input, NULL, TRUE);
	input = copy;
	}

	if(do_filter) {
	filter_xml(input, filter, DIMOF(filter), TRUE);
	}

	buffer = dump_xml(input, FALSE, TRUE);
	CRM_CHECK(buffer != NULL && strlen(buffer) > 0, free_xml(copy); free(buffer); return NULL);

	digest = crm_md5sum(buffer);
	crm_trace("Digest %s: %s\n", digest, buffer);
	crm_log_xml_trace(copy, "digest:source");

	free(buffer);
	free_xml(copy);
	return digest;
	}

	static char *
	calculate_xml_digest_v2(xmlNode *input, gboolean do_filter)
	{
	char *digest = NULL;

	int buffer_len = 0;
	int filter_size = DIMOF(filter);

	xmlDoc *doc = NULL;
	xmlNode *copy = NULL;
	xmlBuffer *xml_buffer = NULL;
	static struct qb_log_callsite *digest_cs = NULL;

	if(do_filter && BEST_EFFORT_STATUS) {
	/* Exclude the status calculation from the digest
	*
	* This doesn't mean it wont be sync'd, we just wont be paranoid
	* about it being an _exact_ copy
	*
	* We don't need it to be exact, since we throw it away and regenerate
	* from our peers whenever a new DC is elected anyway
	*
	* Importantly, this reduces the amount of XML to copy+export as
	* well as the amount of data for MD5 needs to operate on
	*/
	xmlNode *child = NULL;
	xmlAttrPtr pIter = NULL;
	copy = create_xml_node(NULL, XML_TAG_CIB);
	for(pIter = crm_first_attr(input); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	xmlSetProp(copy, (const xmlChar)p_name, (const xmlChar)p_value);
	}

	xml_remove_prop(copy, XML_ATTR_ORIGIN);
	xml_remove_prop(copy, XML_CIB_ATTR_WRITTEN);

	/* We just did all the filtering */

	for(child = __xml_first_child(input); child != NULL; child = __xml_next(child)) {
	if(safe_str_neq(crm_element_name(child), XML_CIB_TAG_STATUS)) {
	add_node_copy(copy, child);
	}
	}

	} else if(do_filter) {
	copy = copy_xml(input);
	filter_xml(copy, filter, filter_size, TRUE);
	input = copy;
	}

	doc = getDocPtr(input);
	xml_buffer = xmlBufferCreate();

	CRM_ASSERT(xml_buffer != NULL);
	CRM_CHECK(doc != NULL, return NULL); /* doc will only be NULL if an_xml_node is */

	buffer_len = xmlNodeDump(xml_buffer, doc, input, 0, FALSE);
	CRM_CHECK(xml_buffer->content != NULL && buffer_len > 0, goto done);

	digest = crm_md5sum((char *)xml_buffer->content);
	crm_trace("Digest %s\n", digest);

	if(digest_cs == NULL) {
	qb_log_callsite_get(__func__, __FILE__, "xml-blog", LOG_TRACE, __LINE__, 0);
	}
	if (digest_cs && digest_cs->targets) {
	FILE *st = NULL;
	char *trace_file = crm_concat("/tmp/cib-digest", digest, '-');
	crm_trace("Saving %s.%s.%s to %s",
	crm_element_value(input, XML_ATTR_GENERATION_ADMIN),
	crm_element_value(input, XML_ATTR_GENERATION),
	crm_element_value(input, XML_ATTR_NUMUPDATES),
	trace_file);
	st = fopen(trace_file, "w");
	if(st) {
	fprintf(st, "%s", xml_buffer->content);
	/* fflush(st); */
	/* fsync(fileno(st)); */
	fclose(st);
	}
	free(trace_file);
	}

	done:
	xmlBufferFree(xml_buffer);
	free_xml(copy);

	return digest;
	}

	char *
	calculate_on_disk_digest(xmlNode *input)
	{
	/* Always use the v1 format for on-disk digests
	* a) its a compatability nightmare
	* b) we only use this once at startup, all other
	* invocations are in a separate child process
	*/
	return calculate_xml_digest_v1(input, FALSE, FALSE);
	}

	char *
	calculate_operation_digest(xmlNode input, const char version)
	{
	/* We still need the sorting for parameter digests */
	return calculate_xml_digest_v1(input, TRUE, FALSE);
	}

	char *
	calculate_xml_digest(xmlNode *input, gboolean sort, gboolean do_filter)
	{
	return calculate_xml_digest_v1(input, sort, do_filter);
	}

	char *
	calculate_xml_versioned_digest(xmlNode input, gboolean sort, gboolean do_filter, const char version)
	{
	/*
	* The sorting associated with v1 digest creation accounted for 23% of
	* the CIB's CPU usage on the server. v2 drops this.
	*
	* The filtering accounts for an additional 2.5% and we may want to
	* remove it in future.
	*
	* v2 also uses the xmlBuffer contents directly to avoid additional copying
	*/
	if(version == NULL \|\| compare_version("3.0.5", version) > 0) {
	crm_trace("Using v1 digest algorithm for %s", crm_str(version));
	return calculate_xml_digest_v1(input, sort, do_filter);
	}
	crm_trace("Using v2 digest algorithm for %s", crm_str(version));
	return calculate_xml_digest_v2(input, do_filter);
	}

	static gboolean
	validate_with_dtd(
	xmlDocPtr doc, gboolean to_logs, const char *dtd_file)
	{
	gboolean valid = TRUE;

	xmlDtdPtr dtd = NULL;
	xmlValidCtxtPtr cvp = NULL;

	CRM_CHECK(doc != NULL, return FALSE);
	CRM_CHECK(dtd_file != NULL, return FALSE);

	dtd = xmlParseDTD(NULL, (const xmlChar *)dtd_file);
	CRM_CHECK(dtd != NULL, crm_err("Could not find/parse %s", dtd_file); goto cleanup);

	cvp = xmlNewValidCtxt();
	CRM_CHECK(cvp != NULL, goto cleanup);

	if(to_logs) {
	cvp->userData = (void *) LOG_ERR;
	cvp->error = (xmlValidityErrorFunc) xml_log;
	cvp->warning = (xmlValidityWarningFunc) xml_log;
	} else {
	cvp->userData = (void *) stderr;
	cvp->error = (xmlValidityErrorFunc) fprintf;
	cvp->warning = (xmlValidityWarningFunc) fprintf;
	}

	if (!xmlValidateDtd(cvp, doc, dtd)) {
	valid = FALSE;
	}

	cleanup:
	if(cvp) {
	xmlFreeValidCtxt(cvp);
	}
	if(dtd) {
	xmlFreeDtd(dtd);
	}

	return valid;
	}

	xmlNode first_named_child(xmlNode parent, const char *name)
	{
	xmlNode *match = NULL;
	for(match = __xml_first_child(parent); match != NULL; match = __xml_next(match)) {
	/*
	* name == NULL gives first child regardless of name; this is
	* semantically incorrect in this funciton, but may be necessary
	* due to prior use of xml_child_iter_filter
	*/
	if(name == NULL \|\| crm_str_eq((const char*)match->name, name, TRUE)) {
	return match;
	}
	}
	return NULL;
	}

	#if 0
	static void relaxng_invalid_stderr(void * userData, xmlErrorPtr error)
	{
	/*
	Structure xmlError
	struct _xmlError {
	int domain : What part of the library raised this er
	int code : The error code, e.g. an xmlParserError
	char * message : human-readable informative error messag
	xmlErrorLevel level : how consequent is the error
	char * file : the filename
	int line : the line number if available
	char * str1 : extra string information
	char * str2 : extra string information
	char * str3 : extra string information
	int int1 : extra number information
	int int2 : column number of the error or 0 if N/A
	void * ctxt : the parser context if available
	void * node : the node in the tree
	}
	*/
	crm_err("Structured error: line=%d, level=%d %s",
	error->line, error->level, error->message);
	}
	#endif

	static gboolean
	validate_with_relaxng(
	xmlDocPtr doc, gboolean to_logs, const char relaxng_file, relaxng_ctx_cache_t *cached_ctx)
	{
	int rc = 0;
	gboolean valid = TRUE;
	relaxng_ctx_cache_t *ctx = NULL;

	CRM_CHECK(doc != NULL, return FALSE);
	CRM_CHECK(relaxng_file != NULL, return FALSE);


	if(cached_ctx && *cached_ctx) {
	ctx = *cached_ctx;

	} else {
	crm_info("Creating RNG parser context");
	ctx = calloc(1, sizeof(relaxng_ctx_cache_t));

	xmlLoadExtDtdDefaultValue = 1;
	ctx->parser = xmlRelaxNGNewParserCtxt(relaxng_file);
	CRM_CHECK(ctx->parser != NULL, goto cleanup);

	if(to_logs) {
	xmlRelaxNGSetParserErrors(ctx->parser,
	(xmlRelaxNGValidityErrorFunc) xml_log,
	(xmlRelaxNGValidityWarningFunc) xml_log,
	GUINT_TO_POINTER(LOG_ERR));
	} else {
	xmlRelaxNGSetParserErrors(ctx->parser,
	(xmlRelaxNGValidityErrorFunc) fprintf,
	(xmlRelaxNGValidityWarningFunc) fprintf,
	stderr);
	}

	ctx->rng = xmlRelaxNGParse(ctx->parser);
	CRM_CHECK(ctx->rng != NULL, crm_err("Could not find/parse %s", relaxng_file); goto cleanup);

	ctx->valid = xmlRelaxNGNewValidCtxt(ctx->rng);
	CRM_CHECK(ctx->valid != NULL, goto cleanup);

	if(to_logs) {
	xmlRelaxNGSetValidErrors(ctx->valid,
	(xmlRelaxNGValidityErrorFunc) xml_log,
	(xmlRelaxNGValidityWarningFunc) xml_log,
	GUINT_TO_POINTER(LOG_ERR));
	} else {
	xmlRelaxNGSetValidErrors(ctx->valid,
	(xmlRelaxNGValidityErrorFunc) fprintf,
	(xmlRelaxNGValidityWarningFunc) fprintf,
	stderr);
	}
	}

	/* xmlRelaxNGSetValidStructuredErrors( */
	/* valid, relaxng_invalid_stderr, valid); */

	xmlLineNumbersDefault(1);
	rc = xmlRelaxNGValidateDoc(ctx->valid, doc);
	if (rc > 0) {
	valid = FALSE;

	} else if (rc < 0) {
	crm_err("Internal libxml error during validation\n");
	}

	cleanup:

	if(cached_ctx) {
	*cached_ctx = ctx;

	} else {
	if(ctx->parser != NULL) {
	xmlRelaxNGFreeParserCtxt(ctx->parser);
	}
	if(ctx->valid != NULL) {
	xmlRelaxNGFreeValidCtxt(ctx->valid);
	}
	if (ctx->rng != NULL) {
	xmlRelaxNGFree(ctx->rng);
	}
	free(ctx);
	}

	return valid;
	}

	void crm_xml_cleanup(void)
	{
	int lpc = 0;
	relaxng_ctx_cache_t *ctx = NULL;

	crm_info("Cleaning up memory from libxml2");
	for(; lpc < all_schemas; lpc++) {
	switch(known_schemas[lpc].type) {
	case 0:
	/* None */
	break;
	case 1:
	/* DTD - Not cached */
	break;
	case 2:
	/* RNG - Cached */
	ctx = (relaxng_ctx_cache_t *)known_schemas[lpc].cache;
	if(ctx == NULL) {
	break;
	}
	if(ctx->parser != NULL) {
	xmlRelaxNGFreeParserCtxt(ctx->parser);
	}
	if(ctx->valid != NULL) {
	xmlRelaxNGFreeValidCtxt(ctx->valid);
	}
	if (ctx->rng != NULL) {
	xmlRelaxNGFree(ctx->rng);
	}
	free(ctx);
	known_schemas[lpc].cache = NULL;
	break;
	default:
	break;
	}
	}
	xmlCleanupParser();
	}

	static gboolean validate_with(xmlNode *xml, int method, gboolean to_logs)
	{
	xmlDocPtr doc = NULL;
	gboolean valid = FALSE;
	int type = known_schemas[method].type;
	char *file = NULL;

	CRM_CHECK(xml != NULL, return FALSE);
	doc = getDocPtr(xml);
	file = get_schema_path(known_schemas[method].location);

	crm_trace("Validating with: %s (type=%d)", crm_str(file), type);
	switch(type) {
	case 0:
	valid = TRUE;
	break;
	case 1:
	valid = validate_with_dtd(doc, to_logs, file);
	break;
	case 2:
	valid = validate_with_relaxng(doc, to_logs, file, (relaxng_ctx_cache_t**)&(known_schemas[method].cache));
	break;
	default:
	crm_err("Unknown validator type: %d", type);
	break;
	}

	free(file);
	return valid;
	}

	#include <stdio.h>
	static void dump_file(const char *filename)
	{

	FILE *fp = NULL;
	int ch, line = 0;

	CRM_CHECK(filename != NULL, return);

	fp = fopen(filename, "r");
	CRM_CHECK(fp != NULL, return);

	fprintf(stderr, "%4d ", ++line);
	do {
	ch = getc(fp);
	if(ch == EOF) {
	putc('\n', stderr);
	break;
	} else if(ch == '\n') {
	fprintf(stderr, "\n%4d ", ++line);
	} else {
	putc(ch, stderr);
	}
	} while(1);

	fclose(fp);
	}

	gboolean validate_xml_verbose(xmlNode *xml_blob)
	{
	xmlDoc *doc = NULL;
	xmlNode *xml = NULL;
	gboolean rc = FALSE;

	char *filename = NULL;
	static char *template = NULL;
	if(template == NULL) {
	template = strdup(CRM_STATE_DIR"/cib-invalid.XXXXXX");
	}

	filename = mktemp(template);
	write_xml_file(xml_blob, filename, FALSE);

	dump_file(filename);

	doc = xmlParseFile(filename);
	xml = xmlDocGetRootElement(doc);
	rc = validate_xml(xml, NULL, FALSE);
	free_xml(xml);

	return rc;
	}

	gboolean validate_xml(xmlNode xml_blob, const char validation, gboolean to_logs)
	{
	int lpc = 0;

	if(validation == NULL) {
	validation = crm_element_value(xml_blob, XML_ATTR_VALIDATION);
	}

	if(validation == NULL) {
	validation = crm_element_value(xml_blob, "ignore-dtd");
	if(crm_is_true(validation)) {
	validation = "none";
	} else {
	validation = "pacemaker-1.0";
	}
	}

	if(safe_str_eq(validation, "none")) {
	return TRUE;
	}

	for(; lpc < all_schemas; lpc++) {
	if(safe_str_eq(validation, known_schemas[lpc].name)) {
	return validate_with(xml_blob, lpc, to_logs);
	}
	}

	crm_err("Unknown validator: %s", validation);
	return FALSE;
	}

	#if HAVE_LIBXSLT
	static xmlNode apply_transformation(xmlNode xml, const char *transform)
	{
	char *xform = NULL;
	xmlNode *out = NULL;
	xmlDocPtr res = NULL;
	xmlDocPtr doc = NULL;
	xsltStylesheet *xslt = NULL;

	CRM_CHECK(xml != NULL, return FALSE);
	doc = getDocPtr(xml);
	xform = get_schema_path(transform);

	xmlLoadExtDtdDefaultValue = 1;
	xmlSubstituteEntitiesDefault(1);

	xslt = xsltParseStylesheetFile((const xmlChar *)xform);
	CRM_CHECK(xslt != NULL, goto cleanup);

	res = xsltApplyStylesheet(xslt, doc, NULL);
	CRM_CHECK(res != NULL, goto cleanup);

	out = xmlDocGetRootElement(res);

	cleanup:
	if(xslt) {
	xsltFreeStylesheet(xslt);
	}

	xsltCleanupGlobals();
	xmlCleanupParser();
	free(xform);

	return out;
	}
	#endif

	const char *get_schema_name(int version)
	{
	if(version < 0 \|\| version >= all_schemas) {
	return "unknown";
	}
	return known_schemas[version].name;
	}


	int get_schema_version(const char *name)
	{
	int lpc = 0;
	for(; lpc < all_schemas; lpc++) {
	if(safe_str_eq(name, known_schemas[lpc].name)) {
	return lpc;
	}
	}
	return -1;
	}

	/* set which validation to use */
	#include <crm/cib.h>
	int update_validation(
	xmlNode *xml_blob, int best, gboolean transform, gboolean to_logs)
	{
	xmlNode *xml = NULL;
	char *value = NULL;
	int lpc = 0, match = -1, rc = pcmk_ok;

	CRM_CHECK(best != NULL, return -EINVAL);
	CRM_CHECK(xml_blob != NULL, return -EINVAL);
	CRM_CHECK(*xml_blob != NULL, return -EINVAL);

	*best = 0;
	xml = *xml_blob;
	value = crm_element_value_copy(xml, XML_ATTR_VALIDATION);

	if(value != NULL) {
	match = get_schema_version(value);

	lpc = match;
	if(lpc >= 0 && transform == FALSE) {
	lpc++;

	} else if(lpc < 0) {
	crm_debug("Unknown validation type");
	lpc = 0;
	}
	}

	if(match >= max_schemas) {
	/* nothing to do */
	free(value);
	*best = match;
	return pcmk_ok;
	}

	for(; lpc < max_schemas; lpc++) {
	gboolean valid = TRUE;
	crm_debug("Testing '%s' validation", known_schemas[lpc].name?known_schemas[lpc].name:"<unset>");
	valid = validate_with(xml, lpc, to_logs);

	if(valid) {
	*best = lpc;
	}

	if(valid && transform) {
	xmlNode *upgrade = NULL;
	int next = known_schemas[lpc].after_transform;
	if(next <= 0) {
	next = lpc+1;
	}

	crm_notice("Upgrading %s-style configuration to %s with %s",
	known_schemas[lpc].name, known_schemas[next].name, known_schemas[lpc].transform?known_schemas[lpc].transform:"no-op");

	if(known_schemas[lpc].transform == NULL) {
	if(validate_with(xml, next, to_logs)) {
	crm_debug("Configuration valid for schema: %s", known_schemas[next].name);
	lpc = next; *best = next;
	rc = pcmk_ok;

	} else {
	crm_info("Configuration not valid for schema: %s", known_schemas[next].name);
	}

	} else {
	#if HAVE_LIBXSLT
	upgrade = apply_transformation(xml, known_schemas[lpc].transform);
	#endif
	if(upgrade == NULL) {
	crm_err("Transformation %s failed", known_schemas[lpc].transform);
	rc = -pcmk_err_transform_failed;

	} else if(validate_with(upgrade, next, to_logs)) {
	crm_info("Transformation %s successful", known_schemas[lpc].transform);
	lpc = next; *best = next;
	free_xml(xml);
	xml = upgrade;
	rc = pcmk_ok;

	} else {
	crm_err("Transformation %s did not produce a valid configuration", known_schemas[lpc].transform);
	crm_log_xml_info(upgrade, "transform:bad");
	free_xml(upgrade);
	rc = -pcmk_err_dtd_validation;
	}
	}
	}
	}

	if(*best > match) {
	crm_notice("Upgraded from %s to %s validation", value?value:"<none>", known_schemas[*best].name);
	crm_xml_add(xml, XML_ATTR_VALIDATION, known_schemas[*best].name);
	}

	*xml_blob = xml;
	free(value);
	return rc;
	}

	xmlNode *
	getXpathResult(xmlXPathObjectPtr xpathObj, int index)
	{
	xmlNode *match = NULL;
	CRM_CHECK(index >= 0, return NULL);
	CRM_CHECK(xpathObj != NULL, return NULL);

	if(index >= xpathObj->nodesetval->nodeNr) {
	crm_err("Requested index %d of only %d items", index, xpathObj->nodesetval->nodeNr);
	return NULL;
	}

	match = xpathObj->nodesetval->nodeTab[index];
	CRM_CHECK(match != NULL, return NULL);

	/*
	* From xpath2.c
	*
	* All the elements returned by an XPath query are pointers to
	* elements from the tree except namespace nodes where the XPath
	* semantic is different from the implementation in libxml2 tree.
	* As a result when a returned node set is freed when
	* xmlXPathFreeObject() is called, that routine must check the
	* element type. But node from the returned set may have been removed
	* by xmlNodeSetContent() resulting in access to freed data.
	* This can be exercised by running
	* valgrind xpath2 test3.xml '//discarded' discarded
	* There is 2 ways around it:
	* - make a copy of the pointers to the nodes from the result set
	* then call xmlXPathFreeObject() and then modify the nodes
	* or
	* - remove the reference to the modified nodes from the node set
	* as they are processed, if they are not namespace nodes.
	*/
	if (xpathObj->nodesetval->nodeTab[index]->type != XML_NAMESPACE_DECL) {
	xpathObj->nodesetval->nodeTab[index] = NULL;
	}

	if(match->type == XML_DOCUMENT_NODE) {
	/* Will happen if section = '/' */
	match = match->children;

	} else if(match->type != XML_ELEMENT_NODE
	&& match->parent
	&& match->parent->type == XML_ELEMENT_NODE) {
	/* reurning the parent instead */
	match = match->parent;

	} else if(match->type != XML_ELEMENT_NODE) {
	/* We only support searching nodes */
	crm_err("We only support %d not %d", XML_ELEMENT_NODE, match->type);
	match = NULL;
	}
	return match;
	}

	/* the caller needs to check if the result contains a xmlDocPtr or xmlNodePtr */
	xmlXPathObjectPtr
	xpath_search(xmlNode xml_top, const char path)
	{
	xmlDocPtr doc = NULL;
	xmlXPathObjectPtr xpathObj = NULL;
	xmlXPathContextPtr xpathCtx = NULL;
	const xmlChar xpathExpr = (const xmlChar )path;

	CRM_CHECK(path != NULL, return NULL);
	CRM_CHECK(xml_top != NULL, return NULL);
	CRM_CHECK(strlen(path) > 0, return NULL);

	doc = getDocPtr(xml_top);

	crm_trace("Evaluating: %s", path);
	xpathCtx = xmlXPathNewContext(doc);
	CRM_ASSERT(xpathCtx != NULL);

	xpathObj = xmlXPathEvalExpression(xpathExpr, xpathCtx);
	xmlXPathFreeContext(xpathCtx);
	return xpathObj;
	}

	gboolean
	cli_config_update(xmlNode *xml, int best_version, gboolean to_logs)
	{
	gboolean rc = TRUE;
	static int min_version = -1;
	static int max_version = -1;

	const char value = crm_element_value(xml, XML_ATTR_VALIDATION);
	int version = get_schema_version(value);

	if(min_version < 0) {
	min_version = get_schema_version(MINIMUM_SCHEMA_VERSION);
	}
	if(max_version < 0) {
	max_version = get_schema_version(LATEST_SCHEMA_VERSION);
	}

	if(version < min_version) {
	xmlNode *converted = NULL;

	converted = copy_xml(*xml);
	update_validation(&converted, &version, TRUE, to_logs);

	value = crm_element_value(converted, XML_ATTR_VALIDATION);
	if(version < min_version) {
	if(to_logs) {
	crm_config_err("Your current configuration could only be upgraded to %s... "
	"the minimum requirement is %s.\n", crm_str(value), MINIMUM_SCHEMA_VERSION);
	} else {
	fprintf(stderr, "Your current configuration could only be upgraded to %s... "
	"the minimum requirement is %s.\n", crm_str(value), MINIMUM_SCHEMA_VERSION);
	}

	free_xml(converted);
	converted = NULL;
	rc = FALSE;

	} else {
	free_xml(*xml);
	*xml = converted;

	if(version < max_version) {
	crm_config_warn("Your configuration was internally updated to %s... "
	"which is acceptable but not the most recent",
	get_schema_name(version));

	} else if(to_logs){
	crm_info("Your configuration was internally updated to the latest version (%s)",
	get_schema_name(version));
	}
	}
	} else if(version > max_version) {
	if(to_logs){
	crm_config_warn("Configuration validation is currently disabled."
	" It is highly encouraged and prevents many common cluster issues.");

	} else {
	fprintf(stderr, "Configuration validation is currently disabled."
	" It is highly encouraged and prevents many common cluster issues.\n");
	}
	}

	if(best_version) {
	*best_version = version;
	}

	return rc;
	}

	xmlNode expand_idref(xmlNode input, xmlNode *top)
	{
	const char *tag = NULL;
	const char *ref = NULL;
	xmlNode *result = input;
	char *xpath_string = NULL;

	if(result == NULL) {
	return NULL;

	} else if(top == NULL) {
	top = input;
	}

	tag = crm_element_name(result);
	ref = crm_element_value(result, XML_ATTR_IDREF);

	if(ref != NULL) {
	int xpath_max = 512, offset = 0;
	xpath_string = calloc(1, xpath_max);

	offset += snprintf(xpath_string + offset, xpath_max - offset, "//%s[@id='%s']", tag, ref);
	result = get_xpath_object(xpath_string, top, LOG_ERR);
	if(result == NULL) {
	char nodePath = (char )xmlGetNodePath(top);
	crm_err("No match for %s found in %s: Invalid configuration", xpath_string, crm_str(nodePath));
	free(nodePath);
	}
	}

	free(xpath_string);
	return result;
	}

	xmlNode*
	get_xpath_object_relative(const char xpath, xmlNode xml_obj, int error_level)
	{
	int len = 0;
	xmlNode *result = NULL;
	char *xpath_full = NULL;
	char *xpath_prefix = NULL;

	if(xml_obj == NULL \|\| xpath == NULL) {
	return NULL;
	}

	xpath_prefix = (char *)xmlGetNodePath(xml_obj);
	len += strlen(xpath_prefix);
	len += strlen(xpath);

	xpath_full = strdup(xpath_prefix);
	xpath_full = realloc(xpath_full, len+1);
	strncat(xpath_full, xpath, len);

	result = get_xpath_object(xpath_full, xml_obj, error_level);

	free(xpath_prefix);
	free(xpath_full);
	return result;
	}

	xmlNode*
	get_xpath_object(const char xpath, xmlNode xml_obj, int error_level)
	{
	xmlNode *result = NULL;
	xmlXPathObjectPtr xpathObj = NULL;
	char *nodePath = NULL;
	char *matchNodePath = NULL;

	if(xpath == NULL) {
	return xml_obj; /* or return NULL? */
	}

	xpathObj = xpath_search(xml_obj, xpath);
	nodePath = (char *)xmlGetNodePath(xml_obj);
	if(xpathObj == NULL \|\| xpathObj->nodesetval == NULL \|\| xpathObj->nodesetval->nodeNr < 1) {
	do_crm_log(error_level, "No match for %s in %s", xpath, crm_str(nodePath));
	crm_log_xml_trace(xml_obj, "Unexpected Input");

	} else if(xpathObj->nodesetval->nodeNr > 1) {
	int lpc = 0, max = xpathObj->nodesetval->nodeNr;
	do_crm_log(error_level, "Too many matches for %s in %s", xpath, crm_str(nodePath));

	for(lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);
	CRM_CHECK(match != NULL, continue);

	matchNodePath = (char *)xmlGetNodePath(match);
	do_crm_log(error_level, "%s[%d] = %s", xpath, lpc, crm_str(matchNodePath));
	free(matchNodePath);
	}
	crm_log_xml_trace(xml_obj, "Bad Input");

	} else {
	result = getXpathResult(xpathObj, 0);
	}

	if(xpathObj) {
	xmlXPathFreeObject(xpathObj);
	}
	free(nodePath);

	return result;
	}

	const char *
	crm_element_value(xmlNode data, const char name)
	{
	xmlAttr *attr = NULL;

	if(data == NULL) {
	crm_err("Couldn't find %s in NULL", name?name:"<null>");
	return NULL;

	} else if(name == NULL) {
	crm_err("Couldn't find NULL in %s", crm_element_name(data));
	return NULL;
	}

	attr = xmlHasProp(data, (const xmlChar*)name);
	if(attr == NULL \|\| attr->children == NULL) {
	return NULL;
	}
	return (const char*)attr->children->content;
	}

	diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c
	index 254c7cf659..833e1e7c17 100644
	--- a/lib/fencing/st_client.c
	+++ b/lib/fencing/st_client.c
	@@ -1,1964 +1,2008 @@
	/*
	* Copyright (c) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/
	#include <crm_internal.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <ctype.h>

	#include <sys/stat.h>
	#include <sys/types.h>
	#include <sys/wait.h>

	#include <glib.h>
	#include <dirent.h>
	#include <libgen.h> /* Add it for compiling on OSX */

	#include <crm/crm.h>
	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#ifdef HAVE_STONITH_STONITH_H
	# include <stonith/stonith.h>
	# define LHA_STONITH_LIBRARY "libstonith.so.1"
	static void *lha_agents_lib = NULL;
	#endif

	#include <crm/common/mainloop.h>


	CRM_TRACE_INIT_DATA(stonith);

	typedef struct stonith_private_s {
	char *token;
	crm_ipc_t *ipc;
	mainloop_io_t *source;
	GHashTable *stonith_op_callback_table;
	GList *notify_list;

	void (op_callback) (stonith_t st, const xmlNode * msg, int call, int rc, xmlNode * output,
	void *userdata);

	} stonith_private_t;

	typedef struct stonith_notify_client_s {
	const char *event;
	const char obj_id; / implement one day */
	const char obj_type; / implement one day */
	void (notify) (stonith_t st, stonith_event_t *e);

	} stonith_notify_client_t;

	typedef struct stonith_callback_client_s {
	void (callback) (stonith_t st, const xmlNode * msg, int call, int rc, xmlNode * output,
	void *userdata);
	const char *id;
	void *user_data;
	gboolean only_success;
	struct timer_rec_s *timer;

	} stonith_callback_client_t;

	struct notify_blob_s {
	stonith_t *stonith;
	xmlNode *xml;
	};

	struct timer_rec_s {
	int call_id;
	int timeout;
	guint ref;
	stonith_t *stonith;
	};

	typedef int (stonith_op_t) (const char , int, const char , xmlNode ,
	xmlNode , xmlNode , xmlNode , xmlNode );

	static const char META_TEMPLATE[] =
	"<?xml version=\"1.0\"?>\n"
	"<!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n"
	"<resource-agent name=\"%s\">\n"
	" <version>1.0</version>\n"
	" <longdesc lang=\"en\">\n"
	"%s\n"
	" </longdesc>\n"
	" <shortdesc lang=\"en\">%s</shortdesc>\n"
	"%s\n"
	" <actions>\n"
	" <action name=\"start\" timeout=\"20\" />\n"
	" <action name=\"stop\" timeout=\"15\" />\n"
	" <action name=\"status\" timeout=\"20\" />\n"
	" <action name=\"monitor\" timeout=\"20\" interval=\"3600\"/>\n"
	" <action name=\"meta-data\" timeout=\"15\" />\n"
	" </actions>\n"
	" <special tag=\"heartbeat\">\n"
	" <version>2.0</version>\n" " </special>\n" "</resource-agent>\n";

	bool stonith_dispatch(stonith_t * st);
	int stonith_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata);
	void stonith_perform_callback(stonith_t * stonith, xmlNode * msg, int call_id, int rc);
	xmlNode stonith_create_op(int call_id, const char token, const char op, xmlNode data,
	int call_options);
	int stonith_send_command(stonith_t * stonith, const char op, xmlNode data,
	xmlNode ** output_data, int call_options, int timeout);

	static void stonith_connection_destroy(gpointer user_data);
	static void stonith_send_notification(gpointer data, gpointer user_data);

	static void
	stonith_connection_destroy(gpointer user_data)
	{
	stonith_t *stonith = user_data;
	stonith_private_t *native = NULL;
	struct notify_blob_s blob;

	crm_trace("Sending destroyed notification");
	blob.stonith = stonith;
	blob.xml = create_xml_node(NULL, "notify");

	native = stonith->private;
	native->ipc = NULL;
	native->source = NULL;

	stonith->state = stonith_disconnected;
	crm_xml_add(blob.xml, F_TYPE, T_STONITH_NOTIFY);
	crm_xml_add(blob.xml, F_SUBTYPE, T_STONITH_NOTIFY_DISCONNECT);

	g_list_foreach(native->notify_list, stonith_send_notification, &blob);
	free_xml(blob.xml);
	}

	xmlNode *
	create_device_registration_xml(const char id, const char namespace, const char *agent,
	stonith_key_value_t * params)
	{
	xmlNode *data = create_xml_node(NULL, F_STONITH_DEVICE);
	xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);

	crm_xml_add(data, XML_ATTR_ID, id);
	crm_xml_add(data, "origin", __FUNCTION__);
	crm_xml_add(data, "agent", agent);
	crm_xml_add(data, "namespace", namespace);

	for (; params; params = params->next) {
	hash2field((gpointer) params->key, (gpointer) params->value, args);
	}

	return data;
	}

	static int
	stonith_api_register_device(stonith_t * st, int call_options,
	const char id, const char namespace, const char *agent,
	stonith_key_value_t * params)
	{
	int rc = 0;
	xmlNode *data = create_device_registration_xml(id, namespace, agent, params);

	rc = stonith_send_command(st, STONITH_OP_DEVICE_ADD, data, NULL, call_options, 0);
	free_xml(data);

	return rc;
	}

	static int
	stonith_api_remove_device(stonith_t * st, int call_options, const char *name)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = create_xml_node(NULL, F_STONITH_DEVICE);
	crm_xml_add(data, "origin", __FUNCTION__);
	crm_xml_add(data, XML_ATTR_ID, name);
	rc = stonith_send_command(st, STONITH_OP_DEVICE_DEL, data, NULL, call_options, 0);
	free_xml(data);

	return rc;
	}

	static int
	stonith_api_remove_level(stonith_t * st, int options, const char *node, int level)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = create_xml_node(NULL, F_STONITH_LEVEL);
	crm_xml_add(data, "origin", __FUNCTION__);
	crm_xml_add(data, F_STONITH_TARGET, node);
	crm_xml_add_int(data, XML_ATTR_ID, level);
	rc = stonith_send_command(st, STONITH_OP_LEVEL_DEL, data, NULL, options, 0);
	free_xml(data);

	return rc;
	}

	xmlNode *
	create_level_registration_xml(const char node, int level, stonith_key_value_t device_list)
	{
	xmlNode *data = create_xml_node(NULL, F_STONITH_LEVEL);

	crm_xml_add_int(data, XML_ATTR_ID, level);
	crm_xml_add(data, F_STONITH_TARGET, node);
	crm_xml_add(data, "origin", __FUNCTION__);

	for (; device_list; device_list = device_list->next) {
	xmlNode *dev = create_xml_node(data, F_STONITH_DEVICE);

	crm_xml_add(dev, XML_ATTR_ID, device_list->value);
	}

	return data;
	}

	static int
	stonith_api_register_level(stonith_t * st, int options, const char *node, int level,
	stonith_key_value_t * device_list)
	{
	int rc = 0;
	xmlNode *data = create_level_registration_xml(node, level, device_list);

	rc = stonith_send_command(st, STONITH_OP_LEVEL_ADD, data, NULL, options, 0);
	free_xml(data);

	return rc;
	}

	static void
	append_arg(gpointer key, gpointer value, gpointer user_data)
	{
	int len = 3; /* =, \n, \0 */
	int last = 0;
	char **args = user_data;

	CRM_CHECK(key != NULL, return);
	CRM_CHECK(value != NULL, return);

	if (strstr(key, "pcmk_")) {
	return;
	} else if (strstr(key, CRM_META)) {
	return;
	} else if (safe_str_eq(key, "crm_feature_set")) {
	return;
	}

	len += strlen(key);
	len += strlen(value);
	if (*args != NULL) {
	last = strlen(*args);
	}

	args = realloc(args, last + len);
	crm_trace("Appending: %s=%s", (char )key, (char )value);
	sprintf((args) + last, "%s=%s\n", (char )key, (char *)value);
	}

	static void
	append_const_arg(const char key, const char value, char **arg_list)
	{
	char *glib_sucks_key = strdup(key);
	char *glib_sucks_value = strdup(value);

	append_arg(glib_sucks_key, glib_sucks_value, arg_list);

	free(glib_sucks_value);
	free(glib_sucks_key);
	}

	static void
	append_host_specific_args(const char victim, const char map, GHashTable * params, char **arg_list)
	{
	char *name = NULL;
	int last = 0, lpc = 0, max = 0;

	if (map == NULL) {
	/* The best default there is for now... */
	crm_debug("Using default arg map: port=uname");
	append_const_arg("port", victim, arg_list);
	return;
	}

	max = strlen(map);
	crm_debug("Processing arg map: %s", map);
	for (; lpc < max + 1; lpc++) {
	if (isalpha(map[lpc])) {
	/* keep going */

	} else if (map[lpc] == '=' \|\| map[lpc] == ':') {
	free(name);
	name = calloc(1, 1 + lpc - last);
	memcpy(name, map + last, lpc - last);
	crm_debug("Got name: %s", name);
	last = lpc + 1;

	} else if (map[lpc] == 0 \|\| map[lpc] == ',' \|\| isspace(map[lpc])) {
	char *param = NULL;
	const char *value = NULL;

	param = calloc(1, 1 + lpc - last);
	memcpy(param, map + last, lpc - last);
	last = lpc + 1;

	crm_debug("Got key: %s", param);
	if (name == NULL) {
	crm_err("Misparsed '%s', found '%s' without a name", map, param);
	free(param);
	continue;
	}

	if (safe_str_eq(param, "uname")) {
	value = victim;
	} else {
	char *key = crm_meta_name(param);

	value = g_hash_table_lookup(params, key);
	free(key);
	}

	if (value) {
	crm_debug("Setting '%s'='%s' (%s) for %s", name, value, param, victim);
	append_const_arg(name, value, arg_list);

	} else {
	crm_err("No node attribute '%s' for '%s'", name, victim);
	}

	free(name);
	name = NULL;
	free(param);
	if (map[lpc] == 0) {
	break;
	}

	} else if (isspace(map[lpc])) {
	last = lpc;
	}
	}
	free(name);
	}

	static char *
	make_args(const char action, const char victim, GHashTable * device_args, GHashTable * port_map)
	{
	char buffer[512];
	char *arg_list = NULL;
	const char *value = NULL;

	CRM_CHECK(action != NULL, return NULL);

	if (device_args) {
	g_hash_table_foreach(device_args, append_arg, &arg_list);
	}

	buffer[511] = 0;
	snprintf(buffer, 511, "pcmk_%s_action", action);
	if (device_args) {
	value = g_hash_table_lookup(device_args, buffer);
	}

	if (value == NULL && device_args) {
	/* Legacy support for early 1.1 releases - Remove for 1.2 */
	snprintf(buffer, 511, "pcmk_%s_cmd", action);
	value = g_hash_table_lookup(device_args, buffer);
	}

	if (value) {
	crm_info("Substituting action '%s' for requested operation '%s'", value, action);
	action = value;
	}

	append_const_arg(STONITH_ATTR_ACTION_OP, action, &arg_list);
	if (victim && device_args) {
	const char *alias = victim;
	const char *param = g_hash_table_lookup(device_args, STONITH_ATTR_HOSTARG);

	if (port_map && g_hash_table_lookup(port_map, victim)) {
	alias = g_hash_table_lookup(port_map, victim);
	}

	/* Always supply the node's name too:
	* https://fedorahosted.org/cluster/wiki/FenceAgentAPI
	*/
	append_const_arg("nodename", victim, &arg_list);

	/* Check if we need to supply the victim in any other form */
	if (param == NULL) {
	const char *map = g_hash_table_lookup(device_args, STONITH_ATTR_ARGMAP);

	if (map == NULL) {
	param = "port";
	value = g_hash_table_lookup(device_args, param);

	} else {
	/* Legacy handling */
	append_host_specific_args(alias, map, device_args, &arg_list);
	value = map; /* Nothing more to do */
	}

	} else if (safe_str_eq(param, "none")) {
	value = param; /* Nothing more to do */

	} else {
	value = g_hash_table_lookup(device_args, param);
	}

	/* Don't overwrite explictly set values for $param */
	if (value == NULL \|\| safe_str_eq(value, "dynamic")) {
	crm_debug("Performing %s action for node '%s' as '%s=%s'", action, victim, param,
	alias);
	append_const_arg(param, alias, &arg_list);
	}
	}

	crm_trace("Calculated: %s", arg_list);
	return arg_list;
	}

	static gboolean
	st_child_term(gpointer data)
	{
	int rc = 0;
	async_command_t * track = data;
	crm_info("Child %d timed out, sending SIGTERM", track->pid);
	track->timer_sigterm = 0;
	rc = kill(track->pid, SIGTERM);
	if(rc < 0) {
	crm_perror(LOG_ERR, "Couldn't send SIGTERM to %d", track->pid);
	}
	return FALSE;
	}

	static gboolean
	st_child_kill(gpointer data)
	{
	int rc = 0;
	async_command_t * track = data;
	crm_info("Child %d timed out, sending SIGKILL", track->pid);
	track->timer_sigkill = 0;
	rc = kill(track->pid, SIGKILL);
	if(rc < 0) {
	crm_perror(LOG_ERR, "Couldn't send SIGKILL to %d", track->pid);
	}
	return FALSE;
	}

	/* Borrowed from libfence and extended */
	int
	run_stonith_agent(const char agent, const char action, const char *victim,
	GHashTable * device_args, GHashTable * port_map, int agent_result, char *output,
	async_command_t * track)
	{
	char *args = make_args(action, victim, device_args, port_map);
	int pid, status, len, rc = -EPROTO;
	int p_read_fd, p_write_fd; /* parent read/write file descriptors */
	int c_read_fd, c_write_fd; /* child read/write file descriptors */
	int fd1[2];
	int fd2[2];

	c_read_fd = c_write_fd = p_read_fd = p_write_fd = -1;

	if (args == NULL \|\| agent == NULL)
	goto fail;
	len = strlen(args);

	if (pipe(fd1))
	goto fail;
	p_read_fd = fd1[0];
	c_write_fd = fd1[1];

	if (pipe(fd2))
	goto fail;
	c_read_fd = fd2[0];
	p_write_fd = fd2[1];

	crm_debug("forking");
	pid = fork();
	if (pid < 0) {
	rc = -ECHILD;
	goto fail;
	}

	if (pid) {
	/* parent */
	int ret;
	int total = 0;

	fcntl(p_read_fd, F_SETFL, fcntl(p_read_fd, F_GETFL, 0) \| O_NONBLOCK);

	do {
	crm_debug("sending args");
	ret = write(p_write_fd, args + total, len - total);
	if (ret > 0) {
	total += ret;
	}

	} while (errno == EINTR && total < len);

	if (total != len) {
	crm_perror(LOG_ERR, "Sent %d not %d bytes", total, len);
	if (ret >= 0) {
	rc = -EREMOTEIO;
	}
	goto fail;
	}

	close(p_write_fd);

	if (track && track->done) {
	track->stdout = p_read_fd;
	g_child_watch_add(pid, track->done, track);
	crm_trace("Op: %s on %s, pid: %d, timeout: %ds", action, agent, pid, track->timeout);

	if (track->timeout) {
	track->pid = pid;
	track->timer_sigterm = g_timeout_add(1000*track->timeout, st_child_term, track);
	track->timer_sigkill = g_timeout_add(1000*(track->timeout+5), st_child_kill, track);

	} else {
	crm_err("No timeout set for stonith operation %s with device %s", action, agent);
	}

	close(c_write_fd);
	close(c_read_fd);
	free(args);
	return pid;

	} else {
	pid_t p;

	do {
	p = waitpid(pid, &status, 0);
	} while (p < 0 && errno == EINTR);

	if (p < 0) {
	crm_perror(LOG_ERR, "waitpid(%d)", pid);

	} else if (p != pid) {
	crm_err("Waited for %d, got %d", pid, p);
	}

	if (output != NULL) {
	char *local_copy;
	int lpc = 0, last = 0, more;
	len = 0;
	do {
	char buf[500];

	ret = read(p_read_fd, buf, 500);
	if (ret > 0) {
	buf[ret] = 0;
	output = realloc(output, len + ret + 1);
	sprintf((*output) + len, "%s", buf);
	crm_trace("%d: %s", ret, (*output) + len);
	len += ret;
	}

	} while (ret == 500 \|\| (ret < 0 && errno == EINTR));

	if (*output) {
	local_copy = strdup(*output);
	more = strlen(local_copy);
	for(lpc = 0; lpc < more; lpc++) {
	if(local_copy[lpc] == '\n' \|\| local_copy[lpc] == 0) {
	local_copy[lpc] = 0;
	crm_debug("%s: %s", agent, local_copy+last);
	last = lpc+1;
	}
	}
	crm_debug("%s: %s (total %d bytes)", agent, local_copy+last, more);
	free(local_copy);
	}
	}

	rc = -ECONNABORTED;
	*agent_result = -ECONNABORTED;
	if (WIFEXITED(status)) {
	crm_debug("result = %d", WEXITSTATUS(status));
	*agent_result = -WEXITSTATUS(status);
	rc = 0;

	} else if (WIFSIGNALED(status)) {
	crm_err("call %s for %s exited due to signal %d", action, agent, WTERMSIG(status));

	} else {
	crm_err("call %s for %s exited abnormally. stopped=%d, continued=%d",
	action, agent, WIFSTOPPED(status), WIFCONTINUED(status));
	}
	}

	} else {
	/* child */

	close(1);
	/* coverity[leaked_handle] False positive */
	if (dup(c_write_fd) < 0)
	goto fail;
	close(2);
	/* coverity[leaked_handle] False positive */
	if (dup(c_write_fd) < 0)
	goto fail;
	close(0);
	/* coverity[leaked_handle] False positive */
	if (dup(c_read_fd) < 0)
	goto fail;

	/* keep c_write_fd open so parent can report all errors. */
	close(c_read_fd);
	close(p_read_fd);
	close(p_write_fd);

	execlp(agent, agent, NULL);
	exit(EXIT_FAILURE);
	}

	fail:
	free(args);

	if (p_read_fd >= 0) {
	close(p_read_fd);
	}
	if (p_write_fd >= 0) {
	close(p_write_fd);
	}

	if (c_read_fd >= 0) {
	close(c_read_fd);
	}
	if (c_write_fd >= 0) {
	close(c_write_fd);
	}

	return rc;
	}

	static int
	stonith_api_device_list(stonith_t * stonith, int call_options, const char *namespace,
	stonith_key_value_t ** devices, int timeout)
	{
	int count = 0;

	if (devices == NULL) {
	crm_err("Parameter error: stonith_api_device_list");
	return -EFAULT;
	}

	/* Include Heartbeat agents */
	if (namespace == NULL \|\| safe_str_eq("heartbeat", namespace)) {
	#if HAVE_STONITH_STONITH_H
	static gboolean need_init = TRUE;

	char **entry = NULL;
	char **type_list = NULL;
	static char *(type_list_fn) (void) = NULL;
	static void (type_free_fn) (char *) = NULL;

	if(need_init) {
	need_init = FALSE;
	type_list_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_types", FALSE);
	type_free_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_free_hostlist", FALSE);
	}

	if(type_list_fn) {
	type_list = (*type_list_fn)();
	}

	for (entry = type_list; entry != NULL && *entry; ++entry) {
	crm_trace("Added: %s", *entry);
	devices = stonith_key_value_add(devices, NULL, *entry);
	count++;
	}
	if (type_list && type_free_fn) {
	(*type_free_fn)(type_list);
	}
	#else
	if(namespace != NULL) {
	return -EINVAL; /* Heartbeat agents not supported */
	}
	#endif
	}

	/* Include Red Hat agents, basically: ls -1 @sbin_dir@/fence_* */
	if (namespace == NULL \|\| safe_str_eq("redhat", namespace)) {
	struct dirent **namelist;
	int file_num = scandir(RH_STONITH_DIR, &namelist, 0, alphasort);

	if (file_num > 0) {
	struct stat prop;
	char buffer[FILENAME_MAX + 1];

	while (file_num--) {
	if ('.' == namelist[file_num]->d_name[0]) {
	free(namelist[file_num]);
	continue;

	} else if (0 != strncmp(RH_STONITH_PREFIX,
	namelist[file_num]->d_name, strlen(RH_STONITH_PREFIX))) {
	free(namelist[file_num]);
	continue;
	}

	snprintf(buffer, FILENAME_MAX, "%s/%s", RH_STONITH_DIR, namelist[file_num]->d_name);
	if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) {
	devices = stonith_key_value_add(devices, NULL, namelist[file_num]->d_name);
	count++;
	}

	free(namelist[file_num]);
	}
	free(namelist);
	}
	}

	return count;
	}

	static int
	stonith_api_device_metadata(stonith_t * stonith, int call_options, const char *agent,
	const char namespace, char *output, int timeout)
	{
	int rc = 0;
	char *buffer = NULL;
	const char *provider = get_stonith_provider(agent, namespace);

	crm_trace("looking up %s/%s metadata", agent, provider);

	/* By having this in a library, we can access it from stonith_admin
	* when neither lrmd or stonith-ng are running
	* Important for the crm shell's validations...
	*/

	if (safe_str_eq(provider, "redhat")) {

	int exec_rc = run_stonith_agent(agent, "metadata", NULL, NULL, NULL, &rc, &buffer, NULL);

	if (exec_rc < 0 \|\| rc != 0 \|\| buffer == NULL) {
	crm_debug("Query failed: %d %d: %s", exec_rc, rc, crm_str(buffer));
	free(buffer); /* Just in case */
	return -EINVAL;

	} else {

	xmlNode *xml = string2xml(buffer);
	xmlNode *actions = NULL;
	xmlXPathObject *xpathObj = NULL;

	xpathObj = xpath_search(xml, "//actions");
	if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
	actions = getXpathResult(xpathObj, 0);
	}

	/* Now fudge the metadata so that the start/stop actions appear */
	xpathObj = xpath_search(xml, "//action[@name='stop']");
	if (xpathObj == NULL \|\| xpathObj->nodesetval->nodeNr <= 0) {
	xmlNode *tmp = NULL;

	tmp = create_xml_node(actions, "action");
	crm_xml_add(tmp, "name", "stop");
	crm_xml_add(tmp, "timeout", "20s");

	tmp = create_xml_node(actions, "action");
	crm_xml_add(tmp, "name", "start");
	crm_xml_add(tmp, "timeout", "20s");
	}

	/* Now fudge the metadata so that the port isn't required in the configuration */
	xpathObj = xpath_search(xml, "//parameter[@name='port']");
	if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
	/* We'll fill this in */
	xmlNode *tmp = getXpathResult(xpathObj, 0);

	crm_xml_add(tmp, "required", "0");
	}

	free(buffer);
	buffer = dump_xml_formatted(xml);
	free_xml(xml);
	}

	} else {
	#if !HAVE_STONITH_STONITH_H
	return -EINVAL; /* Heartbeat agents not supported */
	#else
	int bufferlen = 0;

	char *xml_meta_longdesc = NULL;
	char *xml_meta_shortdesc = NULL;

	char *meta_param = NULL;
	char *meta_longdesc = NULL;
	char *meta_shortdesc = NULL;
	static const char *no_parameter_info = "<!-- no value -->";

	Stonith *stonith_obj = NULL;

	static gboolean need_init = TRUE;
	static Stonith (st_new_fn) (const char *) = NULL;
	static const char (st_info_fn) (Stonith *, int) = NULL;
	static void (st_del_fn) (Stonith ) = NULL;

	if(need_init) {
	need_init = FALSE;
	st_new_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_new", FALSE);
	st_del_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_delete", FALSE);
	st_info_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_get_info", FALSE);
	}

	if (lha_agents_lib && st_new_fn && st_del_fn && st_info_fn) {
	stonith_obj = (*st_new_fn) (agent);
	if(stonith_obj) {
	meta_longdesc = strdup((*st_info_fn)(stonith_obj, ST_DEVICEDESCR));
	if (meta_longdesc == NULL) {
	crm_warn("no long description in %s's metadata.", agent);
	meta_longdesc = strdup(no_parameter_info);
	}

	meta_shortdesc = strdup((*st_info_fn)(stonith_obj, ST_DEVICEID));
	if (meta_shortdesc == NULL) {
	crm_warn("no short description in %s's metadata.", agent);
	meta_shortdesc = strdup(no_parameter_info);
	}

	meta_param = strdup((*st_info_fn)(stonith_obj, ST_CONF_XML));
	if (meta_param == NULL) {
	crm_warn("no list of parameters in %s's metadata.", agent);
	meta_param = strdup(no_parameter_info);
	}
	(*st_del_fn)(stonith_obj);
	} else {
	return -EINVAL; /* Heartbeat agents not supported */
	}
	}

	xml_meta_longdesc =
	(char )xmlEncodeEntitiesReentrant(NULL, (const unsigned char )meta_longdesc);
	xml_meta_shortdesc =
	(char )xmlEncodeEntitiesReentrant(NULL, (const unsigned char )meta_shortdesc);

	bufferlen = strlen(META_TEMPLATE) + strlen(agent)
	+ strlen(xml_meta_longdesc) + strlen(xml_meta_shortdesc)
	+ strlen(meta_param) + 1;

	buffer = calloc(1, bufferlen);
	snprintf(buffer, bufferlen - 1, META_TEMPLATE,
	agent, xml_meta_longdesc, xml_meta_shortdesc, meta_param);

	xmlFree(xml_meta_longdesc);
	xmlFree(xml_meta_shortdesc);

	free(meta_shortdesc);
	free(meta_longdesc);
	free(meta_param);
	#endif
	}

	if (output) {
	*output = buffer;

	} else {
	free(buffer);
	}

	return rc;
	}

	static int
	stonith_api_query(stonith_t * stonith, int call_options, const char *target,
	stonith_key_value_t ** devices, int timeout)
	{
	int rc = 0, lpc = 0, max = 0;

	xmlNode *data = NULL;
	xmlNode *output = NULL;
	xmlXPathObjectPtr xpathObj = NULL;

	CRM_CHECK(devices != NULL, return -EINVAL);

	data = create_xml_node(NULL, F_STONITH_DEVICE);
	crm_xml_add(data, "origin", __FUNCTION__);
	crm_xml_add(data, F_STONITH_TARGET, target);
	rc = stonith_send_command(stonith, STONITH_OP_QUERY, data, &output, call_options, timeout);

	if (rc < 0) {
	return rc;
	}

	xpathObj = xpath_search(output, "//@agent");
	if (xpathObj) {
	max = xpathObj->nodesetval->nodeNr;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);

	CRM_CHECK(match != NULL, continue);

	crm_info("%s[%d] = %s", "//@agent", lpc, xmlGetNodePath(match));
	devices = stonith_key_value_add(devices, NULL, crm_element_value(match, XML_ATTR_ID));
	}
	}

	free_xml(output);
	free_xml(data);
	return max;
	}

	static int
	-stonith_api_call(stonith_t * stonith, int call_options, const char id, const char action,
	- const char *victim, int timeout)
	+stonith_api_call(stonith_t * stonith,
	+ int call_options,
	+ const char *id,
	+ const char *action,
	+ const char *victim,
	+ int timeout,
	+ xmlNode **output)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = create_xml_node(NULL, F_STONITH_DEVICE);
	crm_xml_add(data, "origin", __FUNCTION__);
	crm_xml_add(data, F_STONITH_DEVICE, id);
	crm_xml_add(data, F_STONITH_ACTION, action);
	crm_xml_add(data, F_STONITH_TARGET, victim);

	- rc = stonith_send_command(stonith, STONITH_OP_EXEC, data, NULL, call_options, timeout);
	+ rc = stonith_send_command(stonith, STONITH_OP_EXEC, data, output, call_options, timeout);
	free_xml(data);

	return rc;
	}

	+static int
	+stonith_api_list(stonith_t * stonith, int call_options, const char id, char *list_info, int timeout)
	+{
	+ int rc;
	+ xmlNode *output = NULL;
	+
	+ rc = stonith_api_call(stonith, call_options, id, "list", NULL, timeout, &output);
	+
	+ if (output && list_info) {
	+ const char *list_str;
	+
	+ list_str = crm_element_value(output, "st_output");
	+
	+ if (list_str) {
	+ *list_info = strdup(list_str);
	+ }
	+ }
	+
	+ if (output) {
	+ free_xml(output);
	+ }
	+
	+ return rc;
	+}
	+
	+static int
	+stonith_api_monitor(stonith_t * stonith, int call_options, const char *id, int timeout)
	+{
	+ return stonith_api_call(stonith, call_options, id, "monitor", NULL, timeout, NULL);
	+}
	+
	+static int
	+stonith_api_status(stonith_t * stonith, int call_options, const char id, const char port, int timeout)
	+{
	+ return stonith_api_call(stonith, call_options, id, "status", port, timeout, NULL);
	+}
	+
	static int
	stonith_api_fence(stonith_t * stonith, int call_options, const char node, const char action,
	int timeout)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = create_xml_node(NULL, __FUNCTION__);
	crm_xml_add(data, F_STONITH_TARGET, node);
	crm_xml_add(data, F_STONITH_ACTION, action);
	crm_xml_add_int(data, F_STONITH_TIMEOUT, timeout);

	rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout);
	free_xml(data);

	return rc;
	}

	static int
	stonith_api_confirm(stonith_t * stonith, int call_options, const char *target)
	{
	return stonith_api_fence(stonith, call_options \| st_opt_manual_ack, target, "off", 0);
	}

	static int
	stonith_api_history(stonith_t * stonith, int call_options, const char *node,
	stonith_history_t ** history, int timeout)
	{
	int rc = 0;
	xmlNode *data = NULL;
	xmlNode *output = NULL;
	stonith_history_t *last = NULL;

	*history = NULL;

	if (node) {
	data = create_xml_node(NULL, __FUNCTION__);
	crm_xml_add(data, F_STONITH_TARGET, node);
	}

	rc = stonith_send_command(stonith, STONITH_OP_FENCE_HISTORY, data, &output,
	call_options \| st_opt_sync_call, timeout);
	free_xml(data);

	if (rc == 0) {
	xmlNode *op = NULL;
	xmlNode *reply = get_xpath_object("//" F_STONITH_HISTORY_LIST, output, LOG_ERR);

	for (op = __xml_first_child(reply); op != NULL; op = __xml_next(op)) {
	stonith_history_t *kvp;

	kvp = calloc(1, sizeof(stonith_history_t));
	kvp->target = crm_element_value_copy(op, F_STONITH_TARGET);
	kvp->action = crm_element_value_copy(op, F_STONITH_ACTION);
	kvp->origin = crm_element_value_copy(op, F_STONITH_ORIGIN);
	kvp->delegate = crm_element_value_copy(op, F_STONITH_DELEGATE);
	crm_element_value_int(op, F_STONITH_DATE, &kvp->completed);
	crm_element_value_int(op, F_STONITH_STATE, &kvp->state);

	if (last) {
	last->next = kvp;
	} else {
	*history = kvp;
	}
	last = kvp;
	}
	}
	return rc;
	}

	gboolean
	is_redhat_agent(const char *agent)
	{
	int rc = 0;
	struct stat prop;
	char buffer[FILENAME_MAX + 1];

	snprintf(buffer, FILENAME_MAX, "%s/%s", RH_STONITH_DIR, agent);
	rc = stat(buffer, &prop);
	if (rc >= 0 && S_ISREG(prop.st_mode)) {
	return TRUE;
	}
	return FALSE;
	}

	const char *
	get_stonith_provider(const char agent, const char provider)
	{
	/* This function sucks */
	if (is_redhat_agent(agent)) {
	return "redhat";

	#if HAVE_STONITH_STONITH_H
	} else {
	Stonith *stonith_obj = NULL;

	static gboolean need_init = TRUE;
	static Stonith (st_new_fn) (const char *) = NULL;
	static void (st_del_fn) (Stonith ) = NULL;

	if(need_init) {
	need_init = FALSE;
	st_new_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_new", FALSE);
	st_del_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_delete", FALSE);
	}

	if (lha_agents_lib && st_new_fn && st_del_fn) {
	stonith_obj = (*st_new_fn) (agent);
	if(stonith_obj) {
	(*st_del_fn)(stonith_obj);
	return "heartbeat";
	}
	}
	#endif
	}

	crm_err("No such device: %s", agent);
	return NULL;
	}

	static gint
	stonithlib_GCompareFunc(gconstpointer a, gconstpointer b)
	{
	int rc = 0;
	const stonith_notify_client_t *a_client = a;
	const stonith_notify_client_t *b_client = b;

	CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0);
	rc = strcmp(a_client->event, b_client->event);
	if (rc == 0) {
	if (a_client->notify == NULL \|\| b_client->notify == NULL) {
	return 0;

	} else if (a_client->notify == b_client->notify) {
	return 0;

	} else if (((long)a_client->notify) < ((long)b_client->notify)) {
	crm_err("callbacks for %s are not equal: %p vs. %p",
	a_client->event, a_client->notify, b_client->notify);
	return -1;
	}
	crm_err("callbacks for %s are not equal: %p vs. %p",
	a_client->event, a_client->notify, b_client->notify);
	return 1;
	}
	return rc;
	}

	xmlNode *
	stonith_create_op(int call_id, const char token, const char op, xmlNode * data, int call_options)
	{
	xmlNode *op_msg = create_xml_node(NULL, "stonith_command");

	CRM_CHECK(op_msg != NULL, return NULL);
	CRM_CHECK(token != NULL, return NULL);

	crm_xml_add(op_msg, F_XML_TAGNAME, "stonith_command");

	crm_xml_add(op_msg, F_TYPE, T_STONITH_NG);
	crm_xml_add(op_msg, F_STONITH_CALLBACK_TOKEN, token);
	crm_xml_add(op_msg, F_STONITH_OPERATION, op);
	crm_xml_add_int(op_msg, F_STONITH_CALLID, call_id);
	crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options);
	crm_xml_add_int(op_msg, F_STONITH_CALLOPTS, call_options);

	if (data != NULL) {
	add_message_xml(op_msg, F_STONITH_CALLDATA, data);
	}

	return op_msg;
	}

	static void
	stonith_destroy_op_callback(gpointer data)
	{
	stonith_callback_client_t *blob = data;

	if (blob->timer && blob->timer->ref > 0) {
	g_source_remove(blob->timer->ref);
	}
	free(blob->timer);
	free(blob);
	}

	static int
	stonith_api_signoff(stonith_t * stonith)
	{
	stonith_private_t *native = stonith->private;

	crm_debug("Signing out of the STONITH Service");

	if (native->ipc != NULL) {
	/* If attached to mainloop and it is active, _close() will result in:
	* - the source being removed from mainloop
	* - the dnotify callback being invoked
	* Otherwise, we are at least correctly disconnecting IPC
	*/
	crm_ipc_close(native->ipc);
	crm_ipc_destroy(native->ipc);
	native->source = NULL;
	native->ipc = NULL;
	}

	stonith->state = stonith_disconnected;
	return pcmk_ok;
	}

	static int
	stonith_api_signon(stonith_t * stonith, const char name, int stonith_fd)
	{
	int rc = pcmk_ok;
	stonith_private_t *native = stonith->private;
	static struct ipc_client_callbacks st_callbacks =
	{
	.dispatch = stonith_dispatch_internal,
	.destroy = stonith_connection_destroy
	};

	crm_trace("Connecting command channel");

	stonith->state = stonith_connected_command;
	if(stonith_fd) {
	/* No mainloop */
	native->ipc = crm_ipc_new("stonith-ng", 0);

	if(native->ipc && crm_ipc_connect(native->ipc)) {
	*stonith_fd = crm_ipc_get_fd(native->ipc);

	} else if(native->ipc) {
	rc = -ENOTCONN;
	}

	} else {
	/* With mainloop */
	native->source = mainloop_add_ipc_client("stonith-ng", 0, stonith, &st_callbacks);
	native->ipc = mainloop_get_ipc_client(native->source);
	}

	if (native->ipc == NULL) {
	crm_debug("Could not connect to the Stonith API");
	rc = -ENOTCONN;
	}

	if (rc == pcmk_ok) {
	xmlNode *reply = NULL;
	xmlNode *hello = create_xml_node(NULL, "stonith_command");

	crm_xml_add(hello, F_TYPE, T_STONITH_NG);
	crm_xml_add(hello, F_STONITH_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(hello, F_STONITH_CLIENTNAME, name);
	rc = crm_ipc_send(native->ipc, hello, &reply, -1);

	if(rc < 0) {
	crm_perror(LOG_DEBUG, "Couldn't complete registration with the fencing API: %d", rc);
	rc = -ECOMM;

	} else if(reply == NULL) {
	crm_err("Did not receive registration reply");
	rc = -EPROTO;

	} else {
	const char *msg_type = crm_element_value(reply, F_STONITH_OPERATION);
	const char *tmp_ticket = crm_element_value(reply, F_STONITH_CLIENTID);

	if (safe_str_neq(msg_type, CRM_OP_REGISTER)) {
	crm_err("Invalid registration message: %s", msg_type);
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;

	} else if (tmp_ticket == NULL) {
	crm_err("No registration token provided");
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;

	} else {
	crm_trace("Obtained registration token: %s", tmp_ticket);
	native->token = strdup(tmp_ticket);
	rc = pcmk_ok;
	}
	}

	free_xml(reply);
	free_xml(hello);
	}

	if (rc == pcmk_ok) {
	#if HAVE_MSGFROMIPC_TIMEOUT
	stonith->call_timeout = MAX_IPC_DELAY;
	#endif
	crm_debug("Connection to STONITH successful");
	return pcmk_ok;
	}

	crm_debug("Connection to STONITH failed: %s", pcmk_strerror(rc));
	stonith->cmds->disconnect(stonith);
	return rc;
	}

	static int
	stonith_set_notification(stonith_t * stonith, const char *callback, int enabled)
	{
	xmlNode *notify_msg = create_xml_node(NULL, __FUNCTION__);
	stonith_private_t *native = stonith->private;

	if (stonith->state != stonith_disconnected) {
	int rc;

	crm_xml_add(notify_msg, F_STONITH_OPERATION, T_STONITH_NOTIFY);
	if (enabled) {
	crm_xml_add(notify_msg, F_STONITH_NOTIFY_ACTIVATE, callback);
	} else {
	crm_xml_add(notify_msg, F_STONITH_NOTIFY_DEACTIVATE, callback);
	}
	rc = crm_ipc_send(native->ipc, notify_msg, NULL, -1);
	if(rc < 0) {
	crm_perror(LOG_DEBUG, "Couldn't register for fencing notifications: %d", rc);
	rc = -ECOMM;
	}
	}

	free_xml(notify_msg);
	return pcmk_ok;
	}

	static int
	stonith_api_add_notification(stonith_t * stonith, const char *event,
	void (callback) (stonith_t stonith, stonith_event_t *e))
	{
	GList *list_item = NULL;
	stonith_notify_client_t *new_client = NULL;
	stonith_private_t *private = NULL;

	private = stonith->private;
	crm_trace("Adding callback for %s events (%d)", event, g_list_length(private->notify_list));

	new_client = calloc(1, sizeof(stonith_notify_client_t));
	new_client->event = event;
	new_client->notify = callback;

	list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc);

	if (list_item != NULL) {
	crm_warn("Callback already present");
	free(new_client);
	return -ENOTUNIQ;

	} else {
	private->notify_list = g_list_append(private->notify_list, new_client);

	stonith_set_notification(stonith, event, 1);

	crm_trace("Callback added (%d)", g_list_length(private->notify_list));
	}
	return pcmk_ok;
	}

	static int
	stonith_api_del_notification(stonith_t * stonith, const char *event)
	{
	GList *list_item = NULL;
	stonith_notify_client_t *new_client = NULL;
	stonith_private_t *private = NULL;

	crm_debug("Removing callback for %s events", event);

	private = stonith->private;
	new_client = calloc(1, sizeof(stonith_notify_client_t));
	new_client->event = event;
	new_client->notify = NULL;

	list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc);

	stonith_set_notification(stonith, event, 0);

	if (list_item != NULL) {
	stonith_notify_client_t *list_client = list_item->data;

	private->notify_list = g_list_remove(private->notify_list, list_client);
	free(list_client);

	crm_trace("Removed callback");

	} else {
	crm_trace("Callback not present");
	}
	free(new_client);
	return pcmk_ok;
	}

	static gboolean
	stonith_async_timeout_handler(gpointer data)
	{
	struct timer_rec_s *timer = data;

	crm_err("Async call %d timed out after %dms", timer->call_id, timer->timeout);
	stonith_perform_callback(timer->stonith, NULL, timer->call_id, -ETIME);

	/* Always return TRUE, never remove the handler
	* We do that in stonith_del_callback()
	*/
	return TRUE;
	}

	static int
	stonith_api_add_callback(stonith_t * stonith, int call_id, int timeout, bool only_success,
	void user_data, const char callback_name,
	void (callback) (stonith_t st, const xmlNode * msg, int call, int rc,
	xmlNode * output, void *userdata))
	{
	stonith_callback_client_t *blob = NULL;
	stonith_private_t *private = NULL;

	CRM_CHECK(stonith != NULL, return -EINVAL);
	CRM_CHECK(stonith->private != NULL, return -EINVAL);
	private = stonith->private;

	if (call_id == 0) {
	private->op_callback = callback;

	} else if (call_id < 0) {
	if (only_success == FALSE) {
	crm_trace("Call failed, calling %s: %s",
	callback_name, pcmk_strerror(call_id));
	callback(stonith, NULL, call_id, call_id, NULL, user_data);
	} else {
	crm_warn("STONITH call failed: %s", pcmk_strerror(call_id));
	}
	return FALSE;
	}

	blob = calloc(1, sizeof(stonith_callback_client_t));
	blob->id = callback_name;
	blob->only_success = only_success;
	blob->user_data = user_data;
	blob->callback = callback;

	if (timeout > 0) {
	struct timer_rec_s *async_timer = NULL;

	async_timer = calloc(1, sizeof(struct timer_rec_s));
	blob->timer = async_timer;

	async_timer->stonith = stonith;
	async_timer->call_id = call_id;
	/* Allow a fair bit of grace to allow the server to tell us of a timeout
	* This is only a fallback
	*/
	async_timer->timeout = (timeout + 60) * 1000;
	async_timer->ref =
	g_timeout_add(async_timer->timeout, stonith_async_timeout_handler, async_timer);
	}

	g_hash_table_insert(private->stonith_op_callback_table, GINT_TO_POINTER(call_id), blob);
	crm_trace("Added callback to %s for call %d", callback_name, call_id);

	return TRUE;
	}

	static int
	stonith_api_del_callback(stonith_t * stonith, int call_id, bool all_callbacks)
	{
	stonith_private_t *private = stonith->private;

	if (all_callbacks) {
	private->op_callback = NULL;
	g_hash_table_destroy(private->stonith_op_callback_table);
	private->stonith_op_callback_table = g_hash_table_new_full(g_direct_hash, g_direct_equal,
	NULL,
	stonith_destroy_op_callback);

	} else if (call_id == 0) {
	private->op_callback = NULL;

	} else {
	g_hash_table_remove(private->stonith_op_callback_table, GINT_TO_POINTER(call_id));
	}
	return pcmk_ok;
	}

	static void
	stonith_dump_pending_op(gpointer key, gpointer value, gpointer user_data)
	{
	int call = GPOINTER_TO_INT(key);
	stonith_callback_client_t *blob = value;

	crm_debug("Call %d (%s): pending", call, crm_str(blob->id));
	}

	void
	stonith_dump_pending_callbacks(stonith_t * stonith)
	{
	stonith_private_t *private = stonith->private;

	if (private->stonith_op_callback_table == NULL) {
	return;
	}
	return g_hash_table_foreach(private->stonith_op_callback_table, stonith_dump_pending_op, NULL);
	}

	void
	stonith_perform_callback(stonith_t * stonith, xmlNode * msg, int call_id, int rc)
	{
	xmlNode *output = NULL;
	stonith_private_t *private = NULL;
	stonith_callback_client_t *blob = NULL;
	stonith_callback_client_t local_blob;

	CRM_CHECK(stonith != NULL, return);
	CRM_CHECK(stonith->private != NULL, return);

	private = stonith->private;

	local_blob.id = NULL;
	local_blob.callback = NULL;
	local_blob.user_data = NULL;
	local_blob.only_success = FALSE;

	if (msg != NULL) {
	crm_element_value_int(msg, F_STONITH_RC, &rc);
	crm_element_value_int(msg, F_STONITH_CALLID, &call_id);
	output = get_message_xml(msg, F_STONITH_CALLDATA);
	}

	CRM_CHECK(call_id > 0, crm_log_xml_err(msg, "Bad result"));

	blob = g_hash_table_lookup(private->stonith_op_callback_table, GINT_TO_POINTER(call_id));

	if (blob != NULL) {
	local_blob = *blob;
	blob = NULL;

	stonith_api_del_callback(stonith, call_id, FALSE);

	} else {
	crm_trace("No callback found for call %d", call_id);
	local_blob.callback = NULL;
	}

	if (local_blob.callback != NULL && (rc == pcmk_ok \|\| local_blob.only_success == FALSE)) {
	crm_trace("Invoking callback %s for call %d", crm_str(local_blob.id), call_id);
	local_blob.callback(stonith, msg, call_id, rc, output, local_blob.user_data);

	} else if (private->op_callback == NULL && rc != pcmk_ok) {
	crm_warn("STONITH command failed: %s", pcmk_strerror(rc));
	crm_log_xml_debug(msg, "Failed STONITH Update");
	}

	if (private->op_callback != NULL) {
	crm_trace("Invoking global callback for call %d", call_id);
	private->op_callback(stonith, msg, call_id, rc, output, NULL);
	}
	crm_trace("OP callback activated.");
	}

	/*
	<notify t="st_notify" subt="st_device_register" st_op="st_device_register" st_rc="0" >
	<st_calldata >
	<stonith_command t="stonith-ng" st_async_id="088fb640-431a-48b9-b2fc-c4ff78d0a2d9" st_op="st_device_register" st_callid="2" st_callopt="4096" st_timeout="0" st_clientid="088fb640-431a-48b9-b2fc-c4ff78d0a2d9" st_clientname="stonith-test" >
	<st_calldata >
	<st_device_id id="test-id" origin="create_device_registration_xml" agent="fence_virsh" namespace="stonith-ng" >
	<attributes ipaddr="localhost" pcmk-portmal="some-host=pcmk-1 pcmk-3=3,4" login="root" identity_file="/root/.ssh/id_dsa" />
	</st_device_id>
	</st_calldata>
	</stonith_command>
	</st_calldata>
	</notify>

	<notify t="st_notify" subt="st_notify_fence" st_op="st_notify_fence" st_rc="0" >
	<st_calldata >
	<st_notify_fence st_rc="0" st_target="some-host" st_op="st_fence" st_delegate="test-id" st_origin="61dd7759-e229-4be7-b1f8-ef49dd14d9f0" />
	</st_calldata>
	</notify>
	*/
	static stonith_event_t *
	xml_to_event(xmlNode *msg)
	{
	stonith_event_t *event = calloc(1, sizeof(stonith_event_t));
	const char *ntype = crm_element_value(msg, F_SUBTYPE);
	char *data_addr = g_strdup_printf("//%s", ntype);
	xmlNode *data = get_xpath_object(data_addr, msg, LOG_ERR);

	crm_log_xml_trace(msg, "stonith_notify");

	crm_element_value_int(msg, F_STONITH_RC, &(event->result));

	if(safe_str_eq(ntype, T_STONITH_NOTIFY_FENCE)) {
	event->origin = crm_element_value_copy(data, F_STONITH_ORIGIN);
	event->target = crm_element_value_copy(data, F_STONITH_TARGET);
	event->operation = crm_element_value_copy(msg, F_STONITH_OPERATION);
	event->executioner = crm_element_value_copy(data, F_STONITH_DELEGATE);
	event->id = crm_element_value_copy(data, F_STONITH_REMOTE);
	}

	g_free(data_addr);
	return event;
	}

	static void
	stonith_send_notification(gpointer data, gpointer user_data)
	{
	struct notify_blob_s *blob = user_data;
	stonith_notify_client_t *entry = data;
	stonith_event_t *st_event = NULL;
	const char *event = NULL;

	if (blob->xml == NULL) {
	crm_warn("Skipping callback - NULL message");
	return;
	}

	event = crm_element_value(blob->xml, F_SUBTYPE);

	if (entry == NULL) {
	crm_warn("Skipping callback - NULL callback client");
	return;

	} else if (entry->notify == NULL) {
	crm_warn("Skipping callback - NULL callback");
	return;

	} else if (safe_str_neq(entry->event, event)) {
	crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event);
	return;
	}

	st_event = xml_to_event(blob->xml);

	crm_trace("Invoking callback for %p/%s event...", entry, event);
	entry->notify(blob->stonith, st_event);
	crm_trace("Callback invoked...");
	}

	int
	stonith_send_command(stonith_t * stonith, const char op, xmlNode data, xmlNode ** output_data,
	int call_options, int timeout)
	{
	int rc = 0;
	int reply_id = -1;

	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	stonith_private_t *native = stonith->private;

	if (stonith->state == stonith_disconnected) {
	return -ENOTCONN;
	}

	if (output_data != NULL) {
	*output_data = NULL;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	stonith->call_id++;
	/* prevent call_id from being negative (or zero) and conflicting
	* with the stonith_errors enum
	* use 2 because we use it as (stonith->call_id - 1) below
	*/
	if (stonith->call_id < 1) {
	stonith->call_id = 1;
	}

	CRM_CHECK(native->token != NULL,;);
	op_msg = stonith_create_op(stonith->call_id, native->token, op, data, call_options);
	if (op_msg == NULL) {
	return -EINVAL;
	}

	crm_xml_add_int(op_msg, F_STONITH_TIMEOUT, timeout);
	crm_trace("Sending %s message to STONITH service, Timeout: %ds", op, timeout);

	rc = crm_ipc_send(native->ipc, op_msg, &op_reply, 1000*(timeout + 60));
	free_xml(op_msg);

	if(rc < 0) {
	crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%ds): %d", op, timeout, rc);
	rc = -ECOMM;
	goto done;
	}

	crm_log_xml_trace(op_reply, "Reply");

	if (!(call_options & st_opt_sync_call)) {
	crm_trace("Async call %d, returning", stonith->call_id);
	CRM_CHECK(stonith->call_id != 0, return -EPROTO);
	free_xml(op_reply);

	return stonith->call_id;
	}

	rc = pcmk_ok;
	crm_element_value_int(op_reply, F_STONITH_CALLID, &reply_id);

	if (reply_id == stonith->call_id) {
	crm_trace("Syncronous reply %d received", reply_id);

	if (crm_element_value_int(op_reply, F_STONITH_RC, &rc) != 0) {
	rc = -ENOMSG;
	}

	if ((call_options & st_opt_discard_reply) \|\| output_data == NULL) {
	crm_trace("Discarding reply");

	} else {
	*output_data = op_reply;
	op_reply = NULL; /* Prevent subsequent free */
	}

	} else if (reply_id <= 0) {
	crm_err("Recieved bad reply: No id set");
	crm_log_xml_err(op_reply, "Bad reply");
	free_xml(op_reply);
	rc = -ENOMSG;

	} else {
	crm_err("Recieved bad reply: %d (wanted %d)", reply_id, stonith->call_id);
	crm_log_xml_err(op_reply, "Old reply");
	free_xml(op_reply);
	rc = -ENOMSG;
	}

	done:
	if (crm_ipc_connected(native->ipc) == FALSE) {
	crm_err("STONITH disconnected");
	stonith->state = stonith_disconnected;
	}

	free_xml(op_reply);
	return rc;
	}

	/* Not used with mainloop */
	bool
	stonith_dispatch(stonith_t * st)
	{
	gboolean stay_connected = TRUE;
	stonith_private_t *private = NULL;

	CRM_ASSERT(st != NULL);
	private = st->private;

	while(crm_ipc_ready(private->ipc)) {

	if(crm_ipc_read(private->ipc) > 0) {
	const char *msg = crm_ipc_buffer(private->ipc);
	stonith_dispatch_internal(msg, strlen(msg), st);
	}

	if(crm_ipc_connected(private->ipc) == FALSE) {
	crm_err("Connection closed");
	stay_connected = FALSE;
	}
	}

	return stay_connected;
	}

	int
	stonith_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata)
	{
	const char *type = NULL;
	struct notify_blob_s blob;

	stonith_t * st = userdata;
	stonith_private_t *private = NULL;

	CRM_ASSERT(st != NULL);
	private = st->private;

	blob.stonith = st;
	blob.xml = string2xml(buffer);
	if (blob.xml == NULL) {
	crm_warn("Received a NULL msg from STONITH service: %s.", buffer);
	return 0;
	}

	/* do callbacks */
	type = crm_element_value(blob.xml, F_TYPE);
	crm_trace("Activating %s callbacks...", type);

	if (safe_str_eq(type, T_STONITH_NG)) {
	stonith_perform_callback(st, blob.xml, 0, 0);

	} else if (safe_str_eq(type, T_STONITH_NOTIFY)) {
	g_list_foreach(private->notify_list, stonith_send_notification, &blob);

	} else {
	crm_err("Unknown message type: %s", type);
	crm_log_xml_warn(blob.xml, "BadReply");
	}

	free_xml(blob.xml);
	return 1;
	}

	static int
	stonith_api_free(stonith_t * stonith)
	{
	int rc = pcmk_ok;

	if (stonith->state != stonith_disconnected) {
	rc = stonith->cmds->disconnect(stonith);
	}

	if (stonith->state == stonith_disconnected) {
	stonith_private_t *private = stonith->private;

	g_hash_table_destroy(private->stonith_op_callback_table);
	free(private->token);
	free(stonith->private);
	free(stonith->cmds);
	free(stonith);
	}

	return rc;
	}

	void
	stonith_api_delete(stonith_t * stonith)
	{
	stonith_private_t *private = stonith->private;
	GList *list = private->notify_list;

	while (list != NULL) {
	stonith_notify_client_t *client = g_list_nth_data(list, 0);

	list = g_list_remove(list, client);
	free(client);
	}

	stonith->cmds->free(stonith);
	stonith = NULL;
	}

	stonith_t *
	stonith_api_new(void)
	{
	stonith_t *new_stonith = NULL;
	stonith_private_t *private = NULL;

	new_stonith = calloc(1, sizeof(stonith_t));
	private = calloc(1, sizeof(stonith_private_t));
	new_stonith->private = private;

	private->stonith_op_callback_table = g_hash_table_new_full(g_direct_hash, g_direct_equal,
	NULL, stonith_destroy_op_callback);
	private->notify_list = NULL;

	new_stonith->call_id = 1;
	new_stonith->state = stonith_disconnected;

	new_stonith->cmds = calloc(1, sizeof(stonith_api_operations_t));

	/* INDENT-OFF */
	new_stonith->cmds->free = stonith_api_free;
	new_stonith->cmds->connect = stonith_api_signon;
	new_stonith->cmds->disconnect = stonith_api_signoff;
	-
	- new_stonith->cmds->call = stonith_api_call;
	+
	+ new_stonith->cmds->list = stonith_api_list;
	+ new_stonith->cmds->monitor = stonith_api_monitor;
	+ new_stonith->cmds->status = stonith_api_status;
	new_stonith->cmds->fence = stonith_api_fence;
	new_stonith->cmds->confirm = stonith_api_confirm;
	new_stonith->cmds->history = stonith_api_history;

	- new_stonith->cmds->list = stonith_api_device_list;
	- new_stonith->cmds->metadata = stonith_api_device_metadata;
	+ new_stonith->cmds->list_agents = stonith_api_device_list;
	+ new_stonith->cmds->metadata = stonith_api_device_metadata;

	new_stonith->cmds->query = stonith_api_query;
	new_stonith->cmds->remove_device = stonith_api_remove_device;
	new_stonith->cmds->register_device = stonith_api_register_device;

	new_stonith->cmds->remove_level = stonith_api_remove_level;
	new_stonith->cmds->register_level = stonith_api_register_level;
	-
	- new_stonith->cmds->remove_callback = stonith_api_del_callback;
	- new_stonith->cmds->register_callback = stonith_api_add_callback;
	+
	+ new_stonith->cmds->remove_callback = stonith_api_del_callback;
	+ new_stonith->cmds->register_callback = stonith_api_add_callback;
	new_stonith->cmds->remove_notification = stonith_api_del_notification;
	new_stonith->cmds->register_notification = stonith_api_add_notification;
	/* INDENT-ON */

	return new_stonith;
	}

	stonith_key_value_t *
	stonith_key_value_add(stonith_key_value_t * head, const char key, const char value)
	{
	stonith_key_value_t p, end;

	p = calloc(1, sizeof(stonith_key_value_t));
	if (key) {
	p->key = strdup(key);
	}
	if (value) {
	p->value = strdup(value);
	}

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	void
	stonith_key_value_freeall(stonith_key_value_t * head, int keys, int values)
	{
	stonith_key_value_t *p;

	while (head) {
	p = head->next;
	if (keys) {
	free(head->key);
	}
	if (values) {
	free(head->value);
	}
	free(head);
	head = p;
	}
	}

	int
	stonith_api_kick(int nodeid, const char *uname, int timeout, bool off)
	{
	char *name = NULL;
	const char *action = "reboot";

	int rc = -EPROTO;
	stonith_t *st = NULL;
	enum stonith_call_options opts = st_opt_sync_call \| st_opt_allow_suicide;

	st = stonith_api_new();
	if (st) {
	rc = st->cmds->connect(st, "stonith-api", NULL);
	}

	if (uname != NULL) {
	name = strdup(uname);

	} else if (nodeid > 0) {
	opts \|= st_opt_cs_nodeid;
	name = crm_itoa(nodeid);
	}

	if (off) {
	action = "off";
	}

	if (rc == pcmk_ok) {
	rc = st->cmds->fence(st, opts, name, action, timeout);
	}

	if (st) {
	st->cmds->disconnect(st);
	stonith_api_delete(st);
	}

	free(name);
	return rc;
	}

	time_t
	stonith_api_time(int nodeid, const char *uname, bool in_progress)
	{
	int rc = 0;
	char *name = NULL;

	time_t when = 0;
	time_t progress = 0;
	stonith_t *st = NULL;
	stonith_history_t history, hp = NULL;
	enum stonith_call_options opts = st_opt_sync_call;

	st = stonith_api_new();
	if (st) {
	rc = st->cmds->connect(st, "stonith-api", NULL);
	}

	if (uname != NULL) {
	name = strdup(uname);

	} else if (nodeid > 0) {
	opts \|= st_opt_cs_nodeid;
	name = crm_itoa(nodeid);
	}

	if (st && rc == pcmk_ok) {
	st->cmds->history(st, st_opt_sync_call \| st_opt_cs_nodeid, name, &history, 120);

	for (hp = history; hp; hp = hp->next) {
	if (in_progress) {
	if (hp->state != st_done && hp->state != st_failed) {
	progress = time(NULL);
	}

	} else if (hp->state == st_done) {
	when = hp->completed;
	}
	}
	}

	if (progress) {
	when = progress;
	}

	if (st) {
	st->cmds->disconnect(st);
	stonith_api_delete(st);
	}

	free(name);
	return when;
	}
	diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c
	index 0b26c11ec3..c96ab99979 100644
	--- a/lib/lrmd/lrmd_client.c
	+++ b/lib/lrmd/lrmd_client.c
	@@ -1,1007 +1,1007 @@
	/*
	* Copyright (c) 2012 David Vossel <dvossel@redhat.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	#include <crm_internal.h>

	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <ctype.h>

	#include <sys/types.h>
	#include <sys/wait.h>

	#include <glib.h>
	#include <dirent.h>

	#include <crm/crm.h>
	#include <crm/lrmd.h>
	#include <crm/services.h>
	#include <crm/common/mainloop.h>
	#include <crm/msg_xml.h>

	#include <crm/stonith-ng.h>

	CRM_TRACE_INIT_DATA(lrmd);

	static stonith_t *stonith_api = NULL;

	typedef struct lrmd_private_s {
	int call_id;

	char *token;
	crm_ipc_t *ipc;
	mainloop_io_t *source;

	lrmd_event_callback callback;

	} lrmd_private_t;

	static lrmd_list_t *
	lrmd_list_add(lrmd_list_t * head, const char *value)
	{
	lrmd_list_t p, end;

	p = calloc(1, sizeof(lrmd_list_t));
	p->val = strdup(value);

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	void
	lrmd_list_freeall(lrmd_list_t * head)
	{
	lrmd_list_t *p;

	while (head) {
	char val = (char )head->val;

	p = head->next;
	free(val);
	free(head);
	head = p;
	}
	}

	lrmd_key_value_t *
	lrmd_key_value_add(lrmd_key_value_t * head, const char key, const char value)
	{
	lrmd_key_value_t p, end;

	p = calloc(1, sizeof(lrmd_key_value_t));
	p->key = strdup(key);
	p->value = strdup(value);

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	static void
	lrmd_key_value_freeall(lrmd_key_value_t * head)
	{
	lrmd_key_value_t *p;

	while (head) {
	p = head->next;
	free(head->key);
	free(head->value);
	free(head);
	head = p;
	}
	}

	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	g_hash_table_replace(user_data, strdup(key), strdup(value));
	}

	lrmd_event_data_t *
	lrmd_copy_event(lrmd_event_data_t * event)
	{
	lrmd_event_data_t *copy = NULL;

	copy = calloc(1, sizeof(lrmd_event_data_t));

	/* This will get all the int values.
	* we just have to be careful not to leave any
	* dangling pointers to strings. */
	memcpy(copy, event, sizeof(lrmd_event_data_t));

	copy->rsc_id = event->rsc_id ? strdup(event->rsc_id) : NULL;
	copy->op_type = event->op_type ? strdup(event->op_type) : NULL;
	copy->user_data = event->user_data ? strdup(event->user_data) : NULL;
	copy->output = event->output ? strdup(event->output) : NULL;

	if (event->params) {
	copy->params = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	if (copy->params != NULL) {
	g_hash_table_foreach(event->params, dup_attr, copy->params);
	}
	}

	return copy;
	}

	void
	lrmd_free_event(lrmd_event_data_t * event)
	{
	if (!event) {
	return;
	}

	/* free gives me grief if i try to cast */
	free((char *)event->rsc_id);
	free((char *)event->op_type);
	free((char *)event->user_data);
	free((char *)event->output);
	if (event->params) {
	g_hash_table_destroy(event->params);
	}
	free(event);
	}

	static int
	lrmd_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata)
	{
	const char *type;
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->private;
	lrmd_event_data_t event = { 0, };
	xmlNode *msg;

	if (!native->callback) {
	/* no callback set */
	return 1;
	}

	msg = string2xml(buffer);
	type = crm_element_value(msg, F_LRMD_OPERATION);
	crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
	event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);

	if (crm_str_eq(type, LRMD_OP_RSC_REG, TRUE)) {
	event.type = lrmd_event_register;
	} else if (crm_str_eq(type, LRMD_OP_RSC_UNREG, TRUE)) {
	event.type = lrmd_event_unregister;
	} else if (crm_str_eq(type, LRMD_OP_RSC_EXEC, TRUE)) {
	crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
	crm_element_value_int(msg, F_LRMD_RSC_INTERVAL, &event.interval);
	crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
	crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
	crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
	crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);

	crm_element_value_int(msg, F_LRMD_RSC_RUN_TIME, (int *)&event.t_run);
	crm_element_value_int(msg, F_LRMD_RSC_RCCHANGE_TIME, (int *)&event.t_rcchange);
	crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
	crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);

	event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
	event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
	event.output = crm_element_value(msg, F_LRMD_RSC_OUTPUT);
	event.type = lrmd_event_exec_complete;

	event.params = xml2list(msg);
	}

	native->callback(&event);

	if (event.params) {
	g_hash_table_destroy(event.params);
	}
	free_xml(msg);
	return 1;
	}

	/* Not used with mainloop */
	bool
	lrmd_dispatch(lrmd_t * lrmd)
	{
	gboolean stay_connected = TRUE;
	lrmd_private_t *private = NULL;

	CRM_ASSERT(lrmd != NULL);
	private = lrmd->private;
	while (crm_ipc_ready(private->ipc)) {
	if (crm_ipc_read(private->ipc) > 0) {
	const char *msg = crm_ipc_buffer(private->ipc);

	lrmd_dispatch_internal(msg, strlen(msg), lrmd);
	}
	if (crm_ipc_connected(private->ipc) == FALSE) {
	crm_err("Connection closed");
	stay_connected = FALSE;
	}
	}

	return stay_connected;
	}

	static xmlNode *
	lrmd_create_op(int call_id,
	const char token, const char op, xmlNode * data, enum lrmd_call_options options)
	{
	xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");

	CRM_CHECK(op_msg != NULL, return NULL);
	CRM_CHECK(token != NULL, return NULL);

	crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");

	crm_xml_add(op_msg, F_TYPE, T_LRMD);
	crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
	crm_xml_add(op_msg, F_LRMD_OPERATION, op);
	crm_xml_add_int(op_msg, F_LRMD_CALLID, call_id);
	crm_trace("Sending call options: %.8lx, %d", (long)options, options);
	crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);

	if (data != NULL) {
	add_message_xml(op_msg, F_LRMD_CALLDATA, data);
	}

	return op_msg;
	}

	static void
	lrmd_connection_destroy(gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->private;

	crm_info("connection destroyed");
	if (native->callback) {
	lrmd_event_data_t event = { 0, };
	event.type = lrmd_event_disconnect;
	native->callback(&event);
	}
	}

	static int
	lrmd_send_command(lrmd_t * lrmd, const char op, xmlNode data, xmlNode ** output_data, int timeout, /* ms. defaults to 1000 if set to 0 */
	enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	int reply_id = -1;
	lrmd_private_t *native = lrmd->private;
	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	if (!native->ipc) {
	return -ENOTCONN;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	native->call_id++;
	if (native->call_id < 1) {
	native->call_id = 1;
	}

	CRM_CHECK(native->token != NULL,;);

	op_msg = lrmd_create_op(native->call_id, native->token, op, data, options);

	if (op_msg == NULL) {
	return -EINVAL;
	}

	crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);

	rc = crm_ipc_send(native->ipc, op_msg, &op_reply, timeout);
	free_xml(op_msg);

	if (rc < 0) {
	crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
	rc = -ECOMM;
	goto done;
	}

	rc = pcmk_ok;
	crm_element_value_int(op_reply, F_LRMD_CALLID, &reply_id);
	if (reply_id == native->call_id) {
	crm_trace("reply received");
	if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
	rc = -ENOMSG;
	goto done;
	}

	if (output_data) {
	*output_data = op_reply;
	op_reply = NULL; /* Prevent subsequent free */
	}

	} else if (reply_id <= 0) {
	crm_err("Recieved bad reply: No id set");
	crm_log_xml_err(op_reply, "Bad reply");
	rc = -ENOMSG;
	} else {
	crm_err("Recieved bad reply: %d (wanted %d)", reply_id, native->call_id);
	crm_log_xml_err(op_reply, "Old reply");
	rc = -ENOMSG;
	}

	crm_log_xml_trace(op_reply, "Reply");

	done:
	if (crm_ipc_connected(native->ipc) == FALSE) {
	crm_err("LRMD disconnected");
	}

	free_xml(op_reply);
	return rc;
	}

	static int
	lrmd_api_connect(lrmd_t * lrmd, const char name, int fd)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->private;

	static struct ipc_client_callbacks lrmd_callbacks = {
	.dispatch = lrmd_dispatch_internal,
	.destroy = lrmd_connection_destroy
	};

	crm_info("Connecting to lrmd");

	if (fd) {
	/* No mainloop */
	native->ipc = crm_ipc_new("lrmd", 0);
	if (native->ipc && crm_ipc_connect(native->ipc)) {
	*fd = crm_ipc_get_fd(native->ipc);
	} else if (native->ipc) {
	rc = -ENOTCONN;
	}
	} else {
	native->source = mainloop_add_ipc_client("lrmd", 0, lrmd, &lrmd_callbacks);
	native->ipc = mainloop_get_ipc_client(native->source);
	}

	if (native->ipc == NULL) {
	crm_debug("Could not connect to the LRMD API");
	rc = -ENOTCONN;
	}

	if (!rc) {
	xmlNode *reply = NULL;
	xmlNode *hello = create_xml_node(NULL, "lrmd_command");

	crm_xml_add(hello, F_TYPE, T_LRMD);
	crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(hello, F_LRMD_CLIENTNAME, name);

	rc = crm_ipc_send(native->ipc, hello, &reply, -1);

	if (rc < 0) {
	crm_perror(LOG_DEBUG, "Couldn't complete registration with the lrmd API: %d", rc);
	rc = -ECOMM;
	} else if (reply == NULL) {
	crm_err("Did not receive registration reply");
	rc = -EPROTO;
	} else {
	const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
	const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);

	if (safe_str_neq(msg_type, CRM_OP_REGISTER)) {
	crm_err("Invalid registration message: %s", msg_type);
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;
	} else if (tmp_ticket == NULL) {
	crm_err("No registration token provided");
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;
	} else {
	crm_trace("Obtained registration token: %s", tmp_ticket);
	native->token = strdup(tmp_ticket);
	rc = pcmk_ok;
	}
	}

	free_xml(reply);
	free_xml(hello);
	}

	return rc;
	}

	static int
	lrmd_api_disconnect(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->private;

	crm_info("Disconnecting from lrmd service");

	if (native->source) {
	mainloop_del_ipc_client(native->source);
	native->source = NULL;
	native->ipc = NULL;
	} else if (native->ipc) {
	crm_ipc_close(native->ipc);
	crm_ipc_destroy(native->ipc);
	native->source = NULL;
	native->ipc = NULL;
	}

	free(native->token);
	native->token = NULL;
	return 0;
	}

	static int
	lrmd_api_register_rsc(lrmd_t * lrmd,
	const char *rsc_id,
	const char *class,
	const char provider, const char type, enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	xmlNode *data = NULL;

	if (!class \|\| !type \|\| !rsc_id) {
	return -EINVAL;
	}
	if (safe_str_eq(class, "ocf") && !provider) {
	return -EINVAL;
	}

	data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add(data, F_LRMD_CLASS, class);
	crm_xml_add(data, F_LRMD_PROVIDER, provider);
	crm_xml_add(data, F_LRMD_TYPE, type);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options);
	free_xml(data);

	return rc;
	}

	static int
	lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options);
	free_xml(data);

	return rc;
	}

	lrmd_rsc_info_t *
	lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
	{
	lrmd_rsc_info_t *copy = NULL;

	copy = calloc(1, sizeof(lrmd_rsc_info_t));

	copy->id = strdup(rsc_info->id);
	copy->type = strdup(rsc_info->type);
	copy->class = strdup(rsc_info->class);
	if (rsc_info->provider) {
	copy->provider = strdup(rsc_info->provider);
	}

	return copy;
	}

	void
	lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
	{
	if (!rsc_info) {
	return;
	}
	free(rsc_info->id);
	free(rsc_info->type);
	free(rsc_info->class);
	free(rsc_info->provider);
	free(rsc_info);
	}

	static lrmd_rsc_info_t *
	lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	lrmd_rsc_info_t *rsc_info = NULL;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
	xmlNode *output = NULL;
	const char *class = NULL;
	const char *provider = NULL;
	const char *type = NULL;

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options);
	free_xml(data);

	class = crm_element_value(output, F_LRMD_CLASS);
	provider = crm_element_value(output, F_LRMD_PROVIDER);
	type = crm_element_value(output, F_LRMD_TYPE);

	if (!output) {
	return NULL;
	} else if (!class \|\| !type) {
	free_xml(output);
	return NULL;
	} else if (safe_str_eq(class, "ocf") && !provider) {
	free_xml(output);
	return NULL;
	}

	rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
	rsc_info->id = strdup(rsc_id);
	rsc_info->class = strdup(class);
	if (provider) {
	rsc_info->provider = strdup(provider);
	}
	rsc_info->type = strdup(type);

	free_xml(output);
	return rsc_info;
	}

	static void
	lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
	{
	lrmd_private_t *native = lrmd->private;

	native->callback = callback;
	}

	static int
	stonith_get_metadata(const char provider, const char type, char **output)
	{
	int rc = pcmk_ok;

	stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type, provider, output, 0);
	if (*output == NULL) {
	rc = -EIO;
	}
	return rc;
	}

	static int
	lsb_get_metadata(const char type, char *output)
	{

	#define lsb_metadata_template \
	"<?xml version=\"1.0\"?>\n"\
	"<!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n"\
	"<resource-agent name=\"%s\" version=\"0.1\">\n"\
	" <version>1.0</version>\n"\
	" <longdesc lang=\"en\">\n"\
	" %s"\
	" </longdesc>\n"\
	" <shortdesc lang=\"en\">%s</shortdesc>\n"\
	" <parameters>\n"\
	" </parameters>\n"\
	" <actions>\n"\
	" <action name=\"start\" timeout=\"15\" />\n"\
	" <action name=\"stop\" timeout=\"15\" />\n"\
	" <action name=\"status\" timeout=\"15\" />\n"\
	" <action name=\"restart\" timeout=\"15\" />\n"\
	" <action name=\"force-reload\" timeout=\"15\" />\n"\
	" <action name=\"monitor\" timeout=\"15\" interval=\"15\" />\n"\
	" <action name=\"meta-data\" timeout=\"5\" />\n"\
	" </actions>\n"\
	" <special tag=\"LSB\">\n"\
	" <Provides>%s</Provides>\n"\
	" <Required-Start>%s</Required-Start>\n"\
	" <Required-Stop>%s</Required-Stop>\n"\
	" <Should-Start>%s</Should-Start>\n"\
	" <Should-Stop>%s</Should-Stop>\n"\
	" <Default-Start>%s</Default-Start>\n"\
	" <Default-Stop>%s</Default-Stop>\n"\
	" </special>\n"\
	"</resource-agent>\n"

	#define LSB_INITSCRIPT_INFOBEGIN_TAG "### BEGIN INIT INFO"
	#define LSB_INITSCRIPT_INFOEND_TAG "### END INIT INFO"
	#define PROVIDES "# Provides:"
	#define REQ_START "# Required-Start:"
	#define REQ_STOP "# Required-Stop:"
	#define SHLD_START "# Should-Start:"
	#define SHLD_STOP "# Should-Stop:"
	#define DFLT_START "# Default-Start:"
	#define DFLT_STOP "# Default-Stop:"
	#define SHORT_DSCR "# Short-Description:"
	#define DESCRIPTION "# Description:"

	#define lsb_meta_helper_free_value(m) \
	if ((m) != NULL) { \
	xmlFree(m); \
	(m) = NULL; \
	}

	#define lsb_meta_helper_get_value(buffer, ptr, keyword) \
	if (!ptr && !strncasecmp(buffer, keyword, strlen(keyword))) { \
	(ptr) = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST buffer+strlen(keyword)); \
	continue; \
	}

	char ra_pathname[PATH_MAX] = { 0, };
	FILE *fp;
	GString *meta_data = NULL;
	char buffer[1024];
	char *provides = NULL;
	char *req_start = NULL;
	char *req_stop = NULL;
	char *shld_start = NULL;
	char *shld_stop = NULL;
	char *dflt_start = NULL;
	char *dflt_stop = NULL;
	char *s_dscrpt = NULL;
	char *xml_l_dscrpt = NULL;
	GString *l_dscrpt = NULL;

	snprintf(ra_pathname, sizeof(ra_pathname), "%s%s%s",
	type[0] == '/' ? "" : LSB_ROOT_DIR, type[0] == '/' ? "" : "/", type);

	if (!(fp = fopen(ra_pathname, "r"))) {
	return -EIO;
	}

	/* Enter into the lsb-compliant comment block */
	while (fgets(buffer, sizeof(buffer), fp)) {
	/* Now suppose each of the following eight arguments contain only one line */
	lsb_meta_helper_get_value(buffer, provides, PROVIDES)
	lsb_meta_helper_get_value(buffer, req_start, REQ_START)
	lsb_meta_helper_get_value(buffer, req_stop, REQ_STOP)
	lsb_meta_helper_get_value(buffer, shld_start, SHLD_START)
	lsb_meta_helper_get_value(buffer, shld_stop, SHLD_STOP)
	lsb_meta_helper_get_value(buffer, dflt_start, DFLT_START)
	lsb_meta_helper_get_value(buffer, dflt_stop, DFLT_STOP)
	lsb_meta_helper_get_value(buffer, s_dscrpt, SHORT_DSCR)

	/* Long description may cross multiple lines */
	if ((l_dscrpt == NULL) && (0 == strncasecmp(buffer, DESCRIPTION, strlen(DESCRIPTION)))) {
	l_dscrpt = g_string_new(buffer + strlen(DESCRIPTION));
	/* Between # and keyword, more than one space, or a tab character,
	* indicates the continuation line. Extracted from LSB init script standard */
	while (fgets(buffer, sizeof(buffer), fp)) {
	if (!strncmp(buffer, "# ", 3) \|\| !strncmp(buffer, "#\t", 2)) {
	buffer[0] = ' ';
	l_dscrpt = g_string_append(l_dscrpt, buffer);
	} else {
	fputs(buffer, fp);
	break; /* Long description ends */
	}
	}
	continue;
	}
	if (l_dscrpt) {
	xml_l_dscrpt = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST(l_dscrpt->str));
	}
	if (!strncasecmp(buffer, LSB_INITSCRIPT_INFOEND_TAG, strlen(LSB_INITSCRIPT_INFOEND_TAG))) {
	/* Get to the out border of LSB comment block */
	break;
	}
	if (buffer[0] != '#') {
	break; /* Out of comment block in the beginning */
	}
	}
	fclose(fp);

	meta_data = g_string_new("");
	g_string_sprintf(meta_data, lsb_metadata_template, type,
	(xml_l_dscrpt == NULL) ? type : xml_l_dscrpt,
	(s_dscrpt == NULL) ? type : s_dscrpt, (provides == NULL) ? "" : provides,
	(req_start == NULL) ? "" : req_start, (req_stop == NULL) ? "" : req_stop,
	(shld_start == NULL) ? "" : shld_start, (shld_stop == NULL) ? "" : shld_stop,
	(dflt_start == NULL) ? "" : dflt_start, (dflt_stop == NULL) ? "" : dflt_stop);

	lsb_meta_helper_free_value(xml_l_dscrpt);
	lsb_meta_helper_free_value(s_dscrpt);
	lsb_meta_helper_free_value(provides);
	lsb_meta_helper_free_value(req_start);
	lsb_meta_helper_free_value(req_stop);
	lsb_meta_helper_free_value(shld_start);
	lsb_meta_helper_free_value(shld_stop);
	lsb_meta_helper_free_value(dflt_start);
	lsb_meta_helper_free_value(dflt_stop);

	if (l_dscrpt) {
	g_string_free(l_dscrpt, TRUE);
	}

	*output = strdup(meta_data->str);
	g_string_free(meta_data, TRUE);

	return pcmk_ok;
	}

	static int
	generic_get_metadata(const char standard, const char provider, const char type, char *output)
	{
	svc_action_t *action = resources_action_create(type,
	standard,
	provider,
	type,
	"meta-data",
	0,
	5000,
	NULL);

	if (!(services_action_sync(action))) {
	crm_err("Failed to retrieve meta-data for %s:%s:%s", standard, provider, type);
	services_action_free(action);
	return -EIO;
	}

	if (!action->stdout_data) {
	crm_err("Failed to retrieve meta-data for %s:%s:%s", standard, provider, type);
	services_action_free(action);
	return -EIO;
	}

	*output = strdup(action->stdout_data);
	services_action_free(action);

	return pcmk_ok;
	}

	static int
	lrmd_api_get_metadata(lrmd_t * lrmd,
	const char *class,
	const char *provider,
	const char type, char *output, enum lrmd_call_options options)
	{
	if (!class \|\| !type) {
	return -EINVAL;
	}

	if (safe_str_eq(class, "stonith")) {
	return stonith_get_metadata(provider, type, output);
	} else if (safe_str_eq(class, "lsb")) {
	return lsb_get_metadata(type, output);
	}
	return generic_get_metadata(class, provider, type, output);
	}

	static int
	lrmd_api_exec(lrmd_t * lrmd, const char rsc_id, const char action, const char userdata, int interval, / ms */
	int timeout, /* ms */
	int start_delay, /* ms */
	enum lrmd_call_options options, lrmd_key_value_t * params)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
	xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add(data, F_LRMD_RSC_ACTION, action);
	crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
	crm_xml_add_int(data, F_LRMD_RSC_INTERVAL, interval);
	crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
	crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);

	for (; params; params = params->next) {
	hash2field((gpointer) params->key, (gpointer) params->value, args);
	}

	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options);
	free_xml(data);

	lrmd_key_value_freeall(params);
	return rc;
	}

	static int
	lrmd_api_cancel(lrmd_t * lrmd, const char rsc_id, const char action, int interval)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ACTION, action);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add_int(data, F_LRMD_RSC_INTERVAL, interval);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0);
	free_xml(data);
	return rc;
	}

	static int
	list_stonith_agents(lrmd_list_t ** resources)
	{
	int rc = 0;
	stonith_key_value_t *stonith_resources = NULL;
	stonith_key_value_t *dIter = NULL;

	- stonith_api->cmds->list(stonith_api, st_opt_sync_call, NULL, &stonith_resources, 0);
	+ stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL, &stonith_resources, 0);

	for (dIter = stonith_resources; dIter; dIter = dIter->next) {
	rc++;
	if(resources) {
	resources = lrmd_list_add(resources, dIter->value);
	}
	}

	stonith_key_value_freeall(stonith_resources, 1, 0);
	return rc;
	}

	static int
	lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
	const char *provider)
	{
	int rc = 0;

	if (safe_str_eq(class, "stonith")) {
	rc += list_stonith_agents(resources);

	} else {
	GListPtr gIter = NULL;
	GList *agents = resources_list_agents(class, provider);

	for (gIter = agents; gIter != NULL; gIter = gIter->next) {
	resources = lrmd_list_add(resources, (const char *)gIter->data);
	rc++;
	}
	g_list_free_full(agents, free);

	if (!class) {
	rc += list_stonith_agents(resources);
	}
	}

	if(rc == 0) {
	crm_notice("No agents found for class %s", class);
	rc = -EPROTONOSUPPORT;
	}
	return rc;
	}

	static int
	does_provider_have_agent(const char agent, const char provider, const char *class)
	{
	int found = 0;
	GList *agents = NULL;
	GListPtr gIter2 = NULL;

	agents = resources_list_agents(class, provider);
	for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
	if (safe_str_eq(agent, gIter2->data)) {
	found = 1;
	}
	}
	g_list_free_full(agents, free);

	return found;
	}

	static int
	lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char agent, lrmd_list_t * providers)
	{
	int rc = pcmk_ok;
	char *provider = NULL;
	GList *ocf_providers = NULL;
	GListPtr gIter = NULL;

	ocf_providers = resources_list_providers("ocf");

	for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
	provider = gIter->data;
	if (!agent \|\| does_provider_have_agent(agent, provider, "ocf")) {
	providers = lrmd_list_add(providers, (const char *)gIter->data);
	rc++;
	}
	}

	g_list_free_full(ocf_providers, free);
	return rc;
	}

	static int
	lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
	{
	int rc = 0;
	char *standard = NULL;
	GList *standards = NULL;
	GListPtr gIter = NULL;

	standards = resources_list_standards();

	for (gIter = standards; gIter != NULL; gIter = gIter->next) {
	standard = gIter->data;
	supported = lrmd_list_add(supported, (const char *)gIter->data);
	rc++;
	}

	if(list_stonith_agents(NULL) > 0) {
	supported = lrmd_list_add(supported, "stonith");
	rc++;
	}

	g_list_free_full(standards, free);
	return rc;
	}

	lrmd_t *
	lrmd_api_new(void)
	{
	lrmd_t *new_lrmd = NULL;
	lrmd_private_t *pvt = NULL;

	new_lrmd = calloc(1, sizeof(lrmd_t));
	pvt = calloc(1, sizeof(lrmd_private_t));
	new_lrmd->cmds = calloc(1, sizeof(lrmd_api_operations_t));

	new_lrmd->private = pvt;

	new_lrmd->cmds->connect = lrmd_api_connect;
	new_lrmd->cmds->disconnect = lrmd_api_disconnect;
	new_lrmd->cmds->register_rsc = lrmd_api_register_rsc;
	new_lrmd->cmds->unregister_rsc = lrmd_api_unregister_rsc;
	new_lrmd->cmds->get_rsc_info = lrmd_api_get_rsc_info;
	new_lrmd->cmds->set_callback = lrmd_api_set_callback;
	new_lrmd->cmds->get_metadata = lrmd_api_get_metadata;
	new_lrmd->cmds->exec = lrmd_api_exec;
	new_lrmd->cmds->cancel = lrmd_api_cancel;
	new_lrmd->cmds->list_agents = lrmd_api_list_agents;
	new_lrmd->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
	new_lrmd->cmds->list_standards = lrmd_api_list_standards;

	if (!stonith_api) {
	stonith_api = stonith_api_new();
	}

	return new_lrmd;
	}

	void
	lrmd_api_delete(lrmd_t * lrmd)
	{
	lrmd->cmds->disconnect(lrmd); /* no-op if already disconnected */
	free(lrmd->cmds);
	free(lrmd->private);
	free(lrmd);
	}
	diff --git a/lrmd/lrmd.c b/lrmd/lrmd.c
	index 324493fd60..78ae5d1194 100644
	--- a/lrmd/lrmd.c
	+++ b/lrmd/lrmd.c
	@@ -1,991 +1,991 @@
	/*
	* Copyright (c) 2012 David Vossel <dvossel@redhat.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <unistd.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/ipc.h>
	#include <crm/msg_xml.h>

	#include <lrmd_private.h>

	#ifdef HAVE_SYS_TIMEB_H
	# include <sys/timeb.h>
	#endif

	GHashTable *rsc_list = NULL;
	GHashTable *client_list = NULL;

	typedef struct lrmd_cmd_s {
	int timeout;
	int interval;
	int start_delay;

	int call_id;
	int exec_rc;
	int lrmd_op_status;

	/* Timer ids, must be removed on cmd destruction. */
	int delay_id;
	int stonith_recurring_id;

	int rsc_deleted;

	char *only_notify_client;
	char *origin;
	char *rsc_id;
	char *action;
	char *output;
	char *userdata_str;

	#ifdef HAVE_SYS_TIMEB_H
	/* Timestamp of when op ran */
	struct timeb t_run;
	/* Timestamp of when op was queued */
	struct timeb t_queue;
	/* Timestamp of last rc change */
	struct timeb t_rcchange;
	#endif

	GHashTable *params;
	} lrmd_cmd_t;

	static void cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc);
	static gboolean lrmd_rsc_dispatch(gpointer user_data);

	static lrmd_rsc_t *
	build_rsc_from_xml(xmlNode * msg)
	{
	xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, msg, LOG_ERR);
	lrmd_rsc_t *rsc = NULL;

	rsc = calloc(1, sizeof(lrmd_rsc_t));
	rsc->rsc_id = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ID);
	rsc->class = crm_element_value_copy(rsc_xml, F_LRMD_CLASS);
	rsc->provider = crm_element_value_copy(rsc_xml, F_LRMD_PROVIDER);
	rsc->type = crm_element_value_copy(rsc_xml, F_LRMD_TYPE);
	rsc->work = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_rsc_dispatch, rsc);
	return rsc;
	}

	static lrmd_cmd_t *
	create_lrmd_cmd(xmlNode * msg, lrmd_client_t * client)
	{
	int call_options = 0;
	xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, msg, LOG_ERR);
	lrmd_cmd_t *cmd = NULL;

	cmd = calloc(1, sizeof(lrmd_cmd_t));

	crm_element_value_int(msg, F_LRMD_CALLOPTS, &call_options);

	if (call_options & lrmd_opt_notify_orig_only) {
	cmd->only_notify_client = strdup(client->id);
	}

	crm_element_value_int(msg, F_LRMD_CALLID, &cmd->call_id);
	crm_element_value_int(rsc_xml, F_LRMD_RSC_INTERVAL, &cmd->interval);
	crm_element_value_int(rsc_xml, F_LRMD_TIMEOUT, &cmd->timeout);
	crm_element_value_int(rsc_xml, F_LRMD_RSC_START_DELAY, &cmd->start_delay);

	cmd->origin = crm_element_value_copy(rsc_xml, F_LRMD_ORIGIN);
	cmd->action = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ACTION);
	cmd->userdata_str = crm_element_value_copy(rsc_xml, F_LRMD_RSC_USERDATA_STR);
	cmd->rsc_id = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ID);

	cmd->params = xml2list(rsc_xml);

	return cmd;
	}

	static void
	free_lrmd_cmd(lrmd_cmd_t * cmd)
	{
	if (cmd->stonith_recurring_id) {
	g_source_remove(cmd->stonith_recurring_id);
	}
	if (cmd->delay_id) {
	g_source_remove(cmd->delay_id);
	}
	if (cmd->params) {
	g_hash_table_destroy(cmd->params);
	}
	free(cmd->origin);
	free(cmd->action);
	free(cmd->userdata_str);
	free(cmd->rsc_id);
	free(cmd->output);
	free(cmd->only_notify_client);
	free(cmd);
	}

	static gboolean
	stonith_recurring_op_helper(gpointer data)
	{
	lrmd_cmd_t *cmd = data;
	lrmd_rsc_t *rsc;

	cmd->stonith_recurring_id = 0;

	if (!cmd->rsc_id) {
	return FALSE;
	}

	rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id);

	CRM_ASSERT(rsc != NULL);
	/* take it out of recurring_ops list, and put it in the pending ops
	* to be executed */
	rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd);
	rsc->pending_ops = g_list_append(rsc->pending_ops, cmd);
	#ifdef HAVE_SYS_TIMEB_H
	ftime(&cmd->t_queue);
	#endif
	mainloop_set_trigger(rsc->work);

	return FALSE;
	}

	static gboolean
	start_delay_helper(gpointer data)
	{
	lrmd_cmd_t *cmd = data;
	lrmd_rsc_t *rsc = NULL;

	cmd->delay_id = 0;
	rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;

	if (rsc) {
	mainloop_set_trigger(rsc->work);
	}

	return FALSE;
	}

	static void
	schedule_lrmd_cmd(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
	{
	CRM_CHECK(cmd != NULL, return);
	CRM_CHECK(rsc != NULL, return);

	crm_trace("Scheduling %s on %s", cmd->action, rsc->rsc_id);
	rsc->pending_ops = g_list_append(rsc->pending_ops, cmd);
	#ifdef HAVE_SYS_TIMEB_H
	ftime(&cmd->t_queue);
	#endif
	mainloop_set_trigger(rsc->work);

	if (cmd->start_delay) {
	cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
	}

	}

	static void
	send_reply(lrmd_client_t * client, int rc, int call_id)
	{
	int send_rc = 0;
	xmlNode *reply = NULL;

	reply = create_xml_node(NULL, T_LRMD_REPLY);
	crm_xml_add(reply, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add_int(reply, F_LRMD_RC, rc);
	crm_xml_add_int(reply, F_LRMD_CALLID, call_id);

	send_rc = crm_ipcs_send(client->channel, reply, FALSE);

	free_xml(reply);
	if (send_rc < 0) {
	crm_warn("LRMD reply to %s failed: %d", client->name, send_rc);
	}
	}

	static void
	send_client_notify(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNode *update_msg = user_data;
	lrmd_client_t *client = value;

	if (client == NULL) {
	crm_err("Asked to send event to NULL client");
	return;
	} else if (client->channel == NULL) {
	crm_trace("Asked to send event to disconnected client");
	return;
	} else if (client->name == NULL) {
	crm_trace("Asked to send event to client with no name");
	return;
	}

	if (crm_ipcs_send(client->channel, update_msg, TRUE) <= 0) {
	crm_warn("Notification of client %s/%s failed", client->name, client->id);
	}
	}

	#ifdef HAVE_SYS_TIMEB_H
	static int
	time_diff_ms(struct timeb now, struct timeb old)
	{
	int sec = difftime(now->time, old->time);
	int ms = now->millitm - old->millitm;

	if (old->time == 0) {
	return 0;
	}

	return (sec * 1000) + ms;
	}
	#endif

	static void
	send_cmd_complete_notify(lrmd_cmd_t * cmd)
	{
	#ifdef HAVE_SYS_TIMEB_H
	struct timeb now = { 0, };
	#endif
	xmlNode *notify = NULL;

	notify = create_xml_node(NULL, T_LRMD_NOTIFY);

	crm_xml_add(notify, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add_int(notify, F_LRMD_TIMEOUT, cmd->timeout);
	crm_xml_add_int(notify, F_LRMD_RSC_INTERVAL, cmd->interval);
	crm_xml_add_int(notify, F_LRMD_RSC_START_DELAY, cmd->start_delay);
	crm_xml_add_int(notify, F_LRMD_EXEC_RC, cmd->exec_rc);
	crm_xml_add_int(notify, F_LRMD_OP_STATUS, cmd->lrmd_op_status);
	crm_xml_add_int(notify, F_LRMD_CALLID, cmd->call_id);
	crm_xml_add_int(notify, F_LRMD_RSC_DELETED, cmd->rsc_deleted);

	#ifdef HAVE_SYS_TIMEB_H
	ftime(&now);
	crm_xml_add_int(notify, F_LRMD_RSC_RUN_TIME, cmd->t_run.time);
	crm_xml_add_int(notify, F_LRMD_RSC_RCCHANGE_TIME, cmd->t_rcchange.time);
	crm_xml_add_int(notify, F_LRMD_RSC_EXEC_TIME, time_diff_ms(&now, &cmd->t_run));
	crm_xml_add_int(notify, F_LRMD_RSC_QUEUE_TIME, time_diff_ms(&cmd->t_run, &cmd->t_queue));
	#endif

	crm_xml_add(notify, F_LRMD_OPERATION, LRMD_OP_RSC_EXEC);
	crm_xml_add(notify, F_LRMD_RSC_ID, cmd->rsc_id);
	crm_xml_add(notify, F_LRMD_RSC_ACTION, cmd->action);
	crm_xml_add(notify, F_LRMD_RSC_USERDATA_STR, cmd->userdata_str);
	crm_xml_add(notify, F_LRMD_RSC_OUTPUT, cmd->output);

	if (cmd->params) {
	char *key = NULL;
	char *value = NULL;
	GHashTableIter iter;

	xmlNode *args = create_xml_node(notify, XML_TAG_ATTRS);

	g_hash_table_iter_init(&iter, cmd->params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	hash2field((gpointer) key, (gpointer) value, args);
	}
	}

	if (cmd->only_notify_client) {
	lrmd_client_t *client = g_hash_table_lookup(client_list, cmd->only_notify_client);

	if (client) {
	send_client_notify(client->id, client, notify);
	}
	} else {
	g_hash_table_foreach(client_list, send_client_notify, notify);
	}

	free_xml(notify);
	}

	static void
	send_generic_notify(int rc, xmlNode * request)
	{
	int call_id = 0;
	xmlNode *notify = NULL;
	xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
	const char *op = crm_element_value(request, F_LRMD_OPERATION);

	crm_element_value_int(request, F_LRMD_CALLID, &call_id);

	notify = create_xml_node(NULL, T_LRMD_NOTIFY);
	crm_xml_add(notify, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add_int(notify, F_LRMD_RC, rc);
	crm_xml_add_int(notify, F_LRMD_CALLID, call_id);
	crm_xml_add(notify, F_LRMD_OPERATION, op);
	crm_xml_add(notify, F_LRMD_RSC_ID, rsc_id);

	g_hash_table_foreach(client_list, send_client_notify, notify);

	free_xml(notify);
	}

	static void
	cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc)
	{
	crm_trace("Resource operation rsc:%s action:%s completed (%p %p)", cmd->rsc_id, cmd->action, rsc?rsc->active:NULL, cmd);

	if (rsc && (rsc->active == cmd)) {
	rsc->active = NULL;
	mainloop_set_trigger(rsc->work);
	}

	if (!rsc) {
	cmd->rsc_deleted = 1;
	}

	send_cmd_complete_notify(cmd);

	if (cmd->interval && (cmd->lrmd_op_status == PCMK_LRM_OP_CANCELLED)) {
	if (rsc) {
	rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd);
	rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd);
	}
	free_lrmd_cmd(cmd);
	} else if (cmd->interval == 0) {
	if (rsc) {
	rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd);
	}
	free_lrmd_cmd(cmd);
	} else {
	/* Clear all the values pertaining just to the last iteration of a recurring op. */
	cmd->lrmd_op_status = 0;
	memset(&cmd->t_run, 0, sizeof(cmd->t_run));
	memset(&cmd->t_queue, 0, sizeof(cmd->t_queue));
	free(cmd->output);
	cmd->output = NULL;
	}
	}

	static int
	lsb2uniform_rc(const char *action, int rc)
	{
	if (rc < 0) {
	return PCMK_EXECRA_UNKNOWN_ERROR;
	}

	/* status has different return codes that everything else. */
	if (!safe_str_eq(action, "status") && !safe_str_eq(action, "monitor")) {
	if (rc > PCMK_LSB_NOT_RUNNING) {
	return PCMK_EXECRA_UNKNOWN_ERROR;
	}
	return rc;
	}

	switch (rc) {
	case PCMK_LSB_STATUS_OK:
	return PCMK_EXECRA_OK;
	case PCMK_LSB_STATUS_NOT_INSTALLED:
	return PCMK_EXECRA_NOT_INSTALLED;
	case PCMK_LSB_STATUS_VAR_PID:
	case PCMK_LSB_STATUS_VAR_LOCK:
	case PCMK_LSB_STATUS_NOT_RUNNING:
	return PCMK_EXECRA_NOT_RUNNING;
	default:
	return PCMK_EXECRA_UNKNOWN_ERROR;
	}

	return PCMK_EXECRA_UNKNOWN_ERROR;
	}

	static int
	ocf2uniform_rc(int rc)
	{
	if (rc < 0 \|\| rc > PCMK_OCF_FAILED_MASTER) {
	return PCMK_EXECRA_UNKNOWN_ERROR;
	}

	return rc;
	}

	static int
	stonith2uniform_rc(const char *action, int rc)
	{
	if (rc == -ENODEV) {
	if (safe_str_eq(action, "stop")) {
	rc = PCMK_EXECRA_OK;
	} else if (safe_str_eq(action, "start")) {
	rc = PCMK_EXECRA_NOT_INSTALLED;
	} else {
	rc = PCMK_EXECRA_NOT_RUNNING;
	}
	} else if (rc != 0) {
	rc = PCMK_EXECRA_UNKNOWN_ERROR;
	}
	return rc;
	}

	static int
	get_uniform_rc(const char standard, const char action, int rc)
	{
	if (safe_str_eq(standard, "ocf")) {
	return ocf2uniform_rc(rc);
	} else if (safe_str_eq(standard, "stonith")) {
	return stonith2uniform_rc(action, rc);
	} else if (safe_str_eq(standard, "systemd")) {
	return rc;
	} else if (safe_str_eq(standard, "upstart")) {
	return rc;
	} else {
	return lsb2uniform_rc(action, rc);
	}
	}

	static void
	action_complete(svc_action_t * action)
	{
	lrmd_rsc_t *rsc;
	lrmd_cmd_t *cmd = action->cb_data;

	if (!cmd) {
	crm_err("LRMD action (%s) completed does not match any known operations.", action->id);
	return;
	}
	#ifdef HAVE_SYS_TIMEB_H
	if (cmd->exec_rc != action->rc) {
	ftime(&cmd->t_rcchange);
	}
	#endif

	cmd->exec_rc = get_uniform_rc(action->standard, cmd->action, action->rc);
	cmd->lrmd_op_status = action->status;
	rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;

	if (action->stdout_data) {
	cmd->output = strdup(action->stdout_data);
	}

	cmd_finalize(cmd, rsc);
	}

	static int
	lrmd_rsc_execute_stonith(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
	{
	int rc = 0;
	stonith_t *stonith_api = get_stonith_connection();

	if (!stonith_api) {
	cmd->exec_rc = get_uniform_rc("stonith", cmd->action, -ENOTCONN);
	cmd->lrmd_op_status = PCMK_LRM_OP_ERROR;
	cmd_finalize(cmd, rsc);
	return -EUNATCH;
	}

	if (safe_str_eq(cmd->action, "start")) {
	char *key = NULL;
	char *value = NULL;
	stonith_key_value_t *device_params = NULL;

	if (cmd->params) {
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, cmd->params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	device_params = stonith_key_value_add(device_params, key, value);
	}
	}

	rc = stonith_api->cmds->register_device(stonith_api,
	st_opt_sync_call,
	cmd->rsc_id,
	rsc->provider, rsc->type, device_params);

	stonith_key_value_freeall(device_params, 1, 1);
	if (rc == 0) {
	- rc = stonith_api->cmds->call(stonith_api,
	+ rc = stonith_api->cmds->monitor(stonith_api,
	st_opt_sync_call,
	- cmd->rsc_id, "monitor", NULL, cmd->timeout);
	+ cmd->rsc_id, cmd->timeout);
	}
	} else if (safe_str_eq(cmd->action, "stop")) {
	rc = stonith_api->cmds->remove_device(stonith_api, st_opt_sync_call, cmd->rsc_id);
	} else if (safe_str_eq(cmd->action, "monitor")) {
	- rc = stonith_api->cmds->call(stonith_api,
	+ rc = stonith_api->cmds->monitor(stonith_api,
	st_opt_sync_call,
	- cmd->rsc_id, cmd->action, NULL, cmd->timeout);
	+ cmd->rsc_id, cmd->timeout);
	}

	cmd->exec_rc = get_uniform_rc("stonith", cmd->action, rc);

	/* Attempt to map return codes to op status if possible */
	if (rc) {
	switch (rc) {
	case -EPROTONOSUPPORT:
	cmd->lrmd_op_status = PCMK_LRM_OP_NOTSUPPORTED;
	break;
	case -ETIME:
	cmd->lrmd_op_status = PCMK_LRM_OP_TIMEOUT;
	break;
	default:
	cmd->lrmd_op_status = PCMK_LRM_OP_ERROR;
	}
	} else {
	cmd->lrmd_op_status = PCMK_LRM_OP_DONE;
	}

	if (cmd->interval > 0) {
	rsc->recurring_ops = g_list_append(rsc->recurring_ops, cmd);
	cmd->stonith_recurring_id = g_timeout_add(cmd->interval, stonith_recurring_op_helper, cmd);
	}
	cmd_finalize(cmd, rsc);

	return rc;
	}

	static const char *
	normalize_action_name(lrmd_rsc_t * rsc, const char *action)
	{
	if (safe_str_eq(action, "monitor") &&
	(safe_str_eq(rsc->class, "lsb") \|\|
	safe_str_eq(rsc->class, "service") \|\| safe_str_eq(rsc->class, "systemd"))) {
	return "status";
	}
	return action;
	}

	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	g_hash_table_replace(user_data, strdup(key), strdup(value));
	}

	static int
	lrmd_rsc_execute_service_lib(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
	{
	svc_action_t *action = NULL;
	GHashTable *params_copy = NULL;

	crm_trace("Creating action, resource:%s action:%s class:%s provider:%s agent:%s",
	rsc->rsc_id, cmd->action, rsc->class, rsc->provider, rsc->type);

	if (cmd->params) {
	params_copy = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	if (params_copy != NULL) {
	g_hash_table_foreach(cmd->params, dup_attr, params_copy);
	}
	}

	action = resources_action_create(rsc->rsc_id,
	rsc->class,
	rsc->provider,
	rsc->type,
	normalize_action_name(rsc, cmd->action),
	cmd->interval, cmd->timeout, params_copy);

	if (!action) {
	crm_err("Failed to create action, action:%s on resource %s", cmd->action, rsc->rsc_id);
	cmd->lrmd_op_status = PCMK_LRM_OP_ERROR;
	goto exec_done;
	}

	action->cb_data = cmd;
	/* The cmd will be finalized by the action_complete callback after
	* the service library is done with it */
	rsc->active = cmd; /* only one op at a time for a rsc */
	if (cmd->interval) {
	rsc->recurring_ops = g_list_append(rsc->recurring_ops, cmd);
	}

	/* 'cmd' may not be valid after this point
	*
	* Upstart and systemd both synchronously determine monitor/status
	* results and call action_complete (which may free 'cmd') if necessary
	*/
	if (services_action_async(action, action_complete)) {
	return TRUE;
	}

	cmd->exec_rc = action->rc;
	cmd->lrmd_op_status = PCMK_LRM_OP_ERROR;
	services_action_free(action);
	action = NULL;

	exec_done:
	if (cmd) {
	cmd_finalize(cmd, rsc);
	}
	return TRUE;
	}

	static gboolean
	lrmd_rsc_execute(lrmd_rsc_t * rsc)
	{
	lrmd_cmd_t *cmd = NULL;

	CRM_CHECK(rsc != NULL, return FALSE);

	if (rsc->active) {
	crm_trace("%s is still active", rsc->rsc_id);
	return TRUE;
	}

	if (rsc->pending_ops) {
	GList *first = rsc->pending_ops;

	cmd = first->data;
	if (cmd->delay_id) {
	crm_trace
	("Command %s %s was asked to run too early, waiting for start_delay timeout of %dms",
	cmd->rsc_id, cmd->action, cmd->start_delay);
	return TRUE;
	}
	rsc->pending_ops = g_list_remove_link(rsc->pending_ops, first);
	g_list_free_1(first);

	#ifdef HAVE_SYS_TIMEB_H
	ftime(&cmd->t_run);
	}
	#endif

	if (!cmd) {
	crm_trace("Nothing further to do for %s", rsc->rsc_id);
	return TRUE;
	}

	if (safe_str_eq(rsc->class, "stonith")) {
	lrmd_rsc_execute_stonith(rsc, cmd);
	} else {
	lrmd_rsc_execute_service_lib(rsc, cmd);
	}

	return TRUE;
	}

	static gboolean
	lrmd_rsc_dispatch(gpointer user_data)
	{
	return lrmd_rsc_execute(user_data);
	}

	void
	free_rsc(gpointer data)
	{
	GListPtr gIter = NULL;
	lrmd_rsc_t *rsc = data;
	int is_stonith = safe_str_eq(rsc->class, "stonith");

	for (gIter = rsc->pending_ops; gIter != NULL; gIter = gIter->next) {
	lrmd_cmd_t *cmd = gIter->data;

	/* command was never executed */
	cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED;
	cmd_finalize(cmd, NULL);
	}
	/* frees list, but not list elements. */
	g_list_free(rsc->pending_ops);

	for (gIter = rsc->recurring_ops; gIter != NULL; gIter = gIter->next) {
	lrmd_cmd_t *cmd = gIter->data;

	if (is_stonith) {
	cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED;
	cmd_finalize(cmd, NULL);
	} else {
	/* This command is already handed off to service library,
	* let service library cancel it and tell us via the callback
	* when it is cancelled. The rsc can be safely destroyed
	* even if we are waiting for the cancel result */
	services_action_cancel(rsc->rsc_id, cmd->action, cmd->interval);
	}
	}
	/* frees list, but not list elements. */
	g_list_free(rsc->recurring_ops);

	free(rsc->rsc_id);
	free(rsc->class);
	free(rsc->provider);
	free(rsc->type);
	mainloop_destroy_trigger(rsc->work);

	free(rsc);
	}

	static int
	process_lrmd_signon(lrmd_client_t * client, xmlNode * request)
	{
	xmlNode *reply = create_xml_node(NULL, "reply");

	crm_xml_add(reply, F_LRMD_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(reply, F_LRMD_CLIENTID, client->id);
	crm_ipcs_send(client->channel, reply, FALSE);

	free_xml(reply);
	return pcmk_ok;
	}

	static int
	process_lrmd_rsc_register(lrmd_client_t * client, xmlNode * request)
	{
	int rc = pcmk_ok;
	lrmd_rsc_t *rsc = build_rsc_from_xml(request);
	lrmd_rsc_t *dup = g_hash_table_lookup(rsc_list, rsc->rsc_id);

	if (dup &&
	safe_str_eq(rsc->class, dup->class) &&
	safe_str_eq(rsc->provider, dup->provider) && safe_str_eq(rsc->type, dup->type)) {

	crm_warn("Can't add, RSC '%s' already present in the rsc list (%d active resources)",
	rsc->rsc_id, g_hash_table_size(rsc_list));

	free_rsc(rsc);
	return rc;
	}

	g_hash_table_replace(rsc_list, rsc->rsc_id, rsc);
	crm_info("Added '%s' to the rsc list (%d active resources)",
	rsc->rsc_id, g_hash_table_size(rsc_list));

	return rc;
	}

	static void
	process_lrmd_get_rsc_info(lrmd_client_t * client, xmlNode * request)
	{
	int rc = pcmk_ok;
	int send_rc = 0;
	int call_id = 0;
	xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
	xmlNode *reply = NULL;
	lrmd_rsc_t *rsc = NULL;

	crm_element_value_int(request, F_LRMD_CALLID, &call_id);

	if (!rsc_id) {
	rc = -ENODEV;
	goto get_rsc_done;
	}

	if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) {
	crm_info("Resource '%s' not found (%d active resources)",
	rsc_id, g_hash_table_size(rsc_list));
	rc = -ENODEV;
	goto get_rsc_done;
	}

	get_rsc_done:

	reply = create_xml_node(NULL, T_LRMD_REPLY);
	crm_xml_add(reply, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add_int(reply, F_LRMD_RC, rc);
	crm_xml_add_int(reply, F_LRMD_CALLID, call_id);

	if (rsc) {
	crm_xml_add(reply, F_LRMD_RSC_ID, rsc->rsc_id);
	crm_xml_add(reply, F_LRMD_CLASS, rsc->class);
	crm_xml_add(reply, F_LRMD_PROVIDER, rsc->provider);
	crm_xml_add(reply, F_LRMD_TYPE, rsc->type);
	}

	send_rc = crm_ipcs_send(client->channel, reply, FALSE);

	if (send_rc < 0) {
	crm_warn("LRMD reply to %s failed: %d", client->name, send_rc);
	}

	free_xml(reply);
	}

	static int
	process_lrmd_rsc_unregister(lrmd_client_t * client, xmlNode * request)
	{
	int rc = pcmk_ok;
	lrmd_rsc_t *rsc = NULL;
	xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);

	if (!rsc_id) {
	return -ENODEV;
	}

	if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) {
	crm_info("Resource '%s' not found (%d active resources)",
	rsc_id, g_hash_table_size(rsc_list));
	return pcmk_ok;
	}

	if (rsc->active) {
	/* let the caller know there are still active ops on this rsc to watch for */
	crm_trace("Operation still in progress: %p", rsc->active);
	rc = -EINPROGRESS;
	}

	g_hash_table_remove(rsc_list, rsc_id);

	return rc;
	}

	static int
	process_lrmd_rsc_exec(lrmd_client_t * client, xmlNode * request)
	{
	lrmd_rsc_t *rsc = NULL;
	lrmd_cmd_t *cmd = NULL;
	xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);

	if (!rsc_id) {
	return -EINVAL;
	}
	if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) {
	crm_info("Resource '%s' not found (%d active resources)",
	rsc_id, g_hash_table_size(rsc_list));
	return -ENODEV;
	}

	cmd = create_lrmd_cmd(request, client);
	schedule_lrmd_cmd(rsc, cmd);

	return cmd->call_id;
	}

	static int
	cancel_op(const char rsc_id, const char action, int interval)
	{
	GListPtr gIter = NULL;
	lrmd_rsc_t *rsc = g_hash_table_lookup(rsc_list, rsc_id);

	/* How to cancel an action.
	* 1. Check pending ops list, if it hasn't been handed off
	* to the service library or stonith recurring list remove
	* it there and that will stop it.
	* 2. If it isn't in the pending ops list, then its either a
	* recurring op in the stonith recurring list, or the service
	* library's recurring list. Stop it there
	* 3. If not found in any lists, then this operation has either
	* been executed already and is not a recurring operation, or
	* never existed.
	*/
	if (!rsc) {
	return -ENODEV;
	}

	for (gIter = rsc->pending_ops; gIter != NULL; gIter = gIter->next) {
	lrmd_cmd_t *cmd = gIter->data;

	if (safe_str_eq(cmd->action, action) && cmd->interval == interval) {
	cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED;
	cmd_finalize(cmd, rsc);
	return pcmk_ok;
	}
	}

	if (safe_str_eq(rsc->class, "stonith")) {
	/* The service library does not handle stonith operations.
	* We have to handle recurring stonith opereations ourselves. */
	for (gIter = rsc->recurring_ops; gIter != NULL; gIter = gIter->next) {
	lrmd_cmd_t *cmd = gIter->data;

	if (safe_str_eq(cmd->action, action) && cmd->interval == interval) {
	cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED;
	cmd_finalize(cmd, rsc);
	return pcmk_ok;
	}
	}
	} else if (services_action_cancel(rsc_id, normalize_action_name(rsc, action), interval) == TRUE) {
	/* The service library will tell the action_complete callback function
	* this action was cancelled, which will destroy the cmd and remove
	* it from the recurring_op list. Do not do that in this function
	* if the service library says it cancelled it. */
	return pcmk_ok;
	}

	return -EOPNOTSUPP;
	}

	static int
	process_lrmd_rsc_cancel(lrmd_client_t * client, xmlNode * request)
	{
	xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
	const char *action = crm_element_value(rsc_xml, F_LRMD_RSC_ACTION);
	int interval = 0;

	crm_element_value_int(rsc_xml, F_LRMD_RSC_INTERVAL, &interval);

	if (!rsc_id \|\| !action) {
	return -EINVAL;
	}

	return cancel_op(rsc_id, action, interval);
	}

	void
	process_lrmd_message(lrmd_client_t * client, xmlNode * request)
	{
	int rc = pcmk_ok;
	int call_id = 0;
	const char *op = crm_element_value(request, F_LRMD_OPERATION);
	int do_reply = 0;
	int do_notify = 0;
	int exit = 0;

	crm_trace("Processing %s operation from %s", op, client->id);
	crm_element_value_int(request, F_LRMD_CALLID, &call_id);

	if (crm_str_eq(op, CRM_OP_REGISTER, TRUE)) {
	rc = process_lrmd_signon(client, request);
	} else if (crm_str_eq(op, LRMD_OP_RSC_REG, TRUE)) {
	rc = process_lrmd_rsc_register(client, request);
	do_notify = 1;
	do_reply = 1;
	} else if (crm_str_eq(op, LRMD_OP_RSC_INFO, TRUE)) {
	process_lrmd_get_rsc_info(client, request);
	} else if (crm_str_eq(op, LRMD_OP_RSC_UNREG, TRUE)) {
	rc = process_lrmd_rsc_unregister(client, request);
	/* don't notify anyone about failed un-registers */
	if (rc == pcmk_ok \|\| rc == -EINPROGRESS) {
	do_notify = 1;
	}
	do_reply = 1;
	} else if (crm_str_eq(op, LRMD_OP_RSC_EXEC, TRUE)) {
	rc = process_lrmd_rsc_exec(client, request);
	do_reply = 1;
	} else if (crm_str_eq(op, LRMD_OP_RSC_CANCEL, TRUE)) {
	rc = process_lrmd_rsc_cancel(client, request);
	do_reply = 1;
	} else if (crm_str_eq(op, CRM_OP_QUIT, TRUE)) {
	do_reply = 1;
	exit = 1;
	} else {
	rc = -EOPNOTSUPP;
	do_reply = 1;
	crm_err("Unknown %s from %s", op, client->name);
	crm_log_xml_warn(request, "UnknownOp");
	}

	crm_debug("Processed %s operation from %s: rc=%d, reply=%d, notify=%d, exit=%d",
	op, client->id, rc, do_reply, do_notify, exit);

	if (do_reply) {
	send_reply(client, rc, call_id);
	}

	if (do_notify) {
	send_generic_notify(rc, request);
	}

	if (exit) {
	lrmd_shutdown(0);
	}
	}
	diff --git a/pengine/native.c b/pengine/native.c
	index 74b21d8ee9..c907002c5f 100644
	--- a/pengine/native.c
	+++ b/pengine/native.c
	@@ -1,3110 +1,3145 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <pengine.h>
	#include <crm/pengine/rules.h>
	#include <crm/msg_xml.h>
	#include <allocate.h>
	#include <utils.h>

	#define DELETE_THEN_REFRESH 1 /* The crmd will remove the resource from the CIB itself, making this redundant */
	#define INFINITY_HACK (INFINITY * -100)

	#define VARIANT_NATIVE 1
	#include <lib/pengine/variant.h>

	void native_rsc_colocation_rh_must(resource_t * rsc_lh, gboolean update_lh,
	resource_t * rsc_rh, gboolean update_rh);

	void native_rsc_colocation_rh_mustnot(resource_t * rsc_lh, gboolean update_lh,
	resource_t * rsc_rh, gboolean update_rh);

	void Recurring(resource_t * rsc, action_t * start, node_t * node, pe_working_set_t * data_set);
	void RecurringOp(resource_t * rsc, action_t * start, node_t * node,
	xmlNode * operation, pe_working_set_t * data_set);
	void Recurring_Stopped(resource_t * rsc, action_t * start, node_t * node,
	pe_working_set_t * data_set);
	void RecurringOp_Stopped(resource_t * rsc, action_t * start, node_t * node,
	xmlNode * operation, pe_working_set_t * data_set);
	void pe_post_notify(resource_t * rsc, node_t * node, action_t * op,
	notify_data_t * n_data, pe_working_set_t * data_set);

	gboolean DeleteRsc(resource_t * rsc, node_t * node, gboolean optional, pe_working_set_t * data_set);
	gboolean StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean StartRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean DemoteRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean PromoteRsc(resource_t * rsc, node_t * next, gboolean optional,
	pe_working_set_t * data_set);
	gboolean RoleError(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean NullOp(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);

	/* INDENT-OFF */
	enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* Current State */
	/* Next State: Unknown Stopped Started Slave Master */
	/* Unknown */ { RSC_ROLE_UNKNOWN, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, },
	/* Stopped */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, },
	/* Started */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
	/* Slave */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_UNKNOWN, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
	/* Master */ { RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_UNKNOWN, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
	};

	gboolean (rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX])(resource_t,node_t,gboolean,pe_working_set_t) = {
	/* Current State */
	/* Next State: Unknown Stopped Started Slave Master */
	/* Unknown */ { RoleError, StopRsc, RoleError, RoleError, RoleError, },
	/* Stopped */ { RoleError, NullOp, StartRsc, StartRsc, RoleError, },
	/* Started */ { RoleError, StopRsc, NullOp, NullOp, PromoteRsc, },
	/* Slave */ { RoleError, StopRsc, RoleError, NullOp, PromoteRsc, },
	/* Master */ { RoleError, DemoteRsc, RoleError, DemoteRsc, NullOp, },
	};
	/* INDENT-ON */

	struct capacity_data {
	node_t *node;
	resource_t *rsc;
	gboolean is_enough;
	};

	static void
	check_capacity(gpointer key, gpointer value, gpointer user_data)
	{
	int required = 0;
	int remaining = 0;
	struct capacity_data *data = user_data;

	required = crm_parse_int(value, "0");
	remaining = crm_parse_int(g_hash_table_lookup(data->node->details->utilization, key), "0");

	if (required > remaining) {
	crm_debug("Node %s has no enough %s for resource %s: required=%d remaining=%d",
	data->node->details->uname, (char *)key, data->rsc->id, required, remaining);
	data->is_enough = FALSE;
	}
	}

	static gboolean
	have_enough_capacity(node_t * node, resource_t * rsc)
	{
	struct capacity_data data;

	data.node = node;
	data.rsc = rsc;
	data.is_enough = TRUE;

	g_hash_table_foreach(rsc->utilization, check_capacity, &data);

	return data.is_enough;
	}

	static gboolean
	native_choose_node(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
	{
	/*
	1. Sort by weight
	2. color.chosen_node = the node (of those with the highest wieght)
	with the fewest resources
	3. remove color.chosen_node from all other colors
	*/
	int alloc_details = scores_log_level + 1;

	GListPtr nodes = NULL;
	node_t *chosen = NULL;

	int lpc = 0;
	int multiple = 0;
	int length = 0;
	gboolean result = FALSE;

	if (safe_str_neq(data_set->placement_strategy, "default")) {
	GListPtr gIter = NULL;

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	if (have_enough_capacity(node, rsc) == FALSE) {
	crm_debug("Resource %s cannot be allocated to node %s: none of enough capacity",
	rsc->id, node->details->uname);
	resource_location(rsc, node, -INFINITY, "__limit_utilization_", data_set);
	}
	}
	dump_node_scores(alloc_details, rsc, "Post-utilization", rsc->allowed_nodes);
	}

	length = g_hash_table_size(rsc->allowed_nodes);

	if (is_not_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->allocated_to ? TRUE : FALSE;
	}

	if (prefer) {
	chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);
	if (chosen && chosen->weight >= 0 && can_run_resources(chosen)) {
	crm_trace("Using preferred node %s for %s instead of choosing from %d candidates",
	chosen->details->uname, rsc->id, length);
	} else if (chosen && chosen->weight < 0) {
	crm_trace("Preferred node %s for %s was unavailable", chosen->details->uname, rsc->id);
	chosen = NULL;
	} else if (chosen && can_run_resources(chosen)) {
	crm_trace("Preferred node %s for %s was unsuitable", chosen->details->uname, rsc->id);
	chosen = NULL;
	} else {
	crm_trace("Preferred node %s for %s was unknown", prefer->details->uname, rsc->id);
	}
	}

	if (chosen == NULL && rsc->allowed_nodes) {
	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	nodes = g_list_sort_with_data(nodes, sort_node_weight, g_list_nth_data(rsc->running_on, 0));

	chosen = g_list_nth_data(nodes, 0);
	crm_trace("Chose node %s for %s from %d candidates",
	chosen ? chosen->details->uname : "<none>", rsc->id, length);

	if (chosen && chosen->weight > 0 && can_run_resources(chosen)) {
	node_t *running = g_list_nth_data(rsc->running_on, 0);

	if (running && can_run_resources(running) == FALSE) {
	crm_trace("Current node for %s (%s) can't run resources",
	rsc->id, running->details->uname);
	running = NULL;
	}

	for (lpc = 1; lpc < length && running; lpc++) {
	node_t *tmp = g_list_nth_data(nodes, lpc);

	if (tmp->weight == chosen->weight) {
	multiple++;
	if (tmp->details == running->details) {
	/* prefer the existing node if scores are equal */
	chosen = tmp;
	}
	}
	}
	}
	}

	if (multiple > 1) {
	int log_level = LOG_INFO;
	char *score = score2char(chosen->weight);

	if (chosen->weight >= INFINITY) {
	log_level = LOG_WARNING;
	}

	do_crm_log(log_level, "%d nodes with equal score (%s) for"
	" running %s resources. Chose %s.",
	multiple, score, rsc->id, chosen->details->uname);
	free(score);
	}

	result = native_assign_node(rsc, nodes, chosen, FALSE);
	g_list_free(nodes);
	return result;
	}

	static int
	node_list_attr_score(GHashTable * list, const char attr, const char value)
	{
	GHashTableIter iter;
	node_t *node = NULL;
	int best_score = -INFINITY;
	const char *best_node = NULL;

	if (attr == NULL) {
	attr = "#" XML_ATTR_UNAME;
	}

	g_hash_table_iter_init(&iter, list);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	int weight = node->weight;

	if (can_run_resources(node) == FALSE) {
	weight = -INFINITY;
	}
	if (weight > best_score \|\| best_node == NULL) {
	const char *tmp = g_hash_table_lookup(node->details->attrs, attr);

	if (safe_str_eq(value, tmp)) {
	best_score = weight;
	best_node = node->details->uname;
	}
	}
	}

	if (safe_str_neq(attr, "#" XML_ATTR_UNAME)) {
	crm_info("Best score for %s=%s was %s with %d",
	attr, value, best_node ? best_node : "<none>", best_score);
	}

	return best_score;
	}

	static void
	node_hash_update(GHashTable * list1, GHashTable * list2, const char *attr, int factor,
	gboolean only_positive)
	{
	int score = 0;
	int new_score = 0;
	GHashTableIter iter;
	node_t *node = NULL;

	if (attr == NULL) {
	attr = "#" XML_ATTR_UNAME;
	}

	g_hash_table_iter_init(&iter, list1);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	CRM_CHECK(node != NULL, continue);
	score = node_list_attr_score(list2, attr, g_hash_table_lookup(node->details->attrs, attr));
	new_score = merge_weights(factor * score, node->weight);

	if (factor < 0 && score < 0) {
	/* Negative preference for a node with a negative score
	* should not become a positive preference
	*
	* TODO: Decide if we want to filter only if weight == -INFINITY
	*
	*/
	crm_trace("%s: Filtering %d + %d%d (factor score)",
	node->details->uname, node->weight, factor, score);

	} else if (node->weight == INFINITY_HACK) {
	crm_trace("%s: Filtering %d + %d*%d (node < 0)",
	node->details->uname, node->weight, factor, score);

	} else if (only_positive && new_score < 0 && node->weight > 0) {
	node->weight = INFINITY_HACK;
	crm_trace("%s: Filtering %d + %d*%d (score > 0)",
	node->details->uname, node->weight, factor, score);

	} else if (only_positive && new_score < 0 && node->weight == 0) {
	crm_trace("%s: Filtering %d + %d*%d (score == 0)",
	node->details->uname, node->weight, factor, score);

	} else {
	crm_trace("%s: %d + %d*%d", node->details->uname, node->weight, factor, score);
	node->weight = new_score;
	}
	}
	}

	static GHashTable *
	node_hash_dup(GHashTable * hash)
	{
	/* Hack! */
	GListPtr list = g_hash_table_get_values(hash);
	GHashTable *result = node_hash_from_list(list);

	g_list_free(list);
	return result;
	}

	GHashTable *
	native_merge_weights(resource_t * rsc, const char rhs, GHashTable nodes, const char *attr,
	int factor, gboolean allow_rollback, gboolean only_positive)
	{
	enum pe_weights flags = pe_weights_none;

	if (only_positive) {
	set_bit(flags, pe_weights_positive);
	}
	if (allow_rollback) {
	set_bit(flags, pe_weights_rollback);
	}
	return rsc_merge_weights(rsc, rhs, nodes, attr, factor, flags);
	}

	GHashTable *
	rsc_merge_weights(resource_t * rsc, const char rhs, GHashTable nodes, const char *attr,
	int factor, enum pe_weights flags)
	{
	GHashTable *work = NULL;
	int multiplier = 1;

	if (factor < 0) {
	multiplier = -1;
	}

	if (is_set(rsc->flags, pe_rsc_merging)) {
	crm_info("%s: Breaking dependency loop at %s", rhs, rsc->id);
	return nodes;
	}

	set_bit(rsc->flags, pe_rsc_merging);

	if (is_set(flags, pe_weights_init)) {
	if (rsc->variant == pe_group && rsc->children) {
	GListPtr last = rsc->children;

	while (last->next != NULL) {
	last = last->next;
	}

	crm_trace("Merging %s as a group %p %p", rsc->id, rsc->children, last);
	work = rsc_merge_weights(last->data, rhs, NULL, attr, factor, flags);

	} else {
	work = node_hash_dup(rsc->allowed_nodes);
	}
	clear_bit(flags, pe_weights_init);

	} else {
	crm_trace("%s: Combining scores from %s", rhs, rsc->id);
	work = node_hash_dup(nodes);
	node_hash_update(work, rsc->allowed_nodes, attr, factor,
	is_set(flags, pe_weights_positive));
	}

	if (is_set(flags, pe_weights_rollback) && can_run_any(work) == FALSE) {
	crm_info("%s: Rolling back scores from %s", rhs, rsc->id);
	g_hash_table_destroy(work);
	clear_bit(rsc->flags, pe_rsc_merging);
	return nodes;
	}

	if (can_run_any(work)) {
	GListPtr gIter = NULL;

	if (is_set(flags, pe_weights_forward)) {
	gIter = rsc->rsc_cons;
	} else {
	gIter = rsc->rsc_cons_lhs;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t *other = NULL;
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	if (is_set(flags, pe_weights_forward)) {
	other = constraint->rsc_rh;
	} else {
	other = constraint->rsc_lh;
	}

	crm_trace("Applying %s (%s)", constraint->id, other->id);
	work = rsc_merge_weights(other, rhs, work, constraint->node_attribute,
	multiplier * constraint->score / INFINITY, flags);
	dump_node_scores(LOG_TRACE, NULL, rhs, work);
	}

	}

	if(is_set(flags, pe_weights_positive)) {
	node_t *node = NULL;
	GHashTableIter iter;
	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node->weight == INFINITY_HACK) {
	node->weight = 1;
	}
	}
	}

	if (nodes) {
	g_hash_table_destroy(nodes);
	}

	clear_bit(rsc->flags, pe_rsc_merging);
	return work;
	}

	node_t *
	native_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	int alloc_details = scores_log_level + 1;
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (rsc->parent && is_not_set(rsc->parent->flags, pe_rsc_allocating)) {
	/* never allocate children on their own */
	crm_debug("Escalating allocation of %s to its parent: %s", rsc->id, rsc->parent->id);
	rsc->parent->cmds->allocate(rsc->parent, prefer, data_set);
	}

	if (is_not_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->allocated_to;
	}

	if (is_set(rsc->flags, pe_rsc_allocating)) {
	crm_debug("Dependency loop detected involving %s", rsc->id);
	return NULL;
	}

	set_bit(rsc->flags, pe_rsc_allocating);
	print_resource(alloc_details, "Allocating: ", rsc, FALSE);
	dump_node_scores(alloc_details, rsc, "Pre-allloc", rsc->allowed_nodes);

	for (gIter = rsc->rsc_cons; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	GHashTable *archive = NULL;
	resource_t *rsc_rh = constraint->rsc_rh;

	crm_trace("%s: Pre-Processing %s (%s, %d, %s)",
	rsc->id, constraint->id, rsc_rh->id,
	constraint->score, role2text(constraint->role_lh));
	if (constraint->role_lh >= RSC_ROLE_MASTER
	\|\| (constraint->score < 0 && constraint->score > -INFINITY)) {
	archive = node_hash_dup(rsc->allowed_nodes);
	}
	rsc_rh->cmds->allocate(rsc_rh, NULL, data_set);
	rsc->cmds->rsc_colocation_lh(rsc, rsc_rh, constraint);
	if (archive && can_run_any(rsc->allowed_nodes) == FALSE) {
	crm_info("%s: Rolling back scores from %s", rsc->id, rsc_rh->id);
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = archive;
	archive = NULL;
	}
	if (archive) {
	g_hash_table_destroy(archive);
	}
	}

	dump_node_scores(alloc_details, rsc, "Post-coloc", rsc->allowed_nodes);

	for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	rsc->allowed_nodes =
	constraint->rsc_lh->cmds->merge_weights(constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
	constraint->node_attribute,
	constraint->score / INFINITY, TRUE, FALSE);
	}

	for (gIter = rsc->rsc_tickets; gIter != NULL; gIter = gIter->next) {
	rsc_ticket_t rsc_ticket = (rsc_ticket_t ) gIter->data;

	if (rsc_ticket->ticket->granted == FALSE \|\| rsc_ticket->ticket->standby) {
	rsc_ticket_constraint(rsc, rsc_ticket, data_set);
	}
	}

	print_resource(LOG_DEBUG_2, "Allocating: ", rsc, FALSE);
	if (rsc->next_role == RSC_ROLE_STOPPED) {
	crm_trace("Making sure %s doesn't get allocated", rsc->id);
	/* make sure it doesnt come up again */
	resource_location(rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE, data_set);
	}

	dump_node_scores(show_scores ? 0 : scores_log_level, rsc, __PRETTY_FUNCTION__,
	rsc->allowed_nodes);
	if (is_set(data_set->flags, pe_flag_stonith_enabled)
	&& is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
	clear_bit(rsc->flags, pe_rsc_managed);
	}

	if (is_not_set(rsc->flags, pe_rsc_managed)) {
	const char *reason = NULL;
	node_t *assign_to = NULL;

	rsc->next_role = rsc->role;
	if (rsc->running_on == NULL) {
	reason = "inactive";
	} else if (rsc->role == RSC_ROLE_MASTER) {
	assign_to = rsc->running_on->data;
	reason = "master";
	} else if (is_set(rsc->flags, pe_rsc_failed)) {
	reason = "failed";
	} else {
	assign_to = rsc->running_on->data;
	reason = "active";
	}
	crm_info("Unmanaged resource %s allocated to %s: %s", rsc->id,
	assign_to ? assign_to->details->uname : "'nowhere'", reason);
	native_assign_node(rsc, NULL, assign_to, TRUE);

	} else if (is_set(data_set->flags, pe_flag_stop_everything)
	&& safe_str_neq(class, "stonith")) {
	crm_debug("Forcing %s to stop", rsc->id);
	native_assign_node(rsc, NULL, NULL, TRUE);

	} else if (is_set(rsc->flags, pe_rsc_provisional)
	&& native_choose_node(rsc, prefer, data_set)) {
	crm_trace("Allocated resource %s to %s", rsc->id, rsc->allocated_to->details->uname);

	} else if (rsc->allocated_to == NULL) {
	if (is_not_set(rsc->flags, pe_rsc_orphan)) {
	crm_info("Resource %s cannot run anywhere", rsc->id);
	} else if (rsc->running_on != NULL) {
	crm_info("Stopping orphan resource %s", rsc->id);
	}

	} else {
	crm_debug("Pre-Allocated resource %s to %s", rsc->id, rsc->allocated_to->details->uname);
	}

	clear_bit(rsc->flags, pe_rsc_allocating);
	print_resource(LOG_DEBUG_3, "Allocated ", rsc, TRUE);

	return rsc->allocated_to;
	}

	static gboolean
	is_op_dup(resource_t * rsc, const char name, const char interval)
	{
	gboolean dup = FALSE;
	const char *id = NULL;
	const char *value = NULL;
	xmlNode *operation = NULL;

	for (operation = __xml_first_child(rsc->ops_xml); operation != NULL;
	operation = __xml_next(operation)) {
	if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
	value = crm_element_value(operation, "name");
	if (safe_str_neq(value, name)) {
	continue;
	}

	value = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	if (value == NULL) {
	value = "0";
	}

	if (safe_str_neq(value, interval)) {
	continue;
	}

	if (id == NULL) {
	id = ID(operation);

	} else {
	crm_config_err("Operation %s is a duplicate of %s", ID(operation), id);
	crm_config_err
	("Do not use the same (name, interval) combination more than once per resource");
	dup = TRUE;
	}
	}
	}

	return dup;
	}

	void
	RecurringOp(resource_t * rsc, action_t * start, node_t * node,
	xmlNode * operation, pe_working_set_t * data_set)
	{
	char *key = NULL;
	const char *name = NULL;
	const char *value = NULL;
	const char *interval = NULL;
	const char *node_uname = NULL;

	unsigned long long interval_ms = 0;
	action_t *mon = NULL;
	gboolean is_optional = TRUE;
	GListPtr possible_matches = NULL;

	/* Only process for the operations without role="Stopped" */
	value = crm_element_value(operation, "role");
	if (value && text2role(value) == RSC_ROLE_STOPPED) {
	return;
	}

	crm_trace("Creating recurring action %s for %s in role %s on %s",
	ID(operation), rsc->id, role2text(rsc->next_role),
	node ? node->details->uname : "n/a");

	if (node != NULL) {
	node_uname = node->details->uname;
	}

	interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	interval_ms = crm_get_interval(interval);

	if (interval_ms == 0) {
	return;
	}

	name = crm_element_value(operation, "name");
	if (is_op_dup(rsc, name, interval)) {
	return;
	}

	if (safe_str_eq(name, RSC_STOP)
	\|\| safe_str_eq(name, RSC_START)
	\|\| safe_str_eq(name, RSC_DEMOTE)
	\|\| safe_str_eq(name, RSC_PROMOTE)
	) {
	crm_config_err("Invalid recurring action %s wth name: '%s'", ID(operation), name);
	return;
	}

	key = generate_op_key(rsc->id, name, interval_ms);
	if (find_rsc_op_entry(rsc, key) == NULL) {
	/* disabled */
	free(key);
	return;
	}

	if (start != NULL) {
	crm_trace("Marking %s %s due to %s",
	key, is_set(start->flags, pe_action_optional) ? "optional" : "manditory",
	start->uuid);
	is_optional = (rsc->cmds->action_flags(start, NULL) & pe_action_optional);
	} else {
	crm_trace("Marking %s optional", key);
	is_optional = TRUE;
	}

	/* start a monitor for an already active resource */
	possible_matches = find_actions_exact(rsc->actions, key, node);
	if (possible_matches == NULL) {
	is_optional = FALSE;
	crm_trace("Marking %s manditory: not active", key);
	} else {
	g_list_free(possible_matches);
	}

	if ((rsc->next_role == RSC_ROLE_MASTER && value == NULL)
	\|\| (value != NULL && text2role(value) != rsc->next_role)) {
	int log_level = LOG_DEBUG_2;
	const char *result = "Ignoring";

	if (is_optional) {
	char *local_key = strdup(key);

	log_level = LOG_INFO;
	result = "Cancelling";
	/* its running : cancel it */

	mon = custom_action(rsc, local_key, RSC_CANCEL, node, FALSE, TRUE, data_set);

	free(mon->task);
	mon->task = strdup(RSC_CANCEL);
	add_hash_param(mon->meta, XML_LRM_ATTR_INTERVAL, interval);
	add_hash_param(mon->meta, XML_LRM_ATTR_TASK, name);

	local_key = NULL;

	switch (rsc->role) {
	case RSC_ROLE_SLAVE:
	case RSC_ROLE_STARTED:
	if (rsc->next_role == RSC_ROLE_MASTER) {
	local_key = promote_key(rsc);

	} else if (rsc->next_role == RSC_ROLE_STOPPED) {
	local_key = stop_key(rsc);
	}

	break;
	case RSC_ROLE_MASTER:
	local_key = demote_key(rsc);
	break;
	default:
	break;
	}

	if (local_key) {
	custom_action_order(rsc, NULL, mon, rsc, local_key, NULL,
	pe_order_runnable_left, data_set);
	}

	mon = NULL;
	}

	do_crm_log(log_level, "%s action %s (%s vs. %s)",
	result, key, value ? value : role2text(RSC_ROLE_SLAVE),
	role2text(rsc->next_role));

	free(key);
	key = NULL;
	return;
	}

	mon = custom_action(rsc, key, name, node, is_optional, TRUE, data_set);
	key = mon->uuid;
	if (is_optional) {
	crm_trace("%s\t %s (optional)", crm_str(node_uname), mon->uuid);
	}

	if (start == NULL \|\| is_set(start->flags, pe_action_runnable) == FALSE) {
	crm_debug("%s\t %s (cancelled : start un-runnable)", crm_str(node_uname), mon->uuid);
	update_action_flags(mon, pe_action_runnable \| pe_action_clear);

	} else if (node == NULL \|\| node->details->online == FALSE \|\| node->details->unclean) {
	crm_debug("%s\t %s (cancelled : no node available)", crm_str(node_uname), mon->uuid);
	update_action_flags(mon, pe_action_runnable \| pe_action_clear);

	} else if (is_set(mon->flags, pe_action_optional) == FALSE) {
	crm_info(" Start recurring %s (%llus) for %s on %s", mon->task, interval_ms / 1000,
	rsc->id, crm_str(node_uname));
	}

	if (rsc->next_role == RSC_ROLE_MASTER) {
	char *running_master = crm_itoa(PCMK_EXECRA_RUNNING_MASTER);

	add_hash_param(mon->meta, XML_ATTR_TE_TARGET_RC, running_master);
	free(running_master);
	}

	if (node == NULL \|\| is_set(rsc->flags, pe_rsc_managed)) {
	custom_action_order(rsc, start_key(rsc), NULL,
	NULL, strdup(key), mon,
	pe_order_implies_then \| pe_order_runnable_left, data_set);

	if (rsc->next_role == RSC_ROLE_MASTER) {
	custom_action_order(rsc, promote_key(rsc), NULL,
	rsc, NULL, mon,
	pe_order_optional \| pe_order_runnable_left, data_set);

	} else if (rsc->role == RSC_ROLE_MASTER) {
	custom_action_order(rsc, demote_key(rsc), NULL,
	rsc, NULL, mon,
	pe_order_optional \| pe_order_runnable_left, data_set);
	}
	}
	}

	void
	Recurring(resource_t * rsc, action_t * start, node_t * node, pe_working_set_t * data_set)
	{
	if (is_not_set(data_set->flags, pe_flag_maintenance_mode)) {
	xmlNode *operation = NULL;

	for (operation = __xml_first_child(rsc->ops_xml); operation != NULL;
	operation = __xml_next(operation)) {
	if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
	RecurringOp(rsc, start, node, operation, data_set);
	}
	}
	}
	}

	void
	RecurringOp_Stopped(resource_t * rsc, action_t * start, node_t * node,
	xmlNode * operation, pe_working_set_t * data_set)
	{
	char *key = NULL;
	const char *name = NULL;
	const char *role = NULL;
	const char *interval = NULL;
	const char *node_uname = NULL;

	unsigned long long interval_ms = 0;
	GListPtr possible_matches = NULL;
	GListPtr gIter = NULL;

	/* TODO: Support of non-unique clone */
	if (is_set(rsc->flags, pe_rsc_unique) == FALSE) {
	return;
	}

	/* Only process for the operations with role="Stopped" */
	role = crm_element_value(operation, "role");
	if (role == NULL \|\| text2role(role) != RSC_ROLE_STOPPED) {
	return;
	}

	crm_trace
	("Creating recurring actions %s for %s in role %s on nodes where it'll not be running",
	ID(operation), rsc->id, role2text(rsc->next_role));

	if (node != NULL) {
	node_uname = node->details->uname;
	}

	interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	interval_ms = crm_get_interval(interval);

	if (interval_ms == 0) {
	return;
	}

	name = crm_element_value(operation, "name");
	if (is_op_dup(rsc, name, interval)) {
	return;
	}

	if (safe_str_eq(name, RSC_STOP)
	\|\| safe_str_eq(name, RSC_START)
	\|\| safe_str_eq(name, RSC_DEMOTE)
	\|\| safe_str_eq(name, RSC_PROMOTE)
	) {
	crm_config_err("Invalid recurring action %s wth name: '%s'", ID(operation), name);
	return;
	}

	key = generate_op_key(rsc->id, name, interval_ms);
	if (find_rsc_op_entry(rsc, key) == NULL) {
	/* disabled */
	free(key);
	return;
	}

	/* if the monitor exists on the node where the resource will be running, cancel it */
	if (node != NULL) {
	possible_matches = find_actions_exact(rsc->actions, key, node);
	if (possible_matches) {
	action_t *cancel_op = NULL;
	char *local_key = strdup(key);

	g_list_free(possible_matches);

	cancel_op = custom_action(rsc, local_key, RSC_CANCEL, node, FALSE, TRUE, data_set);

	free(cancel_op->task);
	cancel_op->task = strdup(RSC_CANCEL);
	add_hash_param(cancel_op->meta, XML_LRM_ATTR_INTERVAL, interval);
	add_hash_param(cancel_op->meta, XML_LRM_ATTR_TASK, name);

	local_key = NULL;

	if (rsc->next_role == RSC_ROLE_STARTED \|\| rsc->next_role == RSC_ROLE_SLAVE) {
	/* rsc->role == RSC_ROLE_STOPPED: cancel the monitor before start */
	/* rsc->role == RSC_ROLE_STARTED: for a migration, cancel the monitor on the target node before start */
	custom_action_order(rsc, NULL, cancel_op, rsc, start_key(rsc), NULL,
	pe_order_runnable_left, data_set);
	}

	crm_info("Cancel action %s (%s vs. %s) on %s",
	key, role, role2text(rsc->next_role), crm_str(node_uname));
	}
	}

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	node_t stop_node = (node_t ) gIter->data;
	const char *stop_node_uname = stop_node->details->uname;
	gboolean is_optional = TRUE;
	gboolean probe_is_optional = TRUE;
	gboolean stop_is_optional = TRUE;
	action_t *stopped_mon = NULL;
	char *rc_inactive = NULL;
	GListPtr probe_complete_ops = NULL;
	GListPtr stop_ops = NULL;
	GListPtr local_gIter = NULL;
	char *stop_op_key = NULL;

	if (node_uname && safe_str_eq(stop_node_uname, node_uname)) {
	continue;
	}

	crm_trace("Creating recurring action %s for %s on %s",
	ID(operation), rsc->id, crm_str(stop_node_uname));

	/* start a monitor for an already stopped resource */
	possible_matches = find_actions_exact(rsc->actions, key, stop_node);
	if (possible_matches == NULL) {
	crm_trace("Marking %s manditory on %s: not active", key, crm_str(stop_node_uname));
	is_optional = FALSE;
	} else {
	crm_trace("Marking %s optional on %s: already active", key, crm_str(stop_node_uname));
	is_optional = TRUE;
	g_list_free(possible_matches);
	}

	stopped_mon = custom_action(rsc, strdup(key), name, stop_node,
	is_optional, TRUE, data_set);

	rc_inactive = crm_itoa(PCMK_EXECRA_NOT_RUNNING);
	add_hash_param(stopped_mon->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);
	free(rc_inactive);

	probe_complete_ops = find_actions(data_set->actions, CRM_OP_PROBED, NULL);
	for (local_gIter = probe_complete_ops; local_gIter != NULL; local_gIter = local_gIter->next) {
	action_t probe_complete = (action_t ) local_gIter->data;

	if (probe_complete->node == NULL) {
	if (is_set(probe_complete->flags, pe_action_optional) == FALSE) {
	probe_is_optional = FALSE;
	}

	if (is_set(probe_complete->flags, pe_action_runnable) == FALSE) {
	crm_debug("%s\t %s (cancelled : probe un-runnable)",
	crm_str(stop_node_uname), stopped_mon->uuid);
	update_action_flags(stopped_mon, pe_action_runnable \| pe_action_clear);
	}

	if (is_set(rsc->flags, pe_rsc_managed)) {
	custom_action_order(NULL, NULL, probe_complete,
	NULL, strdup(key), stopped_mon,
	pe_order_optional, data_set);
	}
	break;
	}
	}

	if (probe_complete_ops) {
	g_list_free(probe_complete_ops);
	}

	stop_op_key = stop_key(rsc);
	stop_ops = find_actions_exact(rsc->actions, stop_op_key, stop_node);

	for (local_gIter = stop_ops; local_gIter != NULL; local_gIter = local_gIter->next) {
	action_t stop = (action_t ) local_gIter->data;

	if (is_set(stop->flags, pe_action_optional) == FALSE) {
	stop_is_optional = FALSE;
	}

	if (is_set(stop->flags, pe_action_runnable) == FALSE) {
	crm_debug("%s\t %s (cancelled : stop un-runnable)",
	crm_str(stop_node_uname), stopped_mon->uuid);
	update_action_flags(stopped_mon, pe_action_runnable \| pe_action_clear);
	}

	if (is_set(rsc->flags, pe_rsc_managed)) {
	custom_action_order(rsc, strdup(stop_op_key), stop,
	NULL, strdup(key), stopped_mon,
	pe_order_implies_then \| pe_order_runnable_left, data_set);
	}

	}

	if (stop_ops) {
	g_list_free(stop_ops);
	}
	free(stop_op_key);

	if (is_optional == FALSE && probe_is_optional && stop_is_optional
	&& is_set(rsc->flags, pe_rsc_managed) == FALSE) {
	crm_trace("Marking %s optional on %s due to unmanaged",
	key, crm_str(stop_node_uname));
	update_action_flags(stopped_mon, pe_action_optional);
	}

	if (is_set(stopped_mon->flags, pe_action_optional)) {
	crm_trace("%s\t %s (optional)", crm_str(stop_node_uname), stopped_mon->uuid);
	}

	if (stop_node->details->online == FALSE \|\| stop_node->details->unclean) {
	crm_debug("%s\t %s (cancelled : no node available)",
	crm_str(stop_node_uname), stopped_mon->uuid);
	update_action_flags(stopped_mon, pe_action_runnable \| pe_action_clear);
	}

	if (is_set(stopped_mon->flags, pe_action_runnable)
	&& is_set(stopped_mon->flags, pe_action_optional) == FALSE) {
	crm_notice(" Start recurring %s (%llus) for %s on %s", stopped_mon->task,
	interval_ms / 1000, rsc->id, crm_str(stop_node_uname));
	}
	}

	free(key);
	}

	void
	Recurring_Stopped(resource_t * rsc, action_t * start, node_t * node, pe_working_set_t * data_set)
	{
	if (is_not_set(data_set->flags, pe_flag_maintenance_mode)) {
	xmlNode *operation = NULL;

	for (operation = __xml_first_child(rsc->ops_xml); operation != NULL;
	operation = __xml_next(operation)) {
	if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
	RecurringOp_Stopped(rsc, start, node, operation, data_set);
	}
	}
	}
	}

	void
	native_create_actions(resource_t * rsc, pe_working_set_t * data_set)
	{
	action_t *start = NULL;
	node_t *chosen = NULL;
	node_t *current = NULL;
	gboolean need_stop = FALSE;

	GListPtr gIter = NULL;
	int num_active_nodes = g_list_length(rsc->running_on);
	enum rsc_role_e role = RSC_ROLE_UNKNOWN;
	enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;

	chosen = rsc->allocated_to;
	if (chosen != NULL && rsc->next_role == RSC_ROLE_UNKNOWN) {
	rsc->next_role = RSC_ROLE_STARTED;
	crm_trace("Fixed next_role: unknown -> %s", role2text(rsc->next_role));

	} else if (rsc->next_role == RSC_ROLE_UNKNOWN) {
	rsc->next_role = RSC_ROLE_STOPPED;
	crm_trace("Fixed next_role: unknown -> %s", role2text(rsc->next_role));
	}

	crm_trace("Processing state transition for %s: %s->%s", rsc->id,
	role2text(rsc->role), role2text(rsc->next_role));

	if(rsc->running_on) {
	current = rsc->running_on->data;
	}

	get_rsc_attributes(rsc->parameters, rsc, chosen, data_set);

	for (gIter = rsc->dangling_migrations; gIter != NULL; gIter = gIter->next) {
	node_t current = (node_t ) gIter->data;

	action_t *stop = stop_action(rsc, current, FALSE);

	set_bit(stop->flags, pe_action_dangle);
	crm_trace("Forcing a cleanup of %s on %s", rsc->id, current->details->uname);

	if (is_set(data_set->flags, pe_flag_remove_after_stop)) {
	DeleteRsc(rsc, current, FALSE, data_set);
	}
	}

	if (num_active_nodes > 1) {

	if (num_active_nodes == 2
	&& chosen
	&& rsc->partial_migration_target
	&& (chosen->details == rsc->partial_migration_target->details)) {
	/* Here the chosen node is still the migration target from a partial
	* migration. Attempt to continue the migration instead of recovering
	* by stopping the resource everywhere and starting it on a single node. */
	crm_trace("Will attempt to continue with a partial migration to target %s from %s",
	rsc->partial_migration_target->details->id,
	rsc->partial_migration_source->details->id);
	} else {
	const char *type = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	pe_proc_err("Resource %s (%s::%s) is active on %d nodes %s",
	rsc->id, class, type, num_active_nodes,
	recovery2text(rsc->recovery_type));
	crm_warn("See %s for more information.",
	"http://clusterlabs.org/wiki/FAQ#Resource_is_Too_Active");

	if (rsc->recovery_type == recovery_stop_start) {
	need_stop = TRUE;
	}

	/* If by chance a partial migration is in process,
	* but the migration target is not chosen still, clear all
	* partial migration data. */
	rsc->partial_migration_source = rsc->partial_migration_target = NULL;
	}
	}

	if (is_set(rsc->flags, pe_rsc_start_pending)) {
	start = start_action(rsc, chosen, TRUE);
	set_bit(start->flags, pe_action_print_always);
	}

	if(current && chosen && current->details != chosen->details) {
	crm_trace("Moving %s", rsc->id);
	need_stop = TRUE;

	} else if(is_set(rsc->flags, pe_rsc_failed)) {
	crm_trace("Recovering %s", rsc->id);
	need_stop = TRUE;

	} else if(rsc->role > RSC_ROLE_STARTED && current != NULL && chosen != NULL) {
	/* Recovery of a promoted resource */
	start = start_action(rsc, chosen, TRUE);
	if(is_set(start->flags, pe_action_optional) == FALSE) {
	crm_trace("Forced start %s", rsc->id);
	need_stop = TRUE;
	}
	}

	crm_trace("Creating actions for %s: %s->%s", rsc->id,
	role2text(rsc->role), role2text(rsc->next_role));

	role = rsc->role;
	/* Potentiall optional steps on brining the resource down and back up to the same level */
	while (role != RSC_ROLE_STOPPED) {
	next_role = rsc_state_matrix[role][RSC_ROLE_STOPPED];
	crm_trace("Down: Executing: %s->%s (%s)%s", role2text(role), role2text(next_role), rsc->id, need_stop?" required":"");
	if (rsc_action_matrix[role][next_role] (rsc, current, !need_stop, data_set) == FALSE) {
	break;
	}
	role = next_role;
	}

	while (rsc->role <= rsc->next_role && role != rsc->role) {
	next_role = rsc_state_matrix[role][rsc->role];
	crm_trace("Up: Executing: %s->%s (%s)%s", role2text(role), role2text(next_role), rsc->id, need_stop?" required":"");
	if (rsc_action_matrix[role][next_role] (rsc, chosen, !need_stop, data_set) == FALSE) {
	break;
	}
	role = next_role;
	}
	role = rsc->role;

	/* Required steps from this role to the next */
	while (role != rsc->next_role) {
	next_role = rsc_state_matrix[role][rsc->next_role];
	crm_trace("Role: Executing: %s->%s (%s)", role2text(role), role2text(next_role), rsc->id);
	if (rsc_action_matrix[role][next_role] (rsc, chosen, FALSE, data_set) == FALSE) {
	break;
	}
	role = next_role;
	}

	if (rsc->next_role != RSC_ROLE_STOPPED \|\| is_set(rsc->flags, pe_rsc_managed) == FALSE) {
	start = start_action(rsc, chosen, TRUE);
	Recurring(rsc, start, chosen, data_set);
	Recurring_Stopped(rsc, start, chosen, data_set);
	} else {
	Recurring_Stopped(rsc, NULL, NULL, data_set);
	}
	}

	void
	native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
	{
	/* This function is on the critical path and worth optimizing as much as possible */

	resource_t *top = uber_parent(rsc);
	int type = pe_order_optional \| pe_order_implies_then \| pe_order_restart;
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	custom_action_order(rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL,
	rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, type, data_set);

	if (top->variant == pe_master) {
	custom_action_order(rsc, generate_op_key(rsc->id, RSC_DEMOTE, 0), NULL,
	rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL,
	pe_order_implies_first_master, data_set);

	custom_action_order(rsc, generate_op_key(rsc->id, RSC_START, 0), NULL,
	rsc, generate_op_key(rsc->id, RSC_PROMOTE, 0), NULL,
	pe_order_runnable_left, data_set);
	}

	if (is_not_set(rsc->flags, pe_rsc_managed)) {
	crm_trace("Skipping fencing constraints for unmanaged resource: %s", rsc->id);
	return;
	}

	if (safe_str_neq(class, "stonith")) {
	action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);

	custom_action_order(rsc, stop_key(rsc), NULL,
	NULL, strdup(all_stopped->task), all_stopped,
	pe_order_implies_then \| pe_order_runnable_left, data_set);
	}

	if (g_hash_table_size(rsc->utilization) > 0
	&& safe_str_neq(data_set->placement_strategy, "default")) {
	GHashTableIter iter;
	node_t *next = NULL;
	GListPtr gIter = NULL;

	crm_trace("Creating utilization constraints for %s - strategy: %s",
	rsc->id, data_set->placement_strategy);

	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	node_t current = (node_t ) gIter->data;

	char *load_stopped_task = crm_concat(LOAD_STOPPED, current->details->uname, '_');
	action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);

	if (load_stopped->node == NULL) {
	load_stopped->node = node_copy(current);
	update_action_flags(load_stopped, pe_action_optional \| pe_action_clear);
	}

	custom_action_order(rsc, stop_key(rsc), NULL,
	NULL, load_stopped_task, load_stopped, pe_order_load, data_set);
	}

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&next)) {
	char *load_stopped_task = crm_concat(LOAD_STOPPED, next->details->uname, '_');
	action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);

	if (load_stopped->node == NULL) {
	load_stopped->node = node_copy(next);
	update_action_flags(load_stopped, pe_action_optional \| pe_action_clear);
	}

	custom_action_order(NULL, strdup(load_stopped_task), load_stopped,
	rsc, start_key(rsc), NULL, pe_order_load, data_set);

	custom_action_order(NULL, strdup(load_stopped_task), load_stopped,
	rsc, generate_op_key(rsc->id, RSC_MIGRATE, 0), NULL,
	pe_order_load, data_set);

	free(load_stopped_task);
	}
	}
	}

	void
	native_rsc_colocation_lh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
	{
	if (rsc_lh == NULL) {
	pe_err("rsc_lh was NULL for %s", constraint->id);
	return;

	} else if (constraint->rsc_rh == NULL) {
	pe_err("rsc_rh was NULL for %s", constraint->id);
	return;
	}

	crm_trace("Processing colocation constraint between %s and %s", rsc_lh->id, rsc_rh->id);

	rsc_rh->cmds->rsc_colocation_rh(rsc_lh, rsc_rh, constraint);
	}

	static gboolean
	filter_colocation_constraint(resource_t * rsc_lh, resource_t * rsc_rh,
	rsc_colocation_t * constraint)
	{
	if (constraint->score == 0) {
	return FALSE;
	}

	if (constraint->score > 0
	&& constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh != rsc_lh->next_role) {
	crm_trace( "LH: Skipping constraint: \"%s\" state filter",
	role2text(constraint->role_rh));
	return FALSE;
	}

	if (constraint->score > 0
	&& constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh != rsc_rh->next_role) {
	crm_trace( "RH: Skipping constraint: \"%s\" state filter",
	role2text(constraint->role_rh));
	return FALSE;
	}

	if (constraint->score < 0
	&& constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh == rsc_lh->next_role) {
	crm_trace( "LH: Skipping -ve constraint: \"%s\" state filter",
	role2text(constraint->role_rh));
	return FALSE;
	}

	if (constraint->score < 0
	&& constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh == rsc_rh->next_role) {
	crm_trace( "RH: Skipping -ve constraint: \"%s\" state filter",
	role2text(constraint->role_rh));
	return FALSE;
	}

	return TRUE;
	}

	static void
	colocation_match(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
	{
	const char *tmp = NULL;
	const char *value = NULL;
	const char *attribute = "#id";

	GHashTable *work = NULL;
	gboolean do_check = FALSE;

	GHashTableIter iter;
	node_t *node = NULL;

	if (constraint->node_attribute != NULL) {
	attribute = constraint->node_attribute;
	}

	if (rsc_rh->allocated_to) {
	value = g_hash_table_lookup(rsc_rh->allocated_to->details->attrs, attribute);
	do_check = TRUE;

	} else if (constraint->score < 0) {
	/* nothing to do:
	* anti-colocation with something thats not running
	*/
	return;
	}

	work = node_hash_dup(rsc_lh->allowed_nodes);

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	tmp = g_hash_table_lookup(node->details->attrs, attribute);
	if (do_check && safe_str_eq(tmp, value)) {
	if (constraint->score < INFINITY) {
	crm_trace("%s: %s.%s += %d", constraint->id, rsc_lh->id,
	node->details->uname, constraint->score);
	node->weight = merge_weights(constraint->score, node->weight);
	}

	} else if (do_check == FALSE \|\| constraint->score >= INFINITY) {
	crm_trace("%s: %s.%s -= %d (%s)", constraint->id, rsc_lh->id,
	node->details->uname, constraint->score,
	do_check ? "failed" : "unallocated");
	node->weight = merge_weights(-constraint->score, node->weight);
	}
	}

	if (can_run_any(work)
	\|\| constraint->score <= -INFINITY \|\| constraint->score >= INFINITY) {
	g_hash_table_destroy(rsc_lh->allowed_nodes);
	rsc_lh->allowed_nodes = work;
	work = NULL;

	} else {
	char *score = score2char(constraint->score);

	crm_info("%s: Rolling back scores from %s (%d, %s)",
	rsc_lh->id, rsc_rh->id, do_check, score);
	free(score);
	}

	if (work) {
	g_hash_table_destroy(work);
	}
	}

	void
	native_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
	{
	crm_trace("%sColocating %s with %s (%s, weight=%d)",
	constraint->score >= 0 ? "" : "Anti-",
	rsc_lh->id, rsc_rh->id, constraint->id, constraint->score);

	if (filter_colocation_constraint(rsc_lh, rsc_rh, constraint) == FALSE) {
	return;
	}

	if (is_set(rsc_rh->flags, pe_rsc_provisional)) {
	return;

	} else if (is_not_set(rsc_lh->flags, pe_rsc_provisional)) {
	/* error check */
	struct node_shared_s *details_lh;
	struct node_shared_s *details_rh;

	if ((constraint->score > -INFINITY) && (constraint->score < INFINITY)) {
	return;
	}

	details_rh = rsc_rh->allocated_to ? rsc_rh->allocated_to->details : NULL;
	details_lh = rsc_lh->allocated_to ? rsc_lh->allocated_to->details : NULL;

	if (constraint->score == INFINITY && details_lh != details_rh) {
	crm_err("%s and %s are both allocated"
	" but to different nodes: %s vs. %s",
	rsc_lh->id, rsc_rh->id,
	details_lh ? details_lh->uname : "n/a", details_rh ? details_rh->uname : "n/a");

	} else if (constraint->score == -INFINITY && details_lh == details_rh) {
	crm_err("%s and %s are both allocated"
	" but to the SAME node: %s",
	rsc_lh->id, rsc_rh->id, details_rh ? details_rh->uname : "n/a");
	}

	return;

	} else {
	colocation_match(rsc_lh, rsc_rh, constraint);
	}
	}

	static gboolean
	filter_rsc_ticket(resource_t * rsc_lh, rsc_ticket_t * rsc_ticket)
	{
	if (rsc_ticket->role_lh != RSC_ROLE_UNKNOWN && rsc_ticket->role_lh != rsc_lh->role) {
	crm_trace( "LH: Skipping constraint: \"%s\" state filter",
	role2text(rsc_ticket->role_lh));
	return FALSE;
	}

	return TRUE;
	}

	void
	rsc_ticket_constraint(resource_t * rsc_lh, rsc_ticket_t * rsc_ticket, pe_working_set_t * data_set)
	{
	if (rsc_ticket == NULL) {
	pe_err("rsc_ticket was NULL");
	return;
	}

	if (rsc_lh == NULL) {
	pe_err("rsc_lh was NULL for %s", rsc_ticket->id);
	return;
	}

	if (rsc_ticket->ticket->granted && rsc_ticket->ticket->standby == FALSE) {
	return;
	}

	if (rsc_lh->children) {
	GListPtr gIter = rsc_lh->children;

	crm_trace("Processing ticket dependencies from %s", rsc_lh->id);

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	rsc_ticket_constraint(child_rsc, rsc_ticket, data_set);
	}
	return;
	}

	crm_trace("%s: Processing ticket dependency on %s (%s, %s)",
	rsc_lh->id, rsc_ticket->ticket->id, rsc_ticket->id, role2text(rsc_ticket->role_lh));

	if (rsc_ticket->ticket->granted == FALSE && g_list_length(rsc_lh->running_on) > 0) {
	GListPtr gIter = NULL;

	switch (rsc_ticket->loss_policy) {
	case loss_ticket_stop:
	resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
	break;

	case loss_ticket_demote:
	/Promotion score will be set to -INFINITY in master_promotion_order() /
	if (rsc_ticket->role_lh != RSC_ROLE_MASTER) {
	resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
	}
	break;

	case loss_ticket_fence:
	if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) {
	return;
	}

	resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);

	for (gIter = rsc_lh->running_on; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	crm_warn("Node %s will be fenced for deadman", node->details->uname);
	node->details->unclean = TRUE;
	}
	break;

	case loss_ticket_freeze:
	if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) {
	return;
	}
	if (g_list_length(rsc_lh->running_on) > 0) {
	clear_bit(rsc_lh->flags, pe_rsc_managed);
	set_bit(rsc_lh->flags, pe_rsc_block);
	}
	break;
	}

	} else if (rsc_ticket->ticket->granted == FALSE){

	if (rsc_ticket->role_lh != RSC_ROLE_MASTER \|\| rsc_ticket->loss_policy == loss_ticket_stop) {
	resource_location(rsc_lh, NULL, -INFINITY, "__no_ticket__", data_set);
	}

	} else if (rsc_ticket->ticket->standby) {

	if (rsc_ticket->role_lh != RSC_ROLE_MASTER \|\| rsc_ticket->loss_policy == loss_ticket_stop) {
	resource_location(rsc_lh, NULL, -INFINITY, "__ticket_standby__", data_set);
	}
	}
	}

	const char convert_non_atomic_task(char raw_task, resource_t * rsc, gboolean allow_notify);

	const char *
	convert_non_atomic_task(char raw_task, resource_t rsc, gboolean allow_notify)
	{
	int task = no_action;
	const char *atomic_task = raw_task;

	crm_trace("Processing %s for %s", crm_str(raw_task), rsc->id);
	if (raw_task == NULL) {
	return NULL;

	} else if (strstr(raw_task, "notify") != NULL) {
	goto done; /* no conversion */

	} else if (rsc->variant < pe_group) {
	goto done; /* no conversion */
	}

	task = text2task(raw_task);
	switch (task) {
	case stop_rsc:
	case start_rsc:
	case action_notify:
	case action_promote:
	case action_demote:
	break;
	case stopped_rsc:
	case started_rsc:
	case action_notified:
	case action_promoted:
	case action_demoted:
	task--;
	break;
	case monitor_rsc:
	case shutdown_crm:
	case stonith_node:
	goto done;
	break;
	default:
	crm_trace("Unknown action: %s", raw_task);
	goto done;
	break;
	}

	if (task != no_action) {
	if (is_set(rsc->flags, pe_rsc_notify) && allow_notify) {
	/* atomic_task = generate_notify_key(rid, "confirmed-post", task2text(task+1)); */
	crm_err("Not handled");

	} else {
	atomic_task = task2text(task + 1);
	}
	crm_trace("Converted %s -> %s", raw_task, atomic_task);
	}

	done:
	return atomic_task;
	}

	enum pe_action_flags
	native_action_flags(action_t * action, node_t * node)
	{
	return action->flags;
	}

	enum pe_graph_flags
	native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
	enum pe_action_flags filter, enum pe_ordering type)
	{
	/* flags == get_action_flags(first, then_node) called from update_action() */
	enum pe_graph_flags changed = pe_graph_none;
	enum pe_action_flags then_flags = then->flags;
	enum pe_action_flags first_flags = first->flags;

	if (type & pe_order_asymmetrical) {
	resource_t *then_rsc = then->rsc;
	enum rsc_role_e then_rsc_role = then_rsc ? then_rsc->fns->state(then_rsc, TRUE) : 0;

	if (!then_rsc) {
	/* ignore */
	} else if ((then_rsc_role == RSC_ROLE_STOPPED) && safe_str_eq(then->task, RSC_STOP)) {
	/* ignore... if 'then' is supposed to be stopped after 'first', but
	* then is already stopped, there is nothing to be done when non-symmetrical. */
	} else if ((then_rsc_role == RSC_ROLE_STARTED) && safe_str_eq(then->task, RSC_START)) {
	/* ignore... if 'then' is supposed to be started after 'first', but
	* then is already started, there is nothing to be done when non-symmetrical. */
	} else if (!(first->flags & pe_action_runnable)) {
	/* prevent 'then' action from happening if 'first' is not runnable and
	* 'then' has not yet occurred. */
	pe_clear_action_bit(then, pe_action_runnable);
	pe_clear_action_bit(then, pe_action_optional);
	crm_trace("Unset optional and runnable on %s", then->uuid);
	} else {
	/* ignore... then is allowed to start/stop if it wants to. */
	}
	}

	if (type & pe_order_implies_first) {
	if ((filter & pe_action_optional) && (flags & pe_action_optional) == 0) {
	crm_trace("Unset optional on %s because of %s", first->uuid, then->uuid);
	pe_clear_action_bit(first, pe_action_optional);
	}
	}

	if (type & pe_order_implies_first_master) {
	if ((filter & pe_action_optional) &&
	((then->flags & pe_action_optional) == FALSE) &&
	then->rsc && (then->rsc->role == RSC_ROLE_MASTER)) {
	clear_bit(first->flags, pe_action_optional);
	}
	}

	if (is_set(type, pe_order_runnable_left)
	&& is_set(filter, pe_action_runnable)
	&& is_set(then->flags, pe_action_runnable)
	&& is_set(flags, pe_action_runnable) == FALSE) {
	crm_trace("Unset runnable on %s because of %s", then->uuid, first->uuid);
	pe_clear_action_bit(then, pe_action_runnable);
	}

	if (is_set(type, pe_order_implies_then)
	&& is_set(filter, pe_action_optional)
	&& is_set(then->flags, pe_action_optional)
	&& is_set(flags, pe_action_optional) == FALSE) {
	crm_trace("Unset optional on %s because of %s", then->uuid, first->uuid);
	pe_clear_action_bit(then, pe_action_optional);
	}

	if (is_set(type, pe_order_restart)) {
	const char *reason = NULL;

	CRM_ASSERT(first->rsc && first->rsc->variant == pe_native);
	CRM_ASSERT(then->rsc && then->rsc->variant == pe_native);

	if ((filter & pe_action_runnable) && (then->flags & pe_action_runnable) == 0) {
	reason = "shutdown";
	}

	if ((filter & pe_action_optional) && (then->flags & pe_action_optional) == 0) {
	reason = "recover";
	}

	if (reason && is_set(first->flags, pe_action_optional)
	&& is_set(first->flags, pe_action_runnable)) {
	crm_trace("Handling %s: %s -> %s", reason, first->uuid, then->uuid);
	pe_clear_action_bit(first, pe_action_optional);
	}

	if (reason && is_not_set(first->flags, pe_action_optional)
	&& is_not_set(first->flags, pe_action_runnable)) {
	crm_trace("Handling %s: %s -> %s", reason, first->uuid, then->uuid);
	pe_clear_action_bit(then, pe_action_runnable);
	}
	}

	if (then_flags != then->flags) {
	changed \|= pe_graph_updated_then;
	crm_trace("Then: Flags for %s on %s are now 0x%.6x (was 0x%.6x) because of %s 0x%.6x",
	then->uuid, then->node ? then->node->details->uname : "[none]", then->flags,
	then_flags, first->uuid, first->flags);
	}

	if (first_flags != first->flags) {
	changed \|= pe_graph_updated_first;
	crm_trace("First: Flags for %s on %s are now 0x%.6x (was 0x%.6x) because of %s 0x%.6x",
	first->uuid, first->node ? first->node->details->uname : "[none]", first->flags,
	first_flags, then->uuid, then->flags);
	}

	return changed;
	}

	void
	native_rsc_location(resource_t * rsc, rsc_to_node_t * constraint)
	{
	GListPtr gIter = NULL;
	GHashTableIter iter;
	node_t *node = NULL;

	if (constraint == NULL) {
	pe_err("Constraint is NULL");
	return;

	} else if (rsc == NULL) {
	pe_err("LHS of rsc_to_node (%s) is NULL", constraint->id);
	return;
	}

	crm_trace("Applying %s (%s) to %s", constraint->id,
	role2text(constraint->role_filter), rsc->id);

	/* take "lifetime" into account */
	if (constraint->role_filter > 0 && constraint->role_filter != rsc->next_role) {
	crm_debug("Constraint (%s) is not active (role : %s)",
	constraint->id, role2text(constraint->role_filter));
	return;

	} else if (is_active(constraint) == FALSE) {
	crm_trace("Constraint (%s) is not active", constraint->id);
	return;
	}

	if (constraint->node_list_rh == NULL) {
	crm_trace("RHS of constraint %s is NULL", constraint->id);
	return;
	}

	for (gIter = constraint->node_list_rh; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;
	node_t *other_node = NULL;

	other_node = (node_t *) pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (other_node != NULL) {
	crm_trace("%s + %s: %d + %d",
	node->details->uname,
	other_node->details->uname, node->weight, other_node->weight);
	other_node->weight = merge_weights(other_node->weight, node->weight);

	} else {
	node_t *new_node = node_copy(node);

	g_hash_table_insert(rsc->allowed_nodes, (gpointer) new_node->details->id, new_node);
	}
	}

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	crm_trace("%s + %s : %d", rsc->id, node->details->uname, node->weight);
	}
	}

	void
	native_expand(resource_t * rsc, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	crm_trace("Processing actions from %s", rsc->id);

	for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	crm_trace("processing action %d for rsc=%s", action->id, rsc->id);
	graph_element_from_action(action, data_set);
	}

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	child_rsc->cmds->expand(child_rsc, data_set);
	}
	}

	void

	#define log_change(fmt, args...) do { \
	if(terminal) { \
	printf(" * "fmt"\n", ##args); \
	} else { \
	crm_notice(fmt, ##args); \
	} \
	} while(0)

	LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal)
	{
	node_t *next = NULL;
	node_t *current = NULL;

	action_t *stop = NULL;
	action_t *start = NULL;
	+ action_t *demote = NULL;
	+ action_t *promote = NULL;

	char *key = NULL;
	gboolean moving = FALSE;
	GListPtr possible_matches = NULL;

	if (rsc->children) {
	GListPtr gIter = NULL;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	LogActions(child_rsc, data_set, terminal);
	}
	return;
	}

	next = rsc->allocated_to;
	if (rsc->running_on) {
	if (g_list_length(rsc->running_on) > 1 && rsc->partial_migration_source) {
	current = rsc->partial_migration_source;
	} else {
	current = rsc->running_on->data;
	}

	if (rsc->role == RSC_ROLE_STOPPED) {
	/*
	* This can occur when resources are being recovered
	* We fiddle with the current role in native_create_actions()
	*/
	rsc->role = RSC_ROLE_STARTED;
	}
	}

	if (current == NULL && is_set(rsc->flags, pe_rsc_orphan)) {
	/* Don't log stopped orphans */
	return;
	}

	if (is_not_set(rsc->flags, pe_rsc_managed)
	\|\| (current == NULL && next == NULL)) {
	crm_info("Leave %s\t(%s%s)",
	rsc->id, role2text(rsc->role), is_not_set(rsc->flags,
	pe_rsc_managed) ? " unmanaged" : "");
	return;
	}

	if (current != NULL && next != NULL && safe_str_neq(current->details->id, next->details->id)) {
	moving = TRUE;
	}

	- key = stop_key(rsc);
	+ key = start_key(rsc);
	possible_matches = find_actions(rsc->actions, key, next);
	free(key);
	+ if (possible_matches) {
	+ start = possible_matches->data;
	+ g_list_free(possible_matches);
	+ }

	+ key = stop_key(rsc);
	+ possible_matches = find_actions(rsc->actions, key, next);
	+ free(key);
	if (possible_matches) {
	stop = possible_matches->data;
	g_list_free(possible_matches);
	}

	- key = start_key(rsc);
	+ key = promote_key(rsc);
	possible_matches = find_actions(rsc->actions, key, next);
	free(key);
	+ if (possible_matches) {
	+ promote = possible_matches->data;
	+ g_list_free(possible_matches);
	+ }

	+ key = demote_key(rsc);
	+ possible_matches = find_actions(rsc->actions, key, next);
	+ free(key);
	if (possible_matches) {
	- start = possible_matches->data;
	+ demote = possible_matches->data;
	g_list_free(possible_matches);
	}

	if (rsc->role == rsc->next_role) {
	key = generate_op_key(rsc->id, RSC_MIGRATED, 0);
	possible_matches = find_actions(rsc->actions, key, next);
	free(key);

	CRM_CHECK(next != NULL,);
	if (next == NULL) {
	} else if (possible_matches && current) {
	log_change("Migrate %s\t(%s %s -> %s)",
	rsc->id, role2text(rsc->role), current->details->uname,
	next->details->uname);
	g_list_free(possible_matches);

	} else if(is_set(rsc->flags, pe_rsc_reload)) {
	log_change("Reload %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname);

	} else if (start == NULL \|\| is_set(start->flags, pe_action_optional)) {
	crm_info("Leave %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname);

	} else if (moving && current) {
	log_change("Move %s\t(%s %s -> %s)",
	rsc->id, role2text(rsc->role), current->details->uname,
	next->details->uname);

	} else if (is_set(rsc->flags, pe_rsc_failed)) {
	log_change("Recover %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname);

	} else if (start && is_set(start->flags, pe_action_runnable) == FALSE) {
	log_change("Stop %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname);

	} else {
	log_change("Restart %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname);
	}

	return;
	}

	if (rsc->role > RSC_ROLE_SLAVE && rsc->role > rsc->next_role) {
	CRM_CHECK(current != NULL,);
	if (current != NULL) {
	- log_change("Demote %s\t(%s -> %s %s)", rsc->id,
	- role2text(rsc->role), role2text(rsc->next_role), current->details->uname);
	+ gboolean allowed = FALSE;
	+ if (demote != NULL && (demote->flags & pe_action_runnable)) {
	+ allowed = TRUE;
	+ }
	+
	+ log_change("Demote %s\t(%s -> %s %s%s)",
	+ rsc->id,
	+ role2text(rsc->role),
	+ role2text(rsc->next_role),
	+ current->details->uname,
	+ allowed ? "" : " - blocked");
	+
	if (stop != NULL && is_not_set(stop->flags, pe_action_optional)
	&& rsc->next_role > RSC_ROLE_STOPPED) {
	if (is_set(rsc->flags, pe_rsc_failed)) {
	log_change("Recover %s\t(%s %s)",
	rsc->id, role2text(rsc->role), next->details->uname);

	} else if(is_set(rsc->flags, pe_rsc_reload)) {
	log_change("Reload %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname);

	} else {
	log_change("Restart %s\t(%s %s)",
	rsc->id, role2text(rsc->role), next->details->uname);
	}
	}
	}

	} else if (rsc->next_role == RSC_ROLE_STOPPED) {
	GListPtr gIter = NULL;

	CRM_CHECK(current != NULL,);
	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	log_change("Stop %s\t(%s)", rsc->id, node->details->uname);
	}
	}

	if (moving) {
	log_change("Move %s\t(%s %s -> %s)",
	rsc->id, role2text(rsc->next_role), current->details->uname,
	next->details->uname);
	}

	if (rsc->role == RSC_ROLE_STOPPED) {
	gboolean allowed = FALSE;
	if(start && (start->flags & pe_action_runnable)) {
	allowed = TRUE;
	}

	CRM_CHECK(next != NULL,);
	if (next != NULL) {
	log_change("Start %s\t(%s%s)", rsc->id, next->details->uname, allowed?"":" - blocked");
	}
	if(allowed == FALSE) {
	return;
	}
	}

	if (rsc->next_role > RSC_ROLE_SLAVE && rsc->role < rsc->next_role) {
	+ gboolean allowed = FALSE;
	CRM_CHECK(next != NULL,);
	if (stop != NULL && is_not_set(stop->flags, pe_action_optional)
	&& rsc->role > RSC_ROLE_STOPPED) {
	if (is_set(rsc->flags, pe_rsc_failed)) {
	log_change("Recover %s\t(%s %s)",
	rsc->id, role2text(rsc->role), next->details->uname);

	} else if(is_set(rsc->flags, pe_rsc_reload)) {
	log_change("Reload %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname);

	} else {
	log_change("Restart %s\t(%s %s)",
	rsc->id, role2text(rsc->role), next->details->uname);
	}
	}

	- log_change("Promote %s\t(%s -> %s %s)", rsc->id,
	- role2text(rsc->role), role2text(rsc->next_role), next->details->uname);
	+ if (promote && (promote->flags & pe_action_runnable)) {
	+ allowed = TRUE;
	+ }
	+
	+ log_change("Promote %s\t(%s -> %s %s%s)",
	+ rsc->id,
	+ role2text(rsc->role),
	+ role2text(rsc->next_role),
	+ next->details->uname,
	+ allowed ? "" : " - blocked");
	}
	}

	gboolean
	StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	crm_trace("%s", rsc->id);

	if (rsc->next_role == RSC_ROLE_STOPPED
	&& rsc->variant == pe_native && safe_str_eq(class, "stonith")) {
	action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);

	custom_action_order(NULL, strdup(all_stopped->task), all_stopped,
	rsc, stop_key(rsc), NULL,
	pe_order_optional \| pe_order_stonith_stop, data_set);
	}

	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	node_t current = (node_t ) gIter->data;
	action_t *stop;

	if (rsc->partial_migration_target) {
	if(rsc->partial_migration_target->details == current->details) {
	crm_trace("Filtered %s -> %s %s", current->details->uname, next->details->uname, rsc->id);
	continue;
	} else {
	crm_trace("Forced on %s %s", current->details->uname, rsc->id);
	optional = FALSE;
	}
	}

	crm_trace("%s on %s", rsc->id, current->details->uname);
	stop = stop_action(rsc, current, optional);

	if(is_not_set(rsc->flags, pe_rsc_managed)) {
	update_action_flags(stop, pe_action_runnable\|pe_action_clear);
	}

	if (is_set(data_set->flags, pe_flag_remove_after_stop)) {
	DeleteRsc(rsc, current, optional, data_set);
	}
	}

	return TRUE;
	}

	gboolean
	StartRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	action_t *start = NULL;

	crm_trace("%s on %s %d", rsc->id, next?next->details->uname:"N/A", optional);
	start = start_action(rsc, next, TRUE);
	if (is_set(start->flags, pe_action_runnable) && optional == FALSE) {
	update_action_flags(start, pe_action_optional \| pe_action_clear);
	}
	return TRUE;
	}

	gboolean
	PromoteRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	char *key = NULL;
	GListPtr gIter = NULL;
	gboolean runnable = TRUE;
	GListPtr action_list = NULL;

	crm_trace("%s on %s", rsc->id, next?next->details->uname:"N/A");

	CRM_CHECK(next != NULL, return FALSE);

	key = start_key(rsc);
	action_list = find_actions_exact(rsc->actions, key, next);
	free(key);

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	action_t start = (action_t ) gIter->data;

	if (is_set(start->flags, pe_action_runnable) == FALSE) {
	runnable = FALSE;
	}
	}
	g_list_free(action_list);

	if (runnable) {
	promote_action(rsc, next, optional);
	return TRUE;
	}

	crm_debug("%s\tPromote %s (canceled)", next->details->uname, rsc->id);

	key = promote_key(rsc);
	action_list = find_actions_exact(rsc->actions, key, next);
	free(key);

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	action_t promote = (action_t ) gIter->data;

	update_action_flags(promote, pe_action_runnable \| pe_action_clear);
	}

	g_list_free(action_list);
	return TRUE;
	}

	gboolean
	DemoteRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	crm_trace("%s", rsc->id);

	/* CRM_CHECK(rsc->next_role == RSC_ROLE_SLAVE, return FALSE); */
	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	node_t current = (node_t ) gIter->data;

	crm_trace("%s on %s", rsc->id, next?next->details->uname:"N/A");
	demote_action(rsc, current, optional);
	}
	return TRUE;
	}

	gboolean
	RoleError(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	crm_err("%s on %s", rsc->id, next?next->details->uname:"N/A");
	CRM_CHECK(FALSE, return FALSE);
	return FALSE;
	}

	gboolean
	NullOp(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	crm_trace("%s", rsc->id);
	return FALSE;
	}

	gboolean
	DeleteRsc(resource_t * rsc, node_t * node, gboolean optional, pe_working_set_t * data_set)
	{
	action_t *delete = NULL;

	#if DELETE_THEN_REFRESH
	action_t *refresh = NULL;
	#endif
	if (is_set(rsc->flags, pe_rsc_failed)) {
	crm_trace("Resource %s not deleted from %s: failed", rsc->id, node->details->uname);
	return FALSE;

	} else if (node == NULL) {
	crm_trace("Resource %s not deleted: NULL node", rsc->id);
	return FALSE;

	} else if (node->details->unclean \|\| node->details->online == FALSE) {
	crm_trace("Resource %s not deleted from %s: unrunnable", rsc->id, node->details->uname);
	return FALSE;
	}

	crm_notice("Removing %s from %s", rsc->id, node->details->uname);

	delete = delete_action(rsc, node, optional);

	new_rsc_order(rsc, RSC_STOP, rsc, RSC_DELETE,
	optional ? pe_order_implies_then : pe_order_optional, data_set);

	new_rsc_order(rsc, RSC_DELETE, rsc, RSC_START,
	optional ? pe_order_implies_then : pe_order_optional, data_set);

	#if DELETE_THEN_REFRESH
	refresh = custom_action(NULL, strdup(CRM_OP_LRM_REFRESH), CRM_OP_LRM_REFRESH,
	node, FALSE, TRUE, data_set);

	add_hash_param(refresh->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);

	order_actions(delete, refresh, pe_order_optional);
	#endif

	return TRUE;
	}

	#include <../lib/pengine/unpack.h>
	#define set_char(x) last_rsc_id[lpc] = x; complete = TRUE;
	static char *
	increment_clone(char *last_rsc_id)
	{
	int lpc = 0;
	int len = 0;
	char *tmp = NULL;
	gboolean complete = FALSE;

	CRM_CHECK(last_rsc_id != NULL, return NULL);
	if (last_rsc_id != NULL) {
	len = strlen(last_rsc_id);
	}

	lpc = len - 1;
	while (complete == FALSE && lpc > 0) {
	switch (last_rsc_id[lpc]) {
	case 0:
	lpc--;
	break;
	case '0':
	set_char('1');
	break;
	case '1':
	set_char('2');
	break;
	case '2':
	set_char('3');
	break;
	case '3':
	set_char('4');
	break;
	case '4':
	set_char('5');
	break;
	case '5':
	set_char('6');
	break;
	case '6':
	set_char('7');
	break;
	case '7':
	set_char('8');
	break;
	case '8':
	set_char('9');
	break;
	case '9':
	last_rsc_id[lpc] = '0';
	lpc--;
	break;
	case ':':
	tmp = last_rsc_id;
	last_rsc_id = calloc(1, len + 2);
	memcpy(last_rsc_id, tmp, len);
	last_rsc_id[++lpc] = '1';
	last_rsc_id[len] = '0';
	last_rsc_id[len + 1] = 0;
	complete = TRUE;
	free(tmp);
	break;
	default:
	crm_err("Unexpected char: %c (%d)", last_rsc_id[lpc], lpc);
	return NULL;
	break;
	}
	}
	return last_rsc_id;
	}

	static node_t *
	probe_grouped_clone(resource_t * rsc, node_t * node, pe_working_set_t * data_set)
	{
	node_t *running = NULL;
	resource_t *top = uber_parent(rsc);

	if (running == NULL && is_set(top->flags, pe_rsc_unique) == FALSE) {
	/* Annoyingly we also need to check any other clone instances
	* Clumsy, but it will work.
	*
	* An alternative would be to update known_on for every peer
	* during process_rsc_state()
	*
	* This code desperately needs optimization
	* ptest -x with 100 nodes, 100 clones and clone-max=10:
	* No probes O(25s)
	* Detection without clone loop O(3m)
	* Detection with clone loop O(8m)

	ptest[32211]: 2010/02/18_14:27:55 CRIT: stage5: Probing for unknown resources
	ptest[32211]: 2010/02/18_14:33:39 CRIT: stage5: Done
	ptest[32211]: 2010/02/18_14:35:05 CRIT: stage7: Updating action states
	ptest[32211]: 2010/02/18_14:35:05 CRIT: stage7: Done

	*/
	char *clone_id = clone_zero(rsc->id);
	resource_t *peer = pe_find_resource(top->children, clone_id);

	while (peer && running == NULL) {
	running = pe_hash_table_lookup(peer->known_on, node->details->id);
	if (running != NULL) {
	/* we already know the status of the resource on this node */
	crm_trace("Skipping active clone: %s", rsc->id);
	free(clone_id);
	return running;
	}
	clone_id = increment_clone(clone_id);
	peer = pe_find_resource(data_set->resources, clone_id);
	}

	free(clone_id);
	}
	return running;
	}

	gboolean
	native_create_probe(resource_t * rsc, node_t * node, action_t * complete,
	gboolean force, pe_working_set_t * data_set)
	{
	char *key = NULL;
	action_t *probe = NULL;
	node_t *running = NULL;
	resource_t *top = uber_parent(rsc);

	static const char *rc_master = NULL;
	static const char *rc_inactive = NULL;

	if (rc_inactive == NULL) {
	rc_inactive = crm_itoa(PCMK_EXECRA_NOT_RUNNING);
	rc_master = crm_itoa(PCMK_EXECRA_RUNNING_MASTER);
	}

	CRM_CHECK(node != NULL, return FALSE);
	if (force == FALSE && is_not_set(data_set->flags, pe_flag_startup_probes)) {
	crm_trace("Skipping active resource detection for %s", rsc->id);
	return FALSE;
	}

	if (rsc->children) {
	GListPtr gIter = NULL;
	gboolean any_created = FALSE;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	any_created = child_rsc->cmds->create_probe(child_rsc, node, complete, force, data_set)
	\|\| any_created;
	}

	return any_created;
	}

	if (is_set(rsc->flags, pe_rsc_orphan)) {
	crm_trace("Skipping orphan: %s", rsc->id);
	return FALSE;
	}

	running = g_hash_table_lookup(rsc->known_on, node->details->id);
	if (running == NULL && is_set(rsc->flags, pe_rsc_unique) == FALSE) {
	/* Anonymous clones */
	if (rsc->parent == top) {
	running = g_hash_table_lookup(rsc->parent->known_on, node->details->id);

	} else {
	/* Grouped anonymous clones need extra special handling */
	running = probe_grouped_clone(rsc, node, data_set);
	}
	}

	if (force == FALSE && running != NULL) {
	/* we already know the status of the resource on this node */
	crm_trace("Skipping active: %s", rsc->id);
	return FALSE;
	}

	key = generate_op_key(rsc->id, RSC_STATUS, 0);
	probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set);
	update_action_flags(probe, pe_action_optional \| pe_action_clear);

	/*
	* We need to know if it's running_on (not just known_on) this node
	* to correctly determine the target rc.
	*/
	running = pe_find_node_id(rsc->running_on, node->details->id);
	if (running == NULL) {
	add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);

	} else if (rsc->role == RSC_ROLE_MASTER) {
	add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_master);
	}

	crm_debug("Probing %s on %s (%s)", rsc->id, node->details->uname, role2text(rsc->role));
	order_actions(probe, complete, pe_order_implies_then);

	return TRUE;
	}

	static void
	native_start_constraints(resource_t * rsc, action_t * stonith_op, gboolean is_stonith,
	pe_working_set_t * data_set)
	{
	node_t *target = stonith_op ? stonith_op->node : NULL;

	if (is_stonith) {
	char *key = start_key(rsc);
	action_t *ready = get_pseudo_op(STONITH_UP, data_set);

	crm_trace("Ordering %s action before stonith events", key);
	custom_action_order(rsc, key, NULL,
	NULL, strdup(ready->task), ready,
	pe_order_optional \| pe_order_implies_then, data_set);

	} else {
	GListPtr gIter = NULL;
	action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
	action_t *stonith_done = get_pseudo_op(STONITH_DONE, data_set);

	for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	if (action->needs == rsc_req_stonith) {
	order_actions(stonith_done, action, pe_order_optional);

	} else if (target != NULL && safe_str_eq(action->task, RSC_START)
	&& NULL == pe_hash_table_lookup(rsc->known_on, target->details->id)) {
	/* if known == NULL, then we dont know if
	* the resource is active on the node
	* we're about to shoot
	*
	* in this case, regardless of action->needs,
	* the only safe option is to wait until
	* the node is shot before doing anything
	* to with the resource
	*
	* its analogous to waiting for all the probes
	* for rscX to complete before starting rscX
	*
	* the most likely explaination is that the
	* DC died and took its status with it
	*/

	crm_debug("Ordering %s after %s recovery", action->uuid, target->details->uname);
	order_actions(all_stopped, action, pe_order_optional \| pe_order_runnable_left);
	}
	}
	}
	}

	static void
	native_stop_constraints(resource_t * rsc, action_t * stonith_op, gboolean is_stonith,
	pe_working_set_t * data_set)
	{
	char *key = NULL;
	GListPtr gIter = NULL;
	GListPtr action_list = NULL;
	resource_t *top = uber_parent(rsc);

	key = stop_key(rsc);
	action_list = find_actions(rsc->actions, key, stonith_op->node);
	free(key);

	/* add the stonith OP as a stop pre-req and the mark the stop
	* as a pseudo op - since its now redundant
	*/

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	if (action->node->details->online
	&& action->node->details->unclean == FALSE && is_set(rsc->flags, pe_rsc_failed)) {
	continue;
	}

	if (is_set(rsc->flags, pe_rsc_failed)) {
	crm_notice("Stop of failed resource %s is"
	" implicit after %s is fenced", rsc->id, action->node->details->uname);
	} else {
	crm_info("%s is implicit after %s is fenced",
	action->uuid, action->node->details->uname);
	}

	/* the stop would never complete and is
	* now implied by the stonith operation
	*/
	update_action_flags(action, pe_action_pseudo);
	update_action_flags(action, pe_action_runnable);
	update_action_flags(action, pe_action_implied_by_stonith);

	if (is_stonith == FALSE) {
	action_t *parent_stop = find_first_action(top->actions, NULL, RSC_STOP, NULL);

	order_actions(stonith_op, action, pe_order_optional);
	order_actions(stonith_op, parent_stop, pe_order_optional);
	}

	if (is_set(rsc->flags, pe_rsc_notify)) {
	/* Create a second notification that will be delivered
	* immediately after the node is fenced
	*
	* Basic problem:
	* - C is a clone active on the node to be shot and stopping on another
	* - R is a resource that depends on C
	*
	* + C.stop depends on R.stop
	* + C.stopped depends on STONITH
	* + C.notify depends on C.stopped
	* + C.healthy depends on C.notify
	* + R.stop depends on C.healthy
	*
	* The extra notification here changes
	* + C.healthy depends on C.notify
	* into:
	* + C.healthy depends on C.notify'
	* + C.notify' depends on STONITH'
	* thus breaking the loop
	*/
	notify_data_t *n_data =
	create_notification_boundaries(rsc, RSC_STOP, NULL, stonith_op, data_set);
	crm_info("Creating secondary notification for %s", action->uuid);

	collect_notification_data(rsc, TRUE, FALSE, n_data);
	g_hash_table_insert(n_data->keys, strdup("notify_stop_resource"),
	strdup(rsc->id));
	g_hash_table_insert(n_data->keys, strdup("notify_stop_uname"),
	strdup(action->node->details->uname));
	create_notifications(uber_parent(rsc), n_data, data_set);
	free_notification_data(n_data);
	}

	/* From Bug #1601, successful fencing must be an input to a failed resources stop action.

	However given group(rA, rB) running on nodeX and B.stop has failed,
	A := stop healthy resource (rA.stop)
	B := stop failed resource (pseudo operation B.stop)
	C := stonith nodeX
	A requires B, B requires C, C requires A
	This loop would prevent the cluster from making progress.

	This block creates the "C requires A" dependency and therefore must (at least
	for now) be disabled.

	Instead, run the block above and treat all resources on nodeX as B would be
	(marked as a pseudo op depending on the STONITH).

	TODO: Break the "A requires B" dependency in update_action() and re-enable this block

	} else if(is_stonith == FALSE) {
	crm_info("Moving healthy resource %s"
	" off %s before fencing",
	rsc->id, node->details->uname);

	* stop healthy resources before the
	* stonith op
	*
	custom_action_order(
	rsc, stop_key(rsc), NULL,
	NULL,strdup(CRM_OP_FENCE),stonith_op,
	pe_order_optional, data_set);
	*/
	}

	g_list_free(action_list);

	key = demote_key(rsc);
	action_list = find_actions(rsc->actions, key, stonith_op->node);
	free(key);

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	if (action->node->details->online == FALSE \|\| action->node->details->unclean == TRUE
	\|\| is_set(rsc->flags, pe_rsc_failed)) {
	if (is_set(rsc->flags, pe_rsc_failed)) {
	crm_info("Demote of failed resource %s is"
	" implict after %s is fenced", rsc->id, action->node->details->uname);
	} else {
	crm_info("%s is implicit after %s is fenced",
	action->uuid, action->node->details->uname);
	}
	/* the stop would never complete and is
	* now implied by the stonith operation
	*/
	crm_trace("here - 1");
	update_action_flags(action, pe_action_pseudo);
	update_action_flags(action, pe_action_runnable);
	if (is_stonith == FALSE) {
	order_actions(stonith_op, action, pe_order_optional);
	}
	}
	}

	g_list_free(action_list);
	}

	void
	rsc_stonith_ordering(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set)
	{
	gboolean is_stonith = FALSE;
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (rsc->children) {
	GListPtr gIter = NULL;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	rsc_stonith_ordering(child_rsc, stonith_op, data_set);
	}
	return;
	}

	if (is_not_set(rsc->flags, pe_rsc_managed)) {
	crm_trace("Skipping fencing constraints for unmanaged resource: %s", rsc->id);
	return;
	}

	if (stonith_op != NULL && safe_str_eq(class, "stonith")) {
	is_stonith = TRUE;
	}

	/* Start constraints */
	native_start_constraints(rsc, stonith_op, is_stonith, data_set);

	/* Stop constraints */
	native_stop_constraints(rsc, stonith_op, is_stonith, data_set);
	}

	enum stack_activity {
	stack_stable = 0,
	stack_starting = 1,
	stack_stopping = 2,
	stack_middle = 4,
	};

	static enum stack_activity
	find_clone_activity_on(resource_t * rsc, resource_t * target, node_t * node, const char *type)
	{
	int mode = stack_stable;
	action_t *active = NULL;

	if (target->children) {
	GListPtr gIter = NULL;

	for (gIter = target->children; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	mode \|= find_clone_activity_on(rsc, child, node, type);
	}
	return mode;
	}

	active = find_first_action(target->actions, NULL, RSC_START, NULL);
	if (active && is_set(active->flags, pe_action_optional) == FALSE
	&& is_set(active->flags, pe_action_pseudo) == FALSE) {
	crm_debug("%s: found scheduled %s action (%s)", rsc->id, active->uuid, type);
	mode \|= stack_starting;
	}

	active = find_first_action(target->actions, NULL, RSC_STOP, node);
	if (active && is_set(active->flags, pe_action_optional) == FALSE
	&& is_set(active->flags, pe_action_pseudo) == FALSE) {
	crm_debug("%s: found scheduled %s action (%s)", rsc->id, active->uuid, type);
	mode \|= stack_stopping;
	}

	return mode;
	}

	static enum stack_activity
	check_stack_element(resource_t * rsc, resource_t * other_rsc, const char *type)
	{
	resource_t *other_p = uber_parent(other_rsc);

	if (other_rsc == NULL \|\| other_rsc == rsc) {
	return stack_stable;

	} else if (other_p->variant == pe_native) {
	crm_notice("Cannot migrate %s due to dependency on %s (%s)", rsc->id, other_rsc->id, type);
	return stack_middle;

	} else if (other_rsc == rsc->parent) {
	int mode = 0;
	GListPtr gIter = NULL;

	for (gIter = other_rsc->rsc_cons; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	if (constraint->score > 0) {
	mode \|= check_stack_element(rsc, constraint->rsc_rh, type);
	}
	}
	return mode;

	} else if (other_p->variant == pe_group) {
	crm_notice("Cannot migrate %s due to dependency on group %s (%s)",
	rsc->id, other_rsc->id, type);
	return stack_middle;

	}

	/* else: >= clone */
	/*

	## Assumption
	A depends on clone(B)

	## Resource Activity During Move

	N1 N2 N3
	--- --- ---
	t0 A.stop
	t1 B.stop B.stop
	t2 B.start B.start
	t3 A.start

	## Resource Activity During Migration

	N1 N2 N3
	--- --- ---
	t0 B.start B.start
	t1 A.stop (1)
	t2 A.start (2)
	t3 B.stop B.stop

	Node 1: Rewritten to be a migrate-to operation
	Node 2: Rewritten to be a migrate-from operation

	# Constraints

	The following constraints already exist in the system.
	The 'ok' and 'fail' column refers to whether they still hold for migration.

	a) A.stop -> A.start - ok
	b) B.stop -> B.start - fail

	c) A.stop -> B.stop - ok
	d) B.start -> A.start - ok

	e) B.stop -> A.start - fail
	f) A.stop -> B.start - fail

	## Scenarios
	B unchanged - ok
	B stopping only - fail - possible after fixing 'e'
	B starting only - fail - possible after fixing 'f'
	B stoping and starting - fail - constraint 'b' is unfixable
	B restarting only on N2 - fail - as-per previous only rarer

	*/
	/* Only allow migration when the clone is either stable, only starting or only stopping */
	return find_clone_activity_on(rsc, other_rsc, NULL, type);
	}

	static gboolean
	at_stack_bottom(resource_t * rsc)
	{
	char *key = NULL;
	action_t *start = NULL;
	action_t *other = NULL;
	int mode = stack_stable;
	GListPtr action_list = NULL;
	GListPtr gIter = NULL;

	key = start_key(rsc);
	action_list = find_actions(rsc->actions, key, NULL);
	free(key);

	crm_trace("%s: processing", rsc->id);
	CRM_CHECK(action_list != NULL, return FALSE);

	start = action_list->data;
	g_list_free(action_list);

	for (gIter = rsc->rsc_cons; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;
	resource_t *target = constraint->rsc_rh;

	crm_trace("Checking %s: %s == %s (%d)", constraint->id, rsc->id, target->id,
	constraint->score);
	if (constraint->score > 0) {
	mode \|= check_stack_element(rsc, target, "coloc");
	if (mode & stack_middle) {
	return FALSE;

	} else if ((mode & stack_stopping) && (mode & stack_starting)) {
	crm_notice("Cannot migrate %s due to colocation activity (last was %s)",
	rsc->id, target->id);
	return FALSE;
	}
	}
	}

	for (gIter = start->actions_before; gIter != NULL; gIter = gIter->next) {
	action_wrapper_t other_w = (action_wrapper_t ) gIter->data;

	other = other_w->action;

	if (other_w->type & pe_order_serialize_only) {
	crm_trace("%s: depends on %s (serialize ordering)", rsc->id, other->uuid);
	continue;
	}

	crm_trace("%s: Checking %s ordering", rsc->id, other->uuid);

	if (is_set(other->flags, pe_action_optional) == FALSE) {
	mode \|= check_stack_element(rsc, other->rsc, "order");
	if (mode & stack_middle) {
	return FALSE;

	} else if ((mode & stack_stopping) && (mode & stack_starting)) {
	crm_notice("Cannot migrate %s due to ordering activity (last was %s)",
	rsc->id, other->rsc->id);
	return FALSE;
	}
	}
	}
	return TRUE;
	}

	static action_t *
	get_first_named_action(resource_t rsc, const char action, gboolean only_valid, node_t *current)
	{
	action_t *a = NULL;
	GListPtr action_list = NULL;
	char *key = generate_op_key(rsc->id, action, 0);

	action_list = find_actions(rsc->actions, key, current);

	if (action_list == NULL \|\| action_list->data == NULL) {
	crm_trace("%s: no %s action", rsc->id, action);
	free(key);
	return NULL;
	}

	a = action_list->data;
	g_list_free(action_list);

	if(only_valid && is_set(a->flags, pe_action_pseudo)) {
	crm_trace("%s: pseudo", key);
	a = NULL;

	} else if(only_valid && is_not_set(a->flags, pe_action_runnable)) {
	crm_trace("%s: runnable", key);
	a = NULL;
	}

	free(key);
	return a;
	}

	static void
	MigrateRsc(resource_t * rsc, action_t stop, action_t start, pe_working_set_t * data_set, gboolean partial)
	{
	action_t *to = NULL;
	action_t *from = NULL;
	action_t *then = NULL;
	action_t *other = NULL;
	action_t *done = get_pseudo_op(STONITH_DONE, data_set);

	GListPtr gIter = NULL;
	const char *value = g_hash_table_lookup(rsc->meta, XML_OP_ATTR_ALLOW_MIGRATE);
	crm_trace("%s %s -> %s", rsc->id, stop->node->details->uname, start->node->details->uname);

	if (crm_is_true(value) == FALSE) {
	return;
	}

	if (rsc->next_role > RSC_ROLE_SLAVE) {
	crm_trace("%s: resource role: role=%s", rsc->id, role2text(rsc->next_role));
	return;
	}

	if(start == NULL \|\| stop == NULL) {
	crm_trace("%s: not exists %p -> %p", rsc->id, stop, start);
	return;

	} else if (start->node == NULL \|\| stop->node == NULL) {
	crm_trace("%s: no node %p -> %p", rsc->id, stop->node, start->node);
	return;

	} else if(is_set(stop->flags, pe_action_optional)) {
	crm_trace("%s: stop action", rsc->id);
	return;

	} else if(is_set(start->flags, pe_action_optional)) {
	crm_trace("%s: start action", rsc->id);
	return;

	} else if (stop->node->details == start->node->details) {
	crm_trace("%s: not moving %p -> %p", rsc->id, stop->node, start->node);
	return;

	} else if (at_stack_bottom(rsc) == FALSE) {
	crm_trace("%s: not at stack bottom", rsc->id);
	return;
	}

	if (partial) {
	crm_info("Completing partial migration of %s from %s to %s", rsc->id,
	stop->node ? stop->node->details->uname : "unknown",
	start->node ? start->node->details->uname : "unknown");
	} else {
	crm_info("Migrating %s from %s to %s", rsc->id,
	stop->node ? stop->node->details->uname : "unknown",
	start->node ? start->node->details->uname : "unknown");
	}

	/* Preserve the stop to ensure the end state is sane on that node,
	* Make the start a pseudo op
	* Create migrate_to, have it depend on everything the stop did
	* Create migrate_from
	* *-> migrate_to -> migrate_from -> stop -> start
	*/

	update_action_flags(start, pe_action_pseudo); /* easier than trying to delete it from the graph
	* but perhaps we should have it run anyway
	*/

	if (!partial) {
	to = custom_action(rsc, generate_op_key(rsc->id, RSC_MIGRATE, 0), RSC_MIGRATE, stop->node,
	FALSE, TRUE, data_set);
	}
	from = custom_action(rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), RSC_MIGRATED, start->node,
	FALSE, TRUE, data_set);

	/* This is slightly sub-optimal if 'to' fails, but always
	* run both halves of the migration before terminating the
	* transition.
	*
	* This can be removed if/when we update unpack_rsc_op() to
	* 'correctly' handle partial migrations.
	*
	* Without this, we end up stopping both sides
	*/
	from->priority = INFINITY;

	if (!partial) {
	order_actions(to, from, pe_order_optional);
	add_hash_param(to->meta, XML_LRM_ATTR_MIGRATE_SOURCE, stop->node->details->uname);
	add_hash_param(to->meta, XML_LRM_ATTR_MIGRATE_TARGET, start->node->details->uname);
	}

	then = to ? to : from;
	order_actions(from, stop, pe_order_optional);
	order_actions(done, then, pe_order_optional);
	add_hash_param(from->meta, XML_LRM_ATTR_MIGRATE_SOURCE, stop->node->details->uname);
	add_hash_param(from->meta, XML_LRM_ATTR_MIGRATE_TARGET, start->node->details->uname);

	/* Create the correct ordering ajustments based on find_clone_activity_on(); */

	for (gIter = rsc->rsc_cons; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;
	resource_t *target = constraint->rsc_rh;

	crm_info("Repairing %s: %s == %s (%d)", constraint->id, rsc->id, target->id,
	constraint->score);

	if (constraint->score > 0) {
	int mode = check_stack_element(rsc, target, "coloc");
	action_t *clone_stop = find_first_action(target->actions, NULL, RSC_STOP, NULL);
	action_t *clone_start = find_first_action(target->actions, NULL, RSC_STARTED, NULL);

	CRM_ASSERT(clone_stop != NULL);
	CRM_ASSERT(clone_start != NULL);
	CRM_ASSERT((mode & stack_middle) == 0);
	CRM_ASSERT(((mode & stack_stopping) && (mode & stack_starting)) == 0);

	if (mode & stack_stopping) {
	#if 0
	crm_debug("Creating %s.start -> %s.stop ordering", rsc->id, target->id);
	order_actions(from, clone_stop, pe_order_optional);
	#endif
	GListPtr lpc2 = NULL;

	for (lpc2 = start->actions_before; lpc2 != NULL; lpc2 = lpc2->next) {
	action_wrapper_t other_w = (action_wrapper_t ) lpc2->data;

	/* Needed if the clone's started pseudo-action ever gets printed in the graph */
	if (other_w->action == clone_start) {
	crm_debug("Breaking %s -> %s ordering", other_w->action->uuid,
	start->uuid);
	other_w->type = pe_order_none;
	}
	}

	} else if (mode & stack_starting) {
	#if 0
	crm_debug("Creating %s.started -> %s.stop ordering", target->id, rsc->id);
	order_actions(clone_start, to, pe_order_optional);
	#endif
	GListPtr lpc2 = NULL;

	for (lpc2 = clone_stop->actions_before; lpc2 != NULL; lpc2 = lpc2->next) {
	action_wrapper_t other_w = (action_wrapper_t ) lpc2->data;

	/* Needed if the clone's stop pseudo-action ever gets printed in the graph */
	if (other_w->action == stop) {
	crm_debug("Breaking %s -> %s ordering", other_w->action->uuid,
	clone_stop->uuid);
	other_w->type = pe_order_none;
	}
	}
	}
	}
	}
	#if 0
	/* Implied now that start/stop are not morphed into migrate ops */

	/* Anything that needed stop to complete, now also needs start to have completed */
	for (gIter = stop->actions_after; gIter != NULL; gIter = gIter->next) {
	action_wrapper_t other_w = (action_wrapper_t ) gIter->data;

	other = other_w->action;
	if (is_set(other->flags, pe_action_optional) \|\| other->rsc != NULL) {
	continue;
	}
	crm_debug("Ordering %s before %s (stop)", from->uuid, other->uuid);
	order_actions(from, other, other_w->type);
	}
	#endif
	/* migrate 'then' action also needs anything that the stop needed to have completed too */
	for (gIter = stop->actions_before; gIter != NULL; gIter = gIter->next) {
	action_wrapper_t other_w = (action_wrapper_t ) gIter->data;

	other = other_w->action;
	if (other->rsc == NULL) {
	/* nothing */

	} else if (is_set(other->flags, pe_action_optional) \|\| other->rsc == rsc
	\|\| other->rsc == rsc->parent) {
	continue;
	}
	crm_debug("Ordering %s before %s (stop)", other_w->action->uuid, stop->uuid);
	order_actions(other, then, other_w->type);
	}

	/* migrate 'then' action also needs anything that the start needed to have completed too */
	for (gIter = start->actions_before; gIter != NULL; gIter = gIter->next) {
	action_wrapper_t other_w = (action_wrapper_t ) gIter->data;

	other = other_w->action;
	if (other->rsc == NULL) {
	/* nothing */
	} else if (is_set(other->flags, pe_action_optional) \|\| other->rsc == rsc
	\|\| other->rsc == rsc->parent) {
	continue;
	}
	crm_debug("Ordering %s before %s (start)", other_w->action->uuid, stop->uuid);
	order_actions(other, then, other_w->type);
	}
	}

	static void
	ReloadRsc(resource_t * rsc, action_t stop, action_t start, pe_working_set_t * data_set)
	{
	action_t *action = NULL;
	action_t *rewrite = NULL;

	if(is_not_set(rsc->flags, pe_rsc_try_reload)) {
	return;

	} else if(is_not_set(stop->flags, pe_action_optional)) {
	crm_trace("%s: stop action", rsc->id);
	return;

	} else if(is_not_set(start->flags, pe_action_optional)) {
	crm_trace("%s: start action", rsc->id);
	return;
	}

	crm_trace("%s on %s", rsc->id, stop->node->details->uname);

	action = get_first_named_action(rsc, RSC_PROMOTE, TRUE, NULL);
	if (action && is_set(action->flags, pe_action_optional) == FALSE) {
	update_action_flags(action, pe_action_pseudo);
	}

	action = get_first_named_action(rsc, RSC_DEMOTE, TRUE, NULL);
	if (action && is_set(action->flags, pe_action_optional) == FALSE) {
	rewrite = action;
	update_action_flags(stop, pe_action_pseudo);

	} else {
	rewrite = start;
	}

	crm_info("Rewriting %s of %s on %s as a reload",
	rewrite->task, rsc->id, stop->node->details->uname);
	set_bit(rsc->flags, pe_rsc_reload);
	update_action_flags(rewrite, pe_action_optional\|pe_action_clear);

	free(rewrite->uuid);
	free(rewrite->task);
	rewrite->task = strdup("reload");
	rewrite->uuid = generate_op_key(rsc->id, rewrite->task, 0);
	}

	void
	rsc_migrate_reload(resource_t * rsc, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	action_t *stop = NULL;
	action_t *start = NULL;
	gboolean partial = FALSE;

	if (rsc->children) {
	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	rsc_migrate_reload(child_rsc, data_set);
	}
	return;

	} else if (rsc->variant > pe_native) {
	return;
	}

	crm_trace("Processing %s", rsc->id);

	if (rsc->partial_migration_target) {
	start = get_first_named_action(rsc, RSC_START, TRUE, rsc->partial_migration_target);
	stop = get_first_named_action(rsc, RSC_STOP, TRUE, rsc->partial_migration_source);
	if (start && stop) {
	partial = TRUE;
	}
	}

	crm_trace("%s %s %p", rsc->id, partial?"partial":"full", stop);

	if (!partial) {
	stop = get_first_named_action(rsc, RSC_STOP, TRUE, rsc->running_on ? rsc->running_on->data : NULL);
	start = get_first_named_action(rsc, RSC_START, TRUE, NULL);
	}

	if (is_not_set(rsc->flags, pe_rsc_managed)
	\|\| is_set(rsc->flags, pe_rsc_failed)
	\|\| is_set(rsc->flags, pe_rsc_start_pending)
	\|\| rsc->next_role < RSC_ROLE_STARTED
	\|\| ((g_list_length(rsc->running_on) != 1) && !partial)) {
	crm_trace("%s: general resource state: flags=0x%.16llx", rsc->id, rsc->flags);
	return;
	}

	if(stop == NULL) {
	return;

	} else if (is_set(stop->flags, pe_action_optional) && is_set(rsc->flags, pe_rsc_try_reload)) {
	ReloadRsc(rsc, stop, start, data_set);

	} else if(is_not_set(stop->flags, pe_action_optional)) {
	MigrateRsc(rsc, stop, start, data_set, partial);
	}
	}


	void
	native_append_meta(resource_t * rsc, xmlNode * xml)
	{
	char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION);

	if (value) {
	char *name = NULL;

	name = crm_meta_name(XML_RSC_ATTR_INCARNATION);
	crm_xml_add(xml, name, value);
	free(name);
	}
	}

File Metadata

Mime Type: text/x-diff
Expires: Sat, Nov 23, 7:27 AM (17 h, 2 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1018429
Default Alt Text: (436 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions