No OneTemporary
Actions

Size

72 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/crmd/te_utils.c b/crmd/te_utils.c
	index 66ed1daa68..69c22e3487 100644
	--- a/crmd/te_utils.c
	+++ b/crmd/te_utils.c
	@@ -1,542 +1,542 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>

	#include <crm/msg_xml.h>

	#include <crm/common/xml.h>
	#include <tengine.h>
	#include <crmd_fsa.h>
	#include <crmd_messages.h>
	#include <throttle.h>
	#include <crm/fencing/internal.h>

	crm_trigger_t *stonith_reconnect = NULL;
	GListPtr stonith_cleanup_list = NULL;

	static gboolean
	fail_incompletable_stonith(crm_graph_t * graph)
	{
	GListPtr lpc = NULL;
	const char *task = NULL;
	xmlNode *last_action = NULL;

	if (graph == NULL) {
	return FALSE;
	}

	for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) {
	GListPtr lpc2 = NULL;
	synapse_t synapse = (synapse_t ) lpc->data;

	if (synapse->confirmed) {
	continue;
	}

	for (lpc2 = synapse->actions; lpc2 != NULL; lpc2 = lpc2->next) {
	crm_action_t action = (crm_action_t ) lpc2->data;

	if (action->type != action_type_crm \|\| action->confirmed) {
	continue;
	}

	task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	if (task && safe_str_eq(task, CRM_OP_FENCE)) {
	action->failed = TRUE;
	last_action = action->xml;
	update_graph(graph, action);
	crm_notice("Failing action %d (%s): STONITHd terminated",
	action->id, ID(action->xml));
	}
	}
	}

	if (last_action != NULL) {
	crm_warn("STONITHd failure resulted in un-runnable actions");
	abort_transition(INFINITY, tg_restart, "Stonith failure", last_action);
	return TRUE;
	}

	return FALSE;
	}

	static void
	tengine_stonith_connection_destroy(stonith_t * st, stonith_event_t * e)
	{
	if (is_set(fsa_input_register, R_ST_REQUIRED)) {
	crm_crit("Fencing daemon connection failed");
	mainloop_set_trigger(stonith_reconnect);

	} else {
	crm_info("Fencing daemon disconnected");
	}

	/* cbchan will be garbage at this point, arrange for it to be reset */
	if(stonith_api) {
	stonith_api->state = stonith_disconnected;
	}

	if (AM_I_DC) {
	fail_incompletable_stonith(transition_graph);
	trigger_graph();
	}
	}

	#if SUPPORT_CMAN
	# include <libfenced.h>
	#endif

	char *te_client_id = NULL;

	#ifdef HAVE_SYS_REBOOT_H
	# include <unistd.h>
	# include <sys/reboot.h>
	#endif

	static void
	tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event)
	{
	if(te_client_id == NULL) {
	te_client_id = g_strdup_printf("%s.%d", crm_system_name, getpid());
	}

	if (st_event == NULL) {
	crm_err("Notify data not found");
	return;
	}

	if (st_event->result == pcmk_ok && safe_str_eq("on", st_event->action)) {
	crm_notice("%s was successfully unfenced by %s (at the request of %s)",
	- st_event->target, st_event->executioner, st_event->origin);
	+ st_event->target, st_event->executioner ? st_event->executioner : "<anyone>", st_event->origin);
	/* TODO: Hook up st_event->device */
	return;

	} else if (safe_str_eq("on", st_event->action)) {
	crm_err("Unfencing of %s by %s failed: %s (%d)",
	- st_event->target, st_event->executioner,
	+ st_event->target, st_event->executioner ? st_event->executioner : "<anyone>",
	pcmk_strerror(st_event->result), st_event->result);
	return;

	} else if (st_event->result == pcmk_ok && crm_str_eq(st_event->target, fsa_our_uname, TRUE)) {
	crm_crit("We were allegedly just fenced by %s for %s!",
	- st_event->executioner, st_event->origin); /* Dumps blackbox if enabled */
	+ st_event->executioner ? st_event->executioner : "<anyone>", st_event->origin); /* Dumps blackbox if enabled */

	qb_log_fini(); /* Try to get the above log message to disk - somehow */

	/* Get out ASAP and do not come back up.
	*
	* Triggering a reboot is also not the worst idea either since
	* the rest of the cluster thinks we're safely down
	*/

	#ifdef RB_HALT_SYSTEM
	reboot(RB_HALT_SYSTEM);
	#endif

	/*
	* If reboot() fails or is not supported, coming back up will
	* probably lead to a situation where the other nodes set our
	* status to 'lost' because of the fencing callback and will
	* discard subsequent election votes with:
	*
	* Election 87 (current: 5171, owner: 103): Processed vote from east-03 (Peer is not part of our cluster)
	*
	* So just stay dead, something is seriously messed up anyway.
	*
	*/
	exit(100); /* None of our wrappers since we already called qb_log_fini() */
	return;
	}

	if (st_event->result == pcmk_ok &&
	safe_str_eq(st_event->operation, T_STONITH_NOTIFY_FENCE)) {
	st_fail_count_reset(st_event->target);
	}

	crm_notice("Peer %s was%s terminated (%s) by %s for %s: %s (ref=%s) by client %s",
	st_event->target, st_event->result == pcmk_ok ? "" : " not",
	st_event->action,
	st_event->executioner ? st_event->executioner : "<anyone>",
	st_event->origin, pcmk_strerror(st_event->result), st_event->id,
	st_event->client_origin ? st_event->client_origin : "<unknown>");

	#if SUPPORT_CMAN
	if (st_event->result == pcmk_ok && is_cman_cluster()) {
	int local_rc = 0;
	int confirm = 0;
	char *target_copy = strdup(st_event->target);

	/* In case fenced hasn't noticed yet
	*
	* Any fencing that has been inititated will be completed by way of the fence_pcmk redirect
	*/
	local_rc = fenced_external(target_copy);
	if (local_rc != 0) {
	crm_err("Could not notify CMAN that '%s' is now fenced: %d", st_event->target,
	local_rc);
	} else {
	crm_notice("Notified CMAN that '%s' is now fenced", st_event->target);
	}

	/* In case fenced is already trying to shoot it */
	confirm = open("/var/run/cluster/fenced_override", O_NONBLOCK\|O_WRONLY);
	if (confirm > 0) {
	int ignore = 0;
	int len = strlen(target_copy);

	errno = 0;
	local_rc = write(confirm, target_copy, len);
	ignore = write(confirm, "\n", 1);

	if(ignore < 0 && errno == EBADF) {
	crm_trace("CMAN not expecting %s to be fenced (yet)", st_event->target);

	} else if (local_rc < len) {
	crm_perror(LOG_ERR, "Confirmation of CMAN fencing event for '%s' failed: %d", st_event->target, local_rc);

	} else {
	fsync(confirm);
	crm_notice("Confirmed CMAN fencing event for '%s'", st_event->target);
	}
	close(confirm);
	}
	free(target_copy);
	}
	#endif

	if (st_event->result == pcmk_ok) {
	crm_node_t *peer = crm_find_peer_full(0, st_event->target, CRM_GET_PEER_REMOTE \| CRM_GET_PEER_CLUSTER);
	const char *uuid = NULL;
	gboolean we_are_executioner = safe_str_eq(st_event->executioner, fsa_our_uname);

	if (peer == NULL) {
	return;
	}

	uuid = crm_peer_uuid(peer);

	crm_trace("target=%s dc=%s", st_event->target, fsa_our_dc);
	if(AM_I_DC) {
	/* The DC always sends updates */
	send_stonith_update(NULL, st_event->target, uuid);

	if (st_event->client_origin && safe_str_neq(st_event->client_origin, te_client_id)) {

	/* Abort the current transition graph if it wasn't us
	* that invoked stonith to fence someone
	*/
	crm_info("External fencing operation from %s fenced %s", st_event->client_origin, st_event->target);
	abort_transition(INFINITY, tg_restart, "External Fencing Operation", NULL);
	}

	/* Assume it was our leader if we dont currently have one */
	} else if (fsa_our_dc == NULL \|\| safe_str_eq(fsa_our_dc, st_event->target)) {
	crm_notice("Target %s our leader %s (recorded: %s)",
	fsa_our_dc ? "was" : "may have been", st_event->target,
	fsa_our_dc ? fsa_our_dc : "<unset>");

	/* Given the CIB resyncing that occurs around elections,
	* have one node update the CIB now and, if the new DC is different,
	* have them do so too after the election
	*/
	if (we_are_executioner) {
	send_stonith_update(NULL, st_event->target, uuid);
	}
	stonith_cleanup_list = g_list_append(stonith_cleanup_list, strdup(st_event->target));

	}

	crmd_peer_down(peer, TRUE);
	}
	}

	gboolean
	te_connect_stonith(gpointer user_data)
	{
	int lpc = 0;
	int rc = pcmk_ok;

	if (stonith_api == NULL) {
	stonith_api = stonith_api_new();
	}

	if (stonith_api->state != stonith_disconnected) {
	crm_trace("Still connected");
	return TRUE;
	}

	for (lpc = 0; lpc < 30; lpc++) {
	crm_debug("Attempting connection to fencing daemon...");

	sleep(1);
	rc = stonith_api->cmds->connect(stonith_api, crm_system_name, NULL);

	if (rc == pcmk_ok) {
	break;
	}

	if (user_data != NULL) {
	crm_err("Sign-in failed: triggered a retry");
	mainloop_set_trigger(stonith_reconnect);
	return TRUE;
	}

	crm_err("Sign-in failed: pausing and trying again in 2s...");
	sleep(1);
	}

	CRM_CHECK(rc == pcmk_ok, return TRUE); /* If not, we failed 30 times... just get out */
	stonith_api->cmds->register_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT,
	tengine_stonith_connection_destroy);

	stonith_api->cmds->register_notification(stonith_api, T_STONITH_NOTIFY_FENCE,
	tengine_stonith_notify);

	crm_trace("Connected");
	return TRUE;
	}

	gboolean
	stop_te_timer(crm_action_timer_t * timer)
	{
	const char *timer_desc = "action timer";

	if (timer == NULL) {
	return FALSE;
	}
	if (timer->reason == timeout_abort) {
	timer_desc = "global timer";
	crm_trace("Stopping %s", timer_desc);
	}

	if (timer->source_id != 0) {
	crm_trace("Stopping %s", timer_desc);
	g_source_remove(timer->source_id);
	timer->source_id = 0;

	} else {
	crm_trace("%s was already stopped", timer_desc);
	return FALSE;
	}

	return TRUE;
	}

	gboolean
	te_graph_trigger(gpointer user_data)
	{
	enum transition_status graph_rc = -1;

	if (transition_graph == NULL) {
	crm_debug("Nothing to do");
	return TRUE;
	}

	crm_trace("Invoking graph %d in state %s", transition_graph->id, fsa_state2string(fsa_state));

	switch (fsa_state) {
	case S_STARTING:
	case S_PENDING:
	case S_NOT_DC:
	case S_HALT:
	case S_ILLEGAL:
	case S_STOPPING:
	case S_TERMINATE:
	return TRUE;
	break;
	default:
	break;
	}

	if (transition_graph->complete == FALSE) {
	int limit = transition_graph->batch_limit;

	transition_graph->batch_limit = throttle_get_total_job_limit(limit);
	graph_rc = run_graph(transition_graph);
	transition_graph->batch_limit = limit; /* Restore the configured value */

	/* significant overhead... */
	/* print_graph(LOG_DEBUG_3, transition_graph); */

	if (graph_rc == transition_active) {
	crm_trace("Transition not yet complete");
	return TRUE;

	} else if (graph_rc == transition_pending) {
	crm_trace("Transition not yet complete - no actions fired");
	return TRUE;
	}

	if (graph_rc != transition_complete) {
	crm_warn("Transition failed: %s", transition_status(graph_rc));
	print_graph(LOG_NOTICE, transition_graph);
	}
	}

	crm_debug("Transition %d is now complete", transition_graph->id);
	transition_graph->complete = TRUE;
	notify_crmd(transition_graph);

	return TRUE;
	}

	void
	trigger_graph_processing(const char *fn, int line)
	{
	crm_trace("%s:%d - Triggered graph processing", fn, line);
	mainloop_set_trigger(transition_trigger);
	}

	void
	abort_transition_graph(int abort_priority, enum transition_action abort_action,
	const char abort_text, xmlNode reason, const char *fn, int line)
	{
	int add[] = { 0, 0, 0 };
	int del[] = { 0, 0, 0 };
	int level = LOG_INFO;
	xmlNode *diff = NULL;
	xmlNode *change = NULL;

	CRM_CHECK(transition_graph != NULL, return);

	switch (fsa_state) {
	case S_STARTING:
	case S_PENDING:
	case S_NOT_DC:
	case S_HALT:
	case S_ILLEGAL:
	case S_STOPPING:
	case S_TERMINATE:
	crm_info("Abort %s suppressed: state=%s (complete=%d)",
	abort_text, fsa_state2string(fsa_state), transition_graph->complete);
	return;
	default:
	break;
	}

	/* Make sure any queued calculations are discarded ASAP */
	free(fsa_pe_ref);
	fsa_pe_ref = NULL;

	if (transition_graph->complete == FALSE) {
	if(update_abort_priority(transition_graph, abort_priority, abort_action, abort_text)) {
	level = LOG_NOTICE;
	}
	}

	if(reason) {
	xmlNode *search = NULL;

	for(search = reason; search; search = search->parent) {
	if (safe_str_eq(XML_TAG_DIFF, TYPE(search))) {
	diff = search;
	break;
	}
	}

	if(diff) {
	xml_patch_versions(diff, add, del);
	for(search = reason; search; search = search->parent) {
	if (safe_str_eq(XML_DIFF_CHANGE, TYPE(search))) {
	change = search;
	break;
	}
	}
	}
	}

	if(reason == NULL) {
	do_crm_log(level, "Transition aborted: %s (source=%s:%d, %d)",
	abort_text, fn, line, transition_graph->complete);

	} else if(change == NULL) {
	char *local_path = xml_get_path(reason);

	do_crm_log(level, "Transition aborted by %s.%s: %s (cib=%d.%d.%d, source=%s:%d, path=%s, %d)",
	TYPE(reason), ID(reason), abort_text, add[0], add[1], add[2], fn, line, local_path, transition_graph->complete);
	free(local_path);

	} else {
	const char *kind = NULL;
	const char *op = crm_element_value(change, XML_DIFF_OP);
	const char *path = crm_element_value(change, XML_DIFF_PATH);

	if(change == reason) {
	if(strcmp(op, "create") == 0) {
	reason = reason->children;

	} else if(strcmp(op, "modify") == 0) {
	reason = first_named_child(reason, XML_DIFF_RESULT);
	if(reason) {
	reason = reason->children;
	}
	}
	}

	kind = TYPE(reason);
	if(strcmp(op, "delete") == 0) {
	const char *shortpath = strrchr(path, '/');

	do_crm_log(level, "Transition aborted by deletion of %s: %s (cib=%d.%d.%d, source=%s:%d, path=%s, %d)",
	shortpath?shortpath+1:path, abort_text, add[0], add[1], add[2], fn, line, path, transition_graph->complete);

	} else if (safe_str_eq(XML_CIB_TAG_NVPAIR, kind)) {
	do_crm_log(level, "Transition aborted by %s, %s=%s: %s (%s cib=%d.%d.%d, source=%s:%d, path=%s, %d)",
	crm_element_value(reason, XML_ATTR_ID),
	crm_element_value(reason, XML_NVPAIR_ATTR_NAME),
	crm_element_value(reason, XML_NVPAIR_ATTR_VALUE),
	abort_text, op, add[0], add[1], add[2], fn, line, path, transition_graph->complete);

	} else if (safe_str_eq(XML_LRM_TAG_RSC_OP, kind)) {
	const char *magic = crm_element_value(reason, XML_ATTR_TRANSITION_MAGIC);

	do_crm_log(level, "Transition aborted by %s '%s' on %s: %s (magic=%s, cib=%d.%d.%d, source=%s:%d, %d)",
	crm_element_value(reason, XML_LRM_ATTR_TASK_KEY), op,
	crm_element_value(reason, XML_LRM_ATTR_TARGET), abort_text,
	magic, add[0], add[1], add[2], fn, line, transition_graph->complete);

	} else if (safe_str_eq(XML_CIB_TAG_STATE, kind)
	\|\| safe_str_eq(XML_CIB_TAG_NODE, kind)) {
	const char *uname = crm_peer_uname(ID(reason));

	do_crm_log(level, "Transition aborted by %s '%s' on %s: %s (cib=%d.%d.%d, source=%s:%d, %d)",
	kind, op, uname ? uname : ID(reason), abort_text,
	add[0], add[1], add[2], fn, line, transition_graph->complete);

	} else {
	do_crm_log(level, "Transition aborted by %s.%s '%s': %s (cib=%d.%d.%d, source=%s:%d, path=%s, %d)",
	TYPE(reason), ID(reason), op?op:"change", abort_text, add[0], add[1], add[2], fn, line, path, transition_graph->complete);
	}
	}

	if (transition_graph->complete) {
	if (transition_timer->period_ms > 0) {
	crm_timer_stop(transition_timer);
	crm_timer_start(transition_timer);
	} else {
	register_fsa_input(C_FSA_INTERNAL, I_PE_CALC, NULL);
	}
	return;
	}

	mainloop_set_trigger(transition_trigger);
	}
	diff --git a/fencing/remote.c b/fencing/remote.c
	index 3f4f5ca2fe..834c440970 100644
	--- a/fencing/remote.c
	+++ b/fencing/remote.c
	@@ -1,1522 +1,1520 @@
	/*
	* Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <ctype.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipcs.h>
	#include <crm/cluster/internal.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>

	#include <crm/common/util.h>
	#include <internal.h>

	#define TIMEOUT_MULTIPLY_FACTOR 1.2

	typedef struct st_query_result_s {
	char *host;
	int devices;
	/* only try peers for non-topology based operations once */
	gboolean tried;
	GListPtr device_list;
	GHashTable *custom_action_timeouts;
	/* Subset of devices that peer has verified connectivity on */
	GHashTable *verified_devices;

	} st_query_result_t;

	GHashTable *remote_op_list = NULL;
	void call_remote_stonith(remote_fencing_op_t * op, st_query_result_t * peer);
	static void remote_op_done(remote_fencing_op_t * op, xmlNode * data, int rc, int dup);
	extern xmlNode stonith_create_op(int call_id, const char token, const char op, xmlNode data,
	int call_options);

	static void report_timeout_period(remote_fencing_op_t * op, int op_timeout);
	static int get_op_total_timeout(remote_fencing_op_t * op, st_query_result_t * chosen_peer,
	int default_timeout);

	static gint
	sort_strings(gconstpointer a, gconstpointer b)
	{
	return strcmp(a, b);
	}

	static void
	free_remote_query(gpointer data)
	{
	if (data) {
	st_query_result_t *query = data;

	crm_trace("Free'ing query result from %s", query->host);
	free(query->host);
	g_list_free_full(query->device_list, free);
	g_hash_table_destroy(query->custom_action_timeouts);
	g_hash_table_destroy(query->verified_devices);
	free(query);
	}
	}

	static void
	clear_remote_op_timers(remote_fencing_op_t * op)
	{
	if (op->query_timer) {
	g_source_remove(op->query_timer);
	op->query_timer = 0;
	}
	if (op->op_timer_total) {
	g_source_remove(op->op_timer_total);
	op->op_timer_total = 0;
	}
	if (op->op_timer_one) {
	g_source_remove(op->op_timer_one);
	op->op_timer_one = 0;
	}
	}

	static void
	free_remote_op(gpointer data)
	{
	remote_fencing_op_t *op = data;

	crm_trace("Free'ing op %s for %s", op->id, op->target);
	crm_log_xml_debug(op->request, "Destroying");

	clear_remote_op_timers(op);

	free(op->id);
	free(op->action);
	free(op->target);
	free(op->client_id);
	free(op->client_name);
	free(op->originator);

	if (op->query_results) {
	g_list_free_full(op->query_results, free_remote_query);
	}
	if (op->request) {
	free_xml(op->request);
	op->request = NULL;
	}
	if (op->devices_list) {
	g_list_free_full(op->devices_list, free);
	op->devices_list = NULL;
	}
	if (op->required_list) {
	g_list_free_full(op->required_list, free);
	op->required_list = NULL;
	}
	free(op);
	}

	static xmlNode *
	create_op_done_notify(remote_fencing_op_t * op, int rc)
	{
	xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE);

	crm_xml_add_int(notify_data, "state", op->state);
	crm_xml_add_int(notify_data, F_STONITH_RC, rc);
	crm_xml_add(notify_data, F_STONITH_TARGET, op->target);
	crm_xml_add(notify_data, F_STONITH_ACTION, op->action);
	crm_xml_add(notify_data, F_STONITH_DELEGATE, op->delegate);
	crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, op->id);
	crm_xml_add(notify_data, F_STONITH_ORIGIN, op->originator);
	crm_xml_add(notify_data, F_STONITH_CLIENTID, op->client_id);
	crm_xml_add(notify_data, F_STONITH_CLIENTNAME, op->client_name);

	return notify_data;
	}

	static void
	bcast_result_to_peers(remote_fencing_op_t * op, int rc)
	{
	static int count = 0;
	xmlNode *bcast = create_xml_node(NULL, T_STONITH_REPLY);
	xmlNode *notify_data = create_op_done_notify(op, rc);

	count++;
	crm_trace("Broadcasting result to peers");
	crm_xml_add(bcast, F_TYPE, T_STONITH_NOTIFY);
	crm_xml_add(bcast, F_SUBTYPE, "broadcast");
	crm_xml_add(bcast, F_STONITH_OPERATION, T_STONITH_NOTIFY);
	crm_xml_add_int(bcast, "count", count);
	add_message_xml(bcast, F_STONITH_CALLDATA, notify_data);
	send_cluster_message(NULL, crm_msg_stonith_ng, bcast, FALSE);
	free_xml(notify_data);
	free_xml(bcast);

	return;
	}

	static void
	handle_local_reply_and_notify(remote_fencing_op_t * op, xmlNode * data, int rc)
	{
	xmlNode *notify_data = NULL;
	xmlNode *reply = NULL;

	if (op->notify_sent == TRUE) {
	/* nothing to do */
	return;
	}

	/* Do notification with a clean data object */
	notify_data = create_op_done_notify(op, rc);
	crm_xml_add_int(data, "state", op->state);
	crm_xml_add(data, F_STONITH_TARGET, op->target);
	crm_xml_add(data, F_STONITH_OPERATION, op->action);

	reply = stonith_construct_reply(op->request, NULL, data, rc);
	crm_xml_add(reply, F_STONITH_DELEGATE, op->delegate);

	/* Send fencing OP reply to local client that initiated fencing */
	do_local_reply(reply, op->client_id, op->call_options & st_opt_sync_call, FALSE);

	/* bcast to all local clients that the fencing operation happend */
	do_stonith_notify(0, T_STONITH_NOTIFY_FENCE, rc, notify_data);

	/* mark this op as having notify's already sent */
	op->notify_sent = TRUE;
	free_xml(reply);
	free_xml(notify_data);
	}

	static void
	handle_duplicates(remote_fencing_op_t * op, xmlNode * data, int rc)
	{
	GListPtr iter = NULL;

	for (iter = op->duplicates; iter != NULL; iter = iter->next) {
	remote_fencing_op_t *other = iter->data;

	if (other->state == st_duplicate) {
	/* Ie. it hasn't timed out already */
	other->state = op->state;
	crm_debug("Peforming duplicate notification for %s@%s.%.8s = %s", other->client_name,
	other->originator, other->id, pcmk_strerror(rc));
	remote_op_done(other, data, rc, TRUE);

	} else {
	crm_err("Skipping duplicate notification for %s@%s - %d", other->client_name,
	other->originator, other->state);
	}
	}
	}

	/*!
	* \internal
	* \brief Finalize a remote operation.
	*
	* \description This function has two code paths.
	*
	* Path 1. This node is the owner of the operation and needs
	* to notify the cpg group via a broadcast as to the operation's
	* results.
	*
	* Path 2. The cpg broadcast is received. All nodes notify their local
	* stonith clients the operation results.
	*
	* So, The owner of the operation first notifies the cluster of the result,
	* and once that cpg notify is received back it notifies all the local clients.
	*
	* Nodes that are passive watchers of the operation will receive the
	* broadcast and only need to notify their local clients the operation finished.
	*
	* \param op, The fencing operation to finalize
	* \param data, The xml msg reply (if present) of the last delegated fencing
	* operation.
	* \param dup, Is this operation a duplicate, if so treat it a little differently
	* making sure the broadcast is not sent out.
	*/
	static void
	remote_op_done(remote_fencing_op_t * op, xmlNode * data, int rc, int dup)
	{
	int level = LOG_ERR;
	const char *subt = NULL;
	xmlNode *local_data = NULL;

	op->completed = time(NULL);
	clear_remote_op_timers(op);

	if (op->notify_sent == TRUE) {
	crm_err("Already sent notifications for '%s of %s by %s' (for=%s@%s.%.8s, state=%d): %s",
	op->action, op->target, op->delegate ? op->delegate : "<no-one>",
	op->client_name, op->originator, op->id, op->state, pcmk_strerror(rc));
	goto remote_op_done_cleanup;
	}

	if (!op->delegate && data) {
	xmlNode *ndata = get_xpath_object("//@" F_STONITH_DELEGATE, data, LOG_WARNING);
	if(ndata) {
	op->delegate = crm_element_value_copy(ndata, F_STONITH_DELEGATE);
	- } else {
	- op->delegate = crm_element_value_copy(data, F_ORIG);
	}
	}

	if (data == NULL) {
	data = create_xml_node(NULL, "remote-op");
	local_data = data;
	}

	/* Tell everyone the operation is done, we will continue
	* with doing the local notifications once we receive
	* the broadcast back. */
	subt = crm_element_value(data, F_SUBTYPE);
	if (dup == FALSE && safe_str_neq(subt, "broadcast")) {
	/* Defer notification until the bcast message arrives */
	bcast_result_to_peers(op, rc);
	goto remote_op_done_cleanup;
	}

	if (rc == pcmk_ok \|\| dup) {
	level = LOG_NOTICE;
	} else if (safe_str_neq(op->originator, stonith_our_uname)) {
	level = LOG_NOTICE;
	}

	do_crm_log(level,
	"Operation %s of %s by %s for %s@%s.%.8s: %s",
	op->action, op->target, op->delegate ? op->delegate : "<no-one>",
	op->client_name, op->originator, op->id, pcmk_strerror(rc));

	handle_local_reply_and_notify(op, data, rc);

	if (dup == FALSE) {
	handle_duplicates(op, data, rc);
	}

	/* Free non-essential parts of the record
	* Keep the record around so we can query the history
	*/
	if (op->query_results) {
	g_list_free_full(op->query_results, free_remote_query);
	op->query_results = NULL;
	}

	if (op->request) {
	free_xml(op->request);
	op->request = NULL;
	}

	remote_op_done_cleanup:
	free_xml(local_data);
	}

	static gboolean
	remote_op_watchdog_done(gpointer userdata)
	{
	remote_fencing_op_t *op = userdata;

	op->op_timer_one = 0;

	crm_notice("Remote %s operation on %s for %s.%8s assumed complete",
	op->action, op->target, op->client_name, op->id);
	op->state = st_done;
	remote_op_done(op, NULL, pcmk_ok, FALSE);
	return FALSE;
	}

	static gboolean
	remote_op_timeout_one(gpointer userdata)
	{
	remote_fencing_op_t *op = userdata;

	op->op_timer_one = 0;

	crm_notice("Remote %s operation on %s for %s.%8s timed out",
	op->action, op->target, op->client_name, op->id);
	call_remote_stonith(op, NULL);
	return FALSE;
	}

	static gboolean
	remote_op_timeout(gpointer userdata)
	{
	remote_fencing_op_t *op = userdata;

	op->op_timer_total = 0;

	if (op->state == st_done) {
	crm_debug("Action %s (%s) for %s (%s) already completed",
	op->action, op->id, op->target, op->client_name);
	return FALSE;
	}

	crm_debug("Action %s (%s) for %s (%s) timed out",
	op->action, op->id, op->target, op->client_name);
	op->state = st_failed;

	remote_op_done(op, NULL, -ETIME, FALSE);

	return FALSE;
	}

	static gboolean
	remote_op_query_timeout(gpointer data)
	{
	remote_fencing_op_t *op = data;

	op->query_timer = 0;
	if (op->state == st_done) {
	crm_debug("Operation %s for %s already completed", op->id, op->target);
	} else if (op->state == st_exec) {
	crm_debug("Operation %s for %s already in progress", op->id, op->target);
	} else if (op->query_results) {
	crm_debug("Query %s for %s complete: %d", op->id, op->target, op->state);
	call_remote_stonith(op, NULL);
	} else {
	crm_debug("Query %s for %s timed out: %d", op->id, op->target, op->state);
	if (op->op_timer_total) {
	g_source_remove(op->op_timer_total);
	op->op_timer_total = 0;
	}
	remote_op_timeout(op);
	}

	return FALSE;
	}

	static gboolean
	topology_is_empty(stonith_topology_t *tp)
	{
	int i;

	if (tp == NULL) {
	return TRUE;
	}

	for (i = 0; i < ST_LEVEL_MAX; i++) {
	if (tp->levels[i] != NULL) {
	return FALSE;
	}
	}
	return TRUE;
	}

	static void
	add_required_device(remote_fencing_op_t * op, const char *device)
	{
	GListPtr match = g_list_find_custom(op->required_list, device, sort_strings);
	if (match) {
	/* device already marked required */
	return;
	}
	op->required_list = g_list_prepend(op->required_list, strdup(device));

	/* make sure the required devices is in the current list of devices to be executed */
	if (op->devices_list) {
	GListPtr match = g_list_find_custom(op->devices_list, device, sort_strings);
	if (match == NULL) {
	op->devices_list = g_list_append(op->devices_list, strdup(device));
	}
	}
	}

	/* deep copy the device list */
	static void
	set_op_device_list(remote_fencing_op_t * op, GListPtr devices)
	{
	GListPtr lpc = NULL;

	if (op->devices_list) {
	g_list_free_full(op->devices_list, free);
	op->devices_list = NULL;
	}
	for (lpc = devices; lpc != NULL; lpc = lpc->next) {
	op->devices_list = g_list_append(op->devices_list, strdup(lpc->data));
	}

	/* tack on whatever required devices have not been executed
	* to the end of the current devices list. This ensures that
	* the required devices will get executed regardless of what topology
	* level they exist at. */
	for (lpc = op->required_list; lpc != NULL; lpc = lpc->next) {
	GListPtr match = g_list_find_custom(op->devices_list, lpc->data, sort_strings);
	if (match == NULL) {
	op->devices_list = g_list_append(op->devices_list, strdup(lpc->data));
	}
	}

	op->devices = op->devices_list;
	}

	static int
	stonith_topology_next(remote_fencing_op_t * op)
	{
	stonith_topology_t *tp = NULL;

	if (op->target) {
	/* Queries don't have a target set */
	tp = g_hash_table_lookup(topology, op->target);
	}
	if (topology_is_empty(tp)) {
	return pcmk_ok;
	}

	set_bit(op->call_options, st_opt_topology);

	do {
	op->level++;

	} while (op->level < ST_LEVEL_MAX && tp->levels[op->level] == NULL);

	if (op->level < ST_LEVEL_MAX) {
	crm_trace("Attempting fencing level %d for %s (%d devices) - %s@%s.%.8s",
	op->level, op->target, g_list_length(tp->levels[op->level]),
	op->client_name, op->originator, op->id);
	set_op_device_list(op, tp->levels[op->level]);
	return pcmk_ok;
	}

	crm_notice("All fencing options to fence %s for %s@%s.%.8s failed",
	op->target, op->client_name, op->originator, op->id);
	return -EINVAL;
	}

	/*!
	* \brief Check to see if this operation is a duplicate of another in flight
	* operation. If so merge this operation into the inflight operation, and mark
	* it as a duplicate.
	*/
	static void
	merge_duplicates(remote_fencing_op_t * op)
	{
	GHashTableIter iter;
	remote_fencing_op_t *other = NULL;

	time_t now = time(NULL);

	g_hash_table_iter_init(&iter, remote_op_list);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&other)) {
	crm_node_t *peer = NULL;

	if (other->state > st_exec) {
	/* Must be in-progress */
	continue;
	} else if (safe_str_neq(op->target, other->target)) {
	/* Must be for the same node */
	continue;
	} else if (safe_str_neq(op->action, other->action)) {
	crm_trace("Must be for the same action: %s vs. ", op->action, other->action);
	continue;
	} else if (safe_str_eq(op->client_name, other->client_name)) {
	crm_trace("Must be for different clients: %s", op->client_name);
	continue;
	} else if (safe_str_eq(other->target, other->originator)) {
	crm_trace("Can't be a suicide operation: %s", other->target);
	continue;
	}

	peer = crm_get_peer(0, other->originator);
	if(fencing_peer_active(peer) == FALSE) {
	crm_notice("Failing stonith action %s for node %s originating from %s@%s.%.8s: Originator is dead",
	other->action, other->target, other->client_name, other->originator, other->id);
	other->state = st_failed;
	continue;

	} else if(other->total_timeout > 0 && now > (other->total_timeout + other->created)) {
	crm_info("Stonith action %s for node %s originating from %s@%s.%.8s is too old: %d vs. %d + %d",
	other->action, other->target, other->client_name, other->originator, other->id,
	now, other->created, other->total_timeout);
	continue;
	}

	/* There is another in-flight request to fence the same host
	* Piggyback on that instead. If it fails, so do we.
	*/
	other->duplicates = g_list_append(other->duplicates, op);
	if (other->total_timeout == 0) {
	crm_trace("Making a best-guess as to the timeout used");
	other->total_timeout = op->total_timeout =
	TIMEOUT_MULTIPLY_FACTOR * get_op_total_timeout(op, NULL, op->base_timeout);
	}
	crm_notice
	("Merging stonith action %s for node %s originating from client %s.%.8s with identical request from %s@%s.%.8s (%ds)",
	op->action, op->target, op->client_name, op->id, other->client_name, other->originator,
	other->id, other->total_timeout);
	report_timeout_period(op, other->total_timeout);
	op->state = st_duplicate;
	}
	}

	static uint32_t fencing_active_peers(void)
	{
	uint32_t count = 0;
	crm_node_t *entry;
	GHashTableIter gIter;

	g_hash_table_iter_init(&gIter, crm_peer_cache);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	if(fencing_peer_active(entry)) {
	count++;
	}
	}
	return count;
	}

	int
	stonith_manual_ack(xmlNode * msg, remote_fencing_op_t * op)
	{
	xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_ERR);

	op->state = st_done;
	op->completed = time(NULL);
	op->delegate = strdup("a human");

	crm_notice("Injecting manual confirmation that %s is safely off/down",
	crm_element_value(dev, F_STONITH_TARGET));

	remote_op_done(op, msg, pcmk_ok, FALSE);

	/* Replies are sent via done_cb->stonith_send_async_reply()->do_local_reply() */
	return -EINPROGRESS;
	}

	/*!
	* \internal
	* \brief Create a new remote stonith op
	* \param client, he local stonith client id that initaited the operation
	* \param request, The request from the client that started the operation
	* \param peer, Is this operation owned by another stonith peer? Operations
	* owned by other peers are stored on all the stonith nodes, but only the
	* owner executes the operation. All the nodes get the results to the operation
	* once the owner finishes executing it.
	*/
	void *
	create_remote_stonith_op(const char client, xmlNode request, gboolean peer)
	{
	remote_fencing_op_t *op = NULL;
	xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_TRACE);
	int call_options = 0;

	if (remote_op_list == NULL) {
	remote_op_list = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_remote_op);
	}

	/* If this operation is owned by another node, check to make
	* sure we haven't already created this operation. */
	if (peer && dev) {
	const char *op_id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);

	CRM_CHECK(op_id != NULL, return NULL);

	op = g_hash_table_lookup(remote_op_list, op_id);
	if (op) {
	crm_debug("%s already exists", op_id);
	return op;
	}
	}

	op = calloc(1, sizeof(remote_fencing_op_t));

	crm_element_value_int(request, F_STONITH_TIMEOUT, (int *)&(op->base_timeout));

	if (peer && dev) {
	op->id = crm_element_value_copy(dev, F_STONITH_REMOTE_OP_ID);
	} else {
	op->id = crm_generate_uuid();
	}

	g_hash_table_replace(remote_op_list, op->id, op);
	CRM_LOG_ASSERT(g_hash_table_lookup(remote_op_list, op->id) != NULL);
	crm_trace("Created %s", op->id);

	op->state = st_query;
	op->replies_expected = fencing_active_peers();
	op->action = crm_element_value_copy(dev, F_STONITH_ACTION);
	op->originator = crm_element_value_copy(dev, F_STONITH_ORIGIN);
	op->delegate = crm_element_value_copy(dev, F_STONITH_DELEGATE); /* May not be set */
	op->created = time(NULL);

	if (op->originator == NULL) {
	/* Local or relayed request */
	op->originator = strdup(stonith_our_uname);
	}

	CRM_LOG_ASSERT(client != NULL);
	if (client) {
	op->client_id = strdup(client);
	}

	op->client_name = crm_element_value_copy(request, F_STONITH_CLIENTNAME);

	op->target = crm_element_value_copy(dev, F_STONITH_TARGET);
	op->request = copy_xml(request); /* TODO: Figure out how to avoid this */
	crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
	op->call_options = call_options;

	crm_element_value_int(request, F_STONITH_CALLID, (int *)&(op->client_callid));

	crm_trace("%s new stonith op: %s - %s of %s for %s",
	(peer
	&& dev) ? "Recorded" : "Generated", op->id, op->action, op->target, op->client_name);

	if (op->call_options & st_opt_cs_nodeid) {
	int nodeid = crm_atoi(op->target, NULL);
	crm_node_t *node = crm_get_peer(nodeid, NULL);

	/* Ensure the conversion only happens once */
	op->call_options &= ~st_opt_cs_nodeid;

	if (node && node->uname) {
	free(op->target);
	op->target = strdup(node->uname);
	} else {
	crm_warn("Could not expand nodeid '%s' into a host name (%p)", op->target, node);
	}
	}

	/* check to see if this is a duplicate operation of another in-flight operation */
	merge_duplicates(op);

	return op;
	}

	remote_fencing_op_t *
	initiate_remote_stonith_op(crm_client_t * client, xmlNode * request, gboolean manual_ack)
	{
	int query_timeout = 0;
	xmlNode *query = NULL;
	const char *client_id = NULL;
	remote_fencing_op_t *op = NULL;

	if (client) {
	client_id = client->id;
	} else {
	client_id = crm_element_value(request, F_STONITH_CLIENTID);
	}

	CRM_LOG_ASSERT(client_id != NULL);
	op = create_remote_stonith_op(client_id, request, FALSE);
	op->owner = TRUE;
	if (manual_ack) {
	crm_notice("Initiating manual confirmation for %s: %s",
	op->target, op->id);
	return op;
	}

	CRM_CHECK(op->action, return NULL);

	if (stonith_topology_next(op) != pcmk_ok) {
	op->state = st_failed;
	}

	switch (op->state) {
	case st_failed:
	crm_warn("Initiation of remote operation %s for %s: failed (%s)", op->action,
	op->target, op->id);
	remote_op_done(op, NULL, -EINVAL, FALSE);
	return op;

	case st_duplicate:
	crm_info("Initiating remote operation %s for %s: %s (duplicate)", op->action,
	op->target, op->id);
	return op;

	default:
	crm_notice("Initiating remote operation %s for %s: %s (%d)", op->action, op->target,
	op->id, op->state);
	}

	query = stonith_create_op(op->client_callid, op->id, STONITH_OP_QUERY, NULL, 0);

	crm_xml_add(query, F_STONITH_REMOTE_OP_ID, op->id);
	crm_xml_add(query, F_STONITH_TARGET, op->target);
	crm_xml_add(query, F_STONITH_ACTION, op->action);
	crm_xml_add(query, F_STONITH_ORIGIN, op->originator);
	crm_xml_add(query, F_STONITH_CLIENTID, op->client_id);
	crm_xml_add(query, F_STONITH_CLIENTNAME, op->client_name);
	crm_xml_add_int(query, F_STONITH_TIMEOUT, op->base_timeout);
	crm_xml_add_int(query, F_STONITH_CALLOPTS, op->call_options);

	send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE);
	free_xml(query);

	query_timeout = op->base_timeout * TIMEOUT_MULTIPLY_FACTOR;
	op->query_timer = g_timeout_add((1000 * query_timeout), remote_op_query_timeout, op);

	return op;
	}

	enum find_best_peer_options {
	/! Skip checking the target peer for capable fencing devices /
	FIND_PEER_SKIP_TARGET = 0x0001,
	/! Only check the target peer for capable fencing devices /
	FIND_PEER_TARGET_ONLY = 0x0002,
	/! Skip peers and devices that are not verified /
	FIND_PEER_VERIFIED_ONLY = 0x0004,
	};

	static st_query_result_t *
	find_best_peer(const char device, remote_fencing_op_t op, enum find_best_peer_options options)
	{
	GListPtr iter = NULL;
	gboolean verified_devices_only = (options & FIND_PEER_VERIFIED_ONLY) ? TRUE : FALSE;

	if (!device && is_set(op->call_options, st_opt_topology)) {
	return NULL;
	}

	for (iter = op->query_results; iter != NULL; iter = iter->next) {
	st_query_result_t *peer = iter->data;

	crm_trace("Testing result from %s for %s with %d devices: %d %x",
	peer->host, op->target, peer->devices, peer->tried, options);
	if ((options & FIND_PEER_SKIP_TARGET) && safe_str_eq(peer->host, op->target)) {
	continue;
	}
	if ((options & FIND_PEER_TARGET_ONLY) && safe_str_neq(peer->host, op->target)) {
	continue;
	}

	if (is_set(op->call_options, st_opt_topology)) {
	/* Do they have the next device of the current fencing level? */
	GListPtr match = NULL;

	if (verified_devices_only && !g_hash_table_lookup(peer->verified_devices, device)) {
	continue;
	}

	match = g_list_find_custom(peer->device_list, device, sort_strings);
	if (match) {
	crm_trace("Removing %s from %s (%d remaining)", (char *)match->data, peer->host,
	g_list_length(peer->device_list));
	peer->device_list = g_list_remove(peer->device_list, match->data);
	return peer;
	}

	} else if (peer->devices > 0 && peer->tried == FALSE) {
	if (verified_devices_only && !g_hash_table_size(peer->verified_devices)) {
	continue;
	}

	/* No topology: Use the current best peer */
	crm_trace("Simple fencing");
	return peer;
	}
	}

	return NULL;
	}

	static st_query_result_t *
	stonith_choose_peer(remote_fencing_op_t * op)
	{
	const char *device = NULL;
	st_query_result_t *peer = NULL;
	uint32_t active = fencing_active_peers();

	do {
	if (op->devices) {
	device = op->devices->data;
	crm_trace("Checking for someone to fence %s with %s", op->target, device);
	} else {
	crm_trace("Checking for someone to fence %s", op->target);
	}

	peer = find_best_peer(device, op, FIND_PEER_SKIP_TARGET\|FIND_PEER_VERIFIED_ONLY);
	if (peer) {
	crm_trace("Found verified peer %s for %s", peer->host, device?device:"<any>");
	return peer;
	}

	if(op->query_timer != 0 && op->replies < QB_MIN(op->replies_expected, active)) {
	crm_trace("Waiting before looking for unverified devices to fence %s", op->target);
	return NULL;
	}

	peer = find_best_peer(device, op, FIND_PEER_SKIP_TARGET);
	if (peer) {
	crm_trace("Found best unverified peer %s", peer->host);
	return peer;
	}

	peer = find_best_peer(device, op, FIND_PEER_TARGET_ONLY);
	if(peer) {
	crm_trace("%s will fence itself", peer->host);
	return peer;
	}

	/* Try the next fencing level if there is one */
	} while (is_set(op->call_options, st_opt_topology)
	&& stonith_topology_next(op) == pcmk_ok);

	crm_notice("Couldn't find anyone to fence %s with %s", op->target, device?device:"<any>");
	return NULL;
	}

	static int
	get_device_timeout(st_query_result_t * peer, const char *device, int default_timeout)
	{
	gpointer res;

	if (!peer \|\| !device) {
	return default_timeout;
	}

	res = g_hash_table_lookup(peer->custom_action_timeouts, device);

	return res ? GPOINTER_TO_INT(res) : default_timeout;
	}

	static int
	get_peer_timeout(st_query_result_t * peer, int default_timeout)
	{
	int total_timeout = 0;

	GListPtr cur = NULL;

	for (cur = peer->device_list; cur; cur = cur->next) {
	total_timeout += get_device_timeout(peer, cur->data, default_timeout);
	}

	return total_timeout ? total_timeout : default_timeout;
	}

	static int
	get_op_total_timeout(remote_fencing_op_t * op, st_query_result_t * chosen_peer, int default_timeout)
	{
	stonith_topology_t *tp = g_hash_table_lookup(topology, op->target);
	int total_timeout = 0;

	if (is_set(op->call_options, st_opt_topology) && tp) {
	int i;
	GListPtr device_list = NULL;
	GListPtr iter = NULL;

	/* Yep, this looks scary, nested loops all over the place.
	* Here is what is going on.
	* Loop1: Iterate through fencing levels.
	* Loop2: If a fencing level has devices, loop through each device
	* Loop3: For each device in a fencing level, see what peer owns it
	* and what that peer has reported the timeout is for the device.
	*/
	for (i = 0; i < ST_LEVEL_MAX; i++) {
	if (!tp->levels[i]) {
	continue;
	}
	for (device_list = tp->levels[i]; device_list; device_list = device_list->next) {
	for (iter = op->query_results; iter != NULL; iter = iter->next) {
	st_query_result_t *peer = iter->data;

	if (g_list_find_custom(peer->device_list, device_list->data, sort_strings)) {
	total_timeout +=
	get_device_timeout(peer, device_list->data, default_timeout);
	break;
	}
	} /* End Loop3: match device with peer that owns device, find device's timeout period */
	} /* End Loop2: iterate through devices at a specific level */
	} /End Loop1: iterate through fencing levels /

	} else if (chosen_peer) {
	total_timeout = get_peer_timeout(chosen_peer, default_timeout);
	} else {
	total_timeout = default_timeout;
	}

	return total_timeout ? total_timeout : default_timeout;
	}

	static void
	report_timeout_period(remote_fencing_op_t * op, int op_timeout)
	{
	GListPtr iter = NULL;
	xmlNode *update = NULL;
	const char *client_node = NULL;
	const char *client_id = NULL;
	const char *call_id = NULL;

	if (op->call_options & st_opt_sync_call) {
	/* There is no reason to report the timeout for a syncronous call. It
	* is impossible to use the reported timeout to do anything when the client
	* is blocking for the response. This update is only important for
	* async calls that require a callback to report the results in. */
	return;
	} else if (!op->request) {
	return;
	}

	crm_trace("Reporting timeout for %s.%.8s", op->client_name, op->id);
	client_node = crm_element_value(op->request, F_STONITH_CLIENTNODE);
	call_id = crm_element_value(op->request, F_STONITH_CALLID);
	client_id = crm_element_value(op->request, F_STONITH_CLIENTID);
	if (!client_node \|\| !call_id \|\| !client_id) {
	return;
	}

	if (safe_str_eq(client_node, stonith_our_uname)) {
	/* The client is connected to this node, send the update direclty to them */
	do_stonith_async_timeout_update(client_id, call_id, op_timeout);
	return;
	}

	/* The client is connected to another node, relay this update to them */
	update = stonith_create_op(op->client_callid, op->id, STONITH_OP_TIMEOUT_UPDATE, NULL, 0);
	crm_xml_add(update, F_STONITH_REMOTE_OP_ID, op->id);
	crm_xml_add(update, F_STONITH_CLIENTID, client_id);
	crm_xml_add(update, F_STONITH_CALLID, call_id);
	crm_xml_add_int(update, F_STONITH_TIMEOUT, op_timeout);

	send_cluster_message(crm_get_peer(0, client_node), crm_msg_stonith_ng, update, FALSE);

	free_xml(update);

	for (iter = op->duplicates; iter != NULL; iter = iter->next) {
	remote_fencing_op_t *dup = iter->data;

	crm_trace("Reporting timeout for duplicate %s.%.8s", dup->client_name, dup->id);
	report_timeout_period(iter->data, op_timeout);
	}
	}

	void
	call_remote_stonith(remote_fencing_op_t * op, st_query_result_t * peer)
	{
	const char *device = NULL;
	int timeout = op->base_timeout;

	crm_trace("State for %s.%.8s: %s %d", op->target, op->client_name, op->id, op->state);
	if (peer == NULL && !is_set(op->call_options, st_opt_topology)) {
	peer = stonith_choose_peer(op);
	}

	if (!op->op_timer_total) {
	int total_timeout = get_op_total_timeout(op, peer, op->base_timeout);

	op->total_timeout = TIMEOUT_MULTIPLY_FACTOR * total_timeout;
	op->op_timer_total = g_timeout_add(1000 * op->total_timeout, remote_op_timeout, op);
	report_timeout_period(op, op->total_timeout);
	crm_info("Total remote op timeout set to %d for fencing of node %s for %s.%.8s",
	total_timeout, op->target, op->client_name, op->id);
	}

	if (is_set(op->call_options, st_opt_topology) && op->devices) {
	/* Ignore any preference, they might not have the device we need */
	/* When using topology, the stonith_choose_peer function pops off
	* the peer from the op's query results. Make sure to calculate
	* the op_timeout before calling this function when topology is in use */
	peer = stonith_choose_peer(op);
	device = op->devices->data;
	timeout = get_device_timeout(peer, device, op->base_timeout);
	}

	if (peer) {
	int timeout_one = 0;
	xmlNode *remote_op = stonith_create_op(op->client_callid, op->id, STONITH_OP_FENCE, NULL, 0);

	crm_xml_add(remote_op, F_STONITH_REMOTE_OP_ID, op->id);
	crm_xml_add(remote_op, F_STONITH_TARGET, op->target);
	crm_xml_add(remote_op, F_STONITH_ACTION, op->action);
	crm_xml_add(remote_op, F_STONITH_ORIGIN, op->originator);
	crm_xml_add(remote_op, F_STONITH_CLIENTID, op->client_id);
	crm_xml_add(remote_op, F_STONITH_CLIENTNAME, op->client_name);
	crm_xml_add_int(remote_op, F_STONITH_TIMEOUT, timeout);
	crm_xml_add_int(remote_op, F_STONITH_CALLOPTS, op->call_options);

	if (device) {
	timeout_one =
	TIMEOUT_MULTIPLY_FACTOR * get_device_timeout(peer, device, op->base_timeout);
	crm_info("Requesting that %s perform op %s %s with %s for %s (%ds)", peer->host,
	op->action, op->target, device, op->client_name, timeout_one);
	crm_xml_add(remote_op, F_STONITH_DEVICE, device);
	crm_xml_add(remote_op, F_STONITH_MODE, "slave");

	} else {
	timeout_one = TIMEOUT_MULTIPLY_FACTOR * get_peer_timeout(peer, op->base_timeout);
	crm_info("Requesting that %s perform op %s %s for %s (%ds)",
	peer->host, op->action, op->target, op->client_name, timeout_one);
	crm_xml_add(remote_op, F_STONITH_MODE, "smart");
	}

	op->state = st_exec;
	if (op->op_timer_one) {
	g_source_remove(op->op_timer_one);
	}

	if(device && stonith_watchdog_timeout_ms && safe_str_eq(device, "watchdog")) {
	op->op_timer_one = g_timeout_add(stonith_watchdog_timeout_ms, remote_op_watchdog_done, op);

	} else {
	op->op_timer_one = g_timeout_add((1000 * timeout_one), remote_op_timeout_one, op);
	}


	send_cluster_message(crm_get_peer(0, peer->host), crm_msg_stonith_ng, remote_op, FALSE);
	peer->tried = TRUE;
	free_xml(remote_op);
	return;

	} else if (op->owner == FALSE) {
	crm_err("The termination of %s for %s is not ours to control", op->target, op->client_name);

	} else if (op->query_timer == 0) {
	/* We've exhausted all available peers */
	crm_info("No remaining peers capable of terminating %s for %s (%d)", op->target,
	op->client_name, op->state);
	CRM_LOG_ASSERT(op->state < st_done);
	remote_op_timeout(op);

	} else if(op->replies >= op->replies_expected \|\| op->replies >= fencing_active_peers()) {
	int rc = -EHOSTUNREACH;

	/* if the operation never left the query state,
	* but we have all the expected replies, then no devices
	* are available to execute the fencing operation. */
	if (op->state == st_query) {
	crm_info("None of the %d peers have devices capable of terminating %s for %s (%d)",
	op->replies, op->target, op->client_name, op->state);

	rc = -ENODEV;
	} else {
	crm_info("None of the %d peers are capable of terminating %s for %s (%d)",
	op->replies, op->target, op->client_name, op->state);
	}

	op->state = st_failed;
	remote_op_done(op, NULL, rc, FALSE);

	} else if (device) {
	crm_info("Waiting for additional peers capable of terminating %s with %s for %s.%.8s",
	op->target, device, op->client_name, op->id);
	} else {
	crm_info("Waiting for additional peers capable of terminating %s for %s%.8s",
	op->target, op->client_name, op->id);
	}
	}

	static gint
	sort_peers(gconstpointer a, gconstpointer b)
	{
	const st_query_result_t *peer_a = a;
	const st_query_result_t *peer_b = a;

	if (peer_a->devices > peer_b->devices) {
	return -1;
	} else if (peer_a->devices > peer_b->devices) {
	return 1;
	}
	return 0;
	}

	/*!
	* \internal
	* \brief Determine if all the devices in the topology are found or not
	*/
	static gboolean
	all_topology_devices_found(remote_fencing_op_t * op)
	{
	GListPtr device = NULL;
	GListPtr iter = NULL;
	GListPtr match = NULL;
	stonith_topology_t *tp = NULL;
	gboolean skip_target = FALSE;
	int i;

	tp = g_hash_table_lookup(topology, op->target);

	if (!tp) {
	return FALSE;
	}
	if (safe_str_eq(op->action, "off") \|\| safe_str_eq(op->action, "reboot")) {
	/* Don't count the devices on the target node if we are killing
	* the target node. */
	skip_target = TRUE;
	}

	for (i = 0; i < ST_LEVEL_MAX; i++) {
	for (device = tp->levels[i]; device; device = device->next) {
	match = FALSE;
	for (iter = op->query_results; iter != NULL; iter = iter->next) {
	st_query_result_t *peer = iter->data;

	if (skip_target && safe_str_eq(peer->host, op->target)) {
	continue;
	}
	match = g_list_find_custom(peer->device_list, device->data, sort_strings);
	}
	if (!match) {
	return FALSE;
	}
	}
	}

	return TRUE;
	}

	int
	process_remote_stonith_query(xmlNode * msg)
	{
	int devices = 0;
	gboolean host_is_target = FALSE;
	const char *id = NULL;
	const char *host = NULL;
	remote_fencing_op_t *op = NULL;
	st_query_result_t *result = NULL;
	uint32_t active = fencing_active_peers();
	xmlNode *dev = get_xpath_object("//@" F_STONITH_REMOTE_OP_ID, msg, LOG_ERR);
	xmlNode *child = NULL;

	CRM_CHECK(dev != NULL, return -EPROTO);

	id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);
	CRM_CHECK(id != NULL, return -EPROTO);

	dev = get_xpath_object("//@st-available-devices", msg, LOG_ERR);
	CRM_CHECK(dev != NULL, return -EPROTO);
	crm_element_value_int(dev, "st-available-devices", &devices);

	op = g_hash_table_lookup(remote_op_list, id);
	if (op == NULL) {
	crm_debug("Unknown or expired remote op: %s", id);
	return -EOPNOTSUPP;
	}

	op->replies++;
	host = crm_element_value(msg, F_ORIG);
	host_is_target = safe_str_eq(host, op->target);

	if (devices <= 0) {
	/* If we're doing 'known' then we might need to fire anyway */
	crm_trace("Query result %d of %d from %s for %s/%s (%d devices) %s",
	op->replies, op->replies_expected, host,
	op->target, op->action, devices, id);
	if(op->state == st_query && (op->replies >= op->replies_expected \|\| op->replies >= active)) {
	crm_info("All queries have arrived, continuing (%d, %d, %d, %s)",
	op->replies_expected, active, op->replies, id);
	call_remote_stonith(op, NULL);
	}
	return pcmk_ok;
	}

	crm_info("Query result %d of %d from %s for %s/%s (%d devices) %s",
	op->replies, op->replies_expected, host,
	op->target, op->action, devices, id);
	result = calloc(1, sizeof(st_query_result_t));
	result->host = strdup(host);
	result->devices = devices;
	result->custom_action_timeouts = g_hash_table_new_full(crm_str_hash, g_str_equal, free, NULL);
	result->verified_devices = g_hash_table_new_full(crm_str_hash, g_str_equal, free, NULL);

	for (child = __xml_first_child(dev); child != NULL; child = __xml_next(child)) {
	const char *device = ID(child);
	int action_timeout = 0;
	int verified = 0;
	int required = 0;

	if (device) {
	result->device_list = g_list_prepend(result->device_list, strdup(device));
	crm_element_value_int(child, F_STONITH_ACTION_TIMEOUT, &action_timeout);
	crm_element_value_int(child, F_STONITH_DEVICE_VERIFIED, &verified);
	crm_element_value_int(child, F_STONITH_DEVICE_REQUIRED, &required);
	if (action_timeout) {
	crm_trace("Peer %s with device %s returned action timeout %d",
	result->host, device, action_timeout);
	g_hash_table_insert(result->custom_action_timeouts,
	strdup(device), GINT_TO_POINTER(action_timeout));
	}
	if (verified) {
	crm_trace("Peer %s has confirmed a verified device %s", result->host, device);
	g_hash_table_insert(result->verified_devices,
	strdup(device), GINT_TO_POINTER(verified));
	}
	if (required) {
	crm_trace("Peer %s requires device %s to execute for action %s",
	result->host, device, op->action);
	/* This matters when executing a topology. Required devices will get
	* executed regardless of their topology level. We use this for unfencing. */
	add_required_device(op, device);
	}
	}
	}

	CRM_CHECK(devices == g_list_length(result->device_list),
	crm_err("Mis-match: Query claimed to have %d devices but %d found", devices,
	g_list_length(result->device_list)));

	op->query_results = g_list_insert_sorted(op->query_results, result, sort_peers);

	if (is_set(op->call_options, st_opt_topology)) {
	/* If we start the fencing before all the topology results are in,
	* it is possible fencing levels will be skipped because of the missing
	* query results. */
	if (op->state == st_query && all_topology_devices_found(op)) {
	/* All the query results are in for the topology, start the fencing ops. */
	crm_trace("All topology devices found");
	call_remote_stonith(op, result);

	} else if(op->state == st_query && (op->replies >= op->replies_expected \|\| op->replies >= active)) {
	crm_info("All topology queries have arrived, continuing (%d, %d, %d) ", op->replies_expected, active, op->replies);
	call_remote_stonith(op, NULL);
	}

	} else if (op->state == st_query) {
	/* We have a result for a non-topology fencing op that looks promising,
	* go ahead and start fencing before query timeout */
	if (host_is_target == FALSE && g_hash_table_size(result->verified_devices)) {
	/* we have a verified device living on a peer that is not the target */
	crm_trace("Found %d verified devices", g_hash_table_size(result->verified_devices));
	call_remote_stonith(op, result);

	} else if(op->replies >= op->replies_expected \|\| op->replies >= active) {
	crm_info("All queries have arrived, continuing (%d, %d, %d) ", op->replies_expected, active, op->replies);
	call_remote_stonith(op, NULL);

	} else {
	crm_trace("Waiting for more peer results before launching fencing operation");
	}

	} else if (op->state == st_done) {
	crm_info("Discarding query result from %s (%d devices): Operation is in state %d",
	result->host, result->devices, op->state);
	}

	return pcmk_ok;
	}

	int
	process_remote_stonith_exec(xmlNode * msg)
	{
	int rc = 0;
	const char *id = NULL;
	const char *device = NULL;
	remote_fencing_op_t *op = NULL;
	xmlNode *dev = get_xpath_object("//@" F_STONITH_REMOTE_OP_ID, msg, LOG_ERR);

	CRM_CHECK(dev != NULL, return -EPROTO);

	id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);
	CRM_CHECK(id != NULL, return -EPROTO);

	dev = get_xpath_object("//@" F_STONITH_RC, msg, LOG_ERR);
	CRM_CHECK(dev != NULL, return -EPROTO);

	crm_element_value_int(dev, F_STONITH_RC, &rc);

	device = crm_element_value(dev, F_STONITH_DEVICE);

	if (remote_op_list) {
	op = g_hash_table_lookup(remote_op_list, id);
	}

	if (op == NULL && rc == pcmk_ok) {
	/* Record successful fencing operations */
	const char *client_id = crm_element_value(dev, F_STONITH_CLIENTID);

	op = create_remote_stonith_op(client_id, dev, TRUE);
	}

	if (op == NULL) {
	/* Could be for an event that began before we started */
	/* TODO: Record the op for later querying */
	crm_info("Unknown or expired remote op: %s", id);
	return -EOPNOTSUPP;
	}

	if (op->devices && device && safe_str_neq(op->devices->data, device)) {
	crm_err
	("Received outdated reply for device %s (instead of %s) to %s node %s. Operation already timed out at remote level.",
	device, op->devices->data, op->action, op->target);
	return rc;
	}

	if (safe_str_eq(crm_element_value(msg, F_SUBTYPE), "broadcast")) {
	crm_debug("Marking call to %s for %s on behalf of %s@%s.%.8s: %s (%d)",
	op->action, op->target, op->client_name, op->id, op->originator,
	pcmk_strerror(rc), rc);
	if (rc == pcmk_ok) {
	op->state = st_done;
	} else {
	op->state = st_failed;
	}
	remote_op_done(op, msg, rc, FALSE);
	return pcmk_ok;
	} else if (safe_str_neq(op->originator, stonith_our_uname)) {
	/* If this isn't a remote level broadcast, and we are not the
	* originator of the operation, we should not be receiving this msg. */
	crm_err
	("%s received non-broadcast fencing result for operation it does not own (device %s targeting %s)",
	stonith_our_uname, device, op->target);
	return rc;
	}

	if (is_set(op->call_options, st_opt_topology)) {
	const char *device = crm_element_value(msg, F_STONITH_DEVICE);

	crm_notice("Call to %s for %s on behalf of %s@%s: %s (%d)",
	device, op->target, op->client_name, op->originator,
	pcmk_strerror(rc), rc);

	/* We own the op, and it is complete. broadcast the result to all nodes
	* and notify our local clients. */
	if (op->state == st_done) {
	remote_op_done(op, msg, rc, FALSE);
	return rc;
	}

	/* An operation completed succesfully but has not yet been marked as done.
	* Continue the topology if more devices exist at the current level, otherwise
	* mark as done. */
	if (rc == pcmk_ok) {
	GListPtr required_match = g_list_find_custom(op->required_list, device, sort_strings);
	if (op->devices) {
	/* Success, are there any more? */
	op->devices = op->devices->next;
	}
	if (required_match) {
	op->required_list = g_list_remove(op->required_list, required_match->data);
	}
	/* if no more devices at this fencing level, we are done,
	* else we need to contine with executing the next device in the list */
	if (op->devices == NULL) {
	crm_trace("Marking complex fencing op for %s as complete", op->target);
	op->state = st_done;
	remote_op_done(op, msg, rc, FALSE);
	return rc;
	}
	} else {
	/* This device failed, time to try another topology level. If no other
	* levels are available, mark this operation as failed and report results. */
	if (stonith_topology_next(op) != pcmk_ok) {
	op->state = st_failed;
	remote_op_done(op, msg, rc, FALSE);
	return rc;
	}
	}
	} else if (rc == pcmk_ok && op->devices == NULL) {
	crm_trace("All done for %s", op->target);

	op->state = st_done;
	remote_op_done(op, msg, rc, FALSE);
	return rc;
	} else if (rc == -ETIME && op->devices == NULL) {
	/* If the operation timed out don't bother retrying other peers. */
	op->state = st_failed;
	remote_op_done(op, msg, rc, FALSE);
	return rc;
	} else {
	/* fall-through and attempt other fencing action using another peer */
	}

	/* Retry on failure or execute the rest of the topology */
	crm_trace("Next for %s on behalf of %s@%s (rc was %d)", op->target, op->originator,
	op->client_name, rc);
	call_remote_stonith(op, NULL);
	return rc;
	}

	int
	stonith_fence_history(xmlNode * msg, xmlNode ** output)
	{
	int rc = 0;
	const char *target = NULL;
	xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_TRACE);

	if (dev) {
	int options = 0;

	target = crm_element_value(dev, F_STONITH_TARGET);
	crm_element_value_int(msg, F_STONITH_CALLOPTS, &options);
	if (target && (options & st_opt_cs_nodeid)) {
	int nodeid = crm_atoi(target, NULL);
	crm_node_t *node = crm_get_peer(nodeid, NULL);

	if (node) {
	target = node->uname;
	}
	}
	}

	crm_trace("Looking for operations on %s in %p", target, remote_op_list);

	*output = create_xml_node(NULL, F_STONITH_HISTORY_LIST);
	if (remote_op_list) {
	GHashTableIter iter;
	remote_fencing_op_t *op = NULL;

	g_hash_table_iter_init(&iter, remote_op_list);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
	xmlNode *entry = NULL;

	if (target && strcmp(op->target, target) != 0) {
	continue;
	}

	rc = 0;
	crm_trace("Attaching op %s", op->id);
	entry = create_xml_node(*output, STONITH_OP_EXEC);
	crm_xml_add(entry, F_STONITH_TARGET, op->target);
	crm_xml_add(entry, F_STONITH_ACTION, op->action);
	crm_xml_add(entry, F_STONITH_ORIGIN, op->originator);
	crm_xml_add(entry, F_STONITH_DELEGATE, op->delegate);
	crm_xml_add(entry, F_STONITH_CLIENTNAME, op->client_name);
	crm_xml_add_int(entry, F_STONITH_DATE, op->completed);
	crm_xml_add_int(entry, F_STONITH_STATE, op->state);
	}
	}

	return rc;
	}

	gboolean
	stonith_check_fence_tolerance(int tolerance, const char target, const char action)
	{
	GHashTableIter iter;
	time_t now = time(NULL);
	remote_fencing_op_t *rop = NULL;

	crm_trace("tolerance=%d, remote_op_list=%p", tolerance, remote_op_list);

	if (tolerance <= 0 \|\| !remote_op_list \|\| target == NULL \|\| action == NULL) {
	return FALSE;
	}

	g_hash_table_iter_init(&iter, remote_op_list);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&rop)) {
	if (strcmp(rop->target, target) != 0) {
	continue;
	} else if (rop->state != st_done) {
	continue;
	} else if (strcmp(rop->action, action) != 0) {
	continue;
	} else if ((rop->completed + tolerance) < now) {
	continue;
	}

	crm_notice("Target %s was fenced (%s) less than %ds ago by %s on behalf of %s",
	target, action, tolerance, rop->delegate, rop->originator);
	return TRUE;
	}
	return FALSE;
	}

File Metadata

Mime Type: text/x-diff
Expires: Tue, Jul 8, 3:45 PM (14 h, 54 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2002214
Default Alt Text: (72 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions