No OneTemporary
Actions

Size

70 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/daemons/execd/pacemaker-execd.c b/daemons/execd/pacemaker-execd.c
	index d848ec328b..718afe5938 100644
	--- a/daemons/execd/pacemaker-execd.c
	+++ b/daemons/execd/pacemaker-execd.c
	@@ -1,523 +1,533 @@
	/*
	* Copyright 2012-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <signal.h>
	#include <sys/types.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/services.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/remote_internal.h>
	#include <crm/lrmd_internal.h>

	#include "pacemaker-execd.h"

	static GMainLoop *mainloop = NULL;
	static qb_ipcs_service_t *ipcs = NULL;
	static stonith_t *stonith_api = NULL;
	int lrmd_call_id = 0;

	#ifdef PCMK__COMPILE_REMOTE
	/* whether shutdown request has been sent */
	static sig_atomic_t shutting_down = FALSE;

	/* timer for waiting for acknowledgment of shutdown request */
	static guint shutdown_ack_timer = 0;

	static gboolean lrmd_exit(gpointer data);
	#endif

	static void
	stonith_connection_destroy_cb(stonith_t * st, stonith_event_t * e)
	{
	stonith_api->state = stonith_disconnected;
	crm_err("Connection to fencer lost");
	stonith_connection_failed();
	}

	stonith_t *
	get_stonith_connection(void)
	{
	if (stonith_api && stonith_api->state == stonith_disconnected) {
	stonith_api_delete(stonith_api);
	stonith_api = NULL;
	}

	if (stonith_api == NULL) {
	int rc = pcmk_ok;

	stonith_api = stonith_api_new();
	if (stonith_api == NULL) {
	crm_err("Could not connect to fencer: API memory allocation failed");
	return NULL;
	}
	rc = stonith_api_connect_retry(stonith_api, crm_system_name, 10);
	if (rc != pcmk_ok) {
	crm_err("Could not connect to fencer in 10 attempts: %s "
	CRM_XS " rc=%d", pcmk_strerror(rc), rc);
	stonith_api_delete(stonith_api);
	stonith_api = NULL;
	} else {
	stonith_api->cmds->register_notification(stonith_api,
	T_STONITH_NOTIFY_DISCONNECT,
	stonith_connection_destroy_cb);
	}
	}
	return stonith_api;
	}

	static int32_t
	lrmd_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	crm_trace("Connection %p", c);
	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	static void
	lrmd_ipc_created(qb_ipcs_connection_t * c)
	{
	pcmk__client_t *new_client = pcmk__find_client(c);

	crm_trace("Connection %p", c);
	CRM_ASSERT(new_client != NULL);
	/* Now that the connection is offically established, alert
	* the other clients a new connection exists. */

	notify_of_new_client(new_client);
	}

	static int32_t
	lrmd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	pcmk__client_t *client = pcmk__find_client(c);
	xmlNode *request = pcmk__client_data2xml(client, data, &id, &flags);

	CRM_CHECK(client != NULL, crm_err("Invalid client");
	return FALSE);
	CRM_CHECK(client->id != NULL, crm_err("Invalid client: %p", client);
	return FALSE);

	CRM_CHECK(flags & crm_ipc_client_response, crm_err("Invalid client request: %p", client);
	return FALSE);

	if (!request) {
	return 0;
	}

	if (!client->name) {
	const char *value = crm_element_value(request, F_LRMD_CLIENTNAME);

	if (value == NULL) {
	client->name = pcmk__itoa(pcmk__client_pid(c));
	} else {
	client->name = strdup(value);
	}
	}

	lrmd_call_id++;
	if (lrmd_call_id < 1) {
	lrmd_call_id = 1;
	}

	crm_xml_add(request, F_LRMD_CLIENTID, client->id);
	crm_xml_add(request, F_LRMD_CLIENTNAME, client->name);
	crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id);

	process_lrmd_message(client, id, request);

	free_xml(request);
	return 0;
	}

	/*!
	* \internal
	* \brief Free a client connection, and exit if appropriate
	*
	* \param[in] client Client connection to free
	*/
	void
	lrmd_client_destroy(pcmk__client_t *client)
	{
	pcmk__free_client(client);

	#ifdef PCMK__COMPILE_REMOTE
	/* If we were waiting to shut down, we can now safely do so
	* if there are no more proxied IPC providers
	*/
	if (shutting_down && (ipc_proxy_get_provider() == NULL)) {
	lrmd_exit(NULL);
	}
	#endif
	}

	static int32_t
	lrmd_ipc_closed(qb_ipcs_connection_t * c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client == NULL) {
	return 0;
	}

	crm_trace("Connection %p", c);
	client_disconnect_cleanup(client->id);
	#ifdef PCMK__COMPILE_REMOTE
	ipc_proxy_remove_provider(client);
	#endif
	lrmd_client_destroy(client);
	return 0;
	}

	static void
	lrmd_ipc_destroy(qb_ipcs_connection_t * c)
	{
	lrmd_ipc_closed(c);
	crm_trace("Connection %p", c);
	}

	static struct qb_ipcs_service_handlers lrmd_ipc_callbacks = {
	.connection_accept = lrmd_ipc_accept,
	.connection_created = lrmd_ipc_created,
	.msg_process = lrmd_ipc_dispatch,
	.connection_closed = lrmd_ipc_closed,
	.connection_destroyed = lrmd_ipc_destroy
	};

	// \return Standard Pacemaker return code
	int
	lrmd_server_send_reply(pcmk__client_t client, uint32_t id, xmlNode reply)
	{
	crm_trace("Sending reply (%d) to client (%s)", id, client->id);
	switch (PCMK__CLIENT_TYPE(client)) {
	case pcmk__client_ipc:
	return pcmk__ipc_send_xml(client, id, reply, FALSE);
	#ifdef PCMK__COMPILE_REMOTE
	case pcmk__client_tls:
	return lrmd__remote_send_xml(client->remote, reply, id, "reply");
	#endif
	default:
	crm_err("Could not send reply: unknown type for client %s "
	CRM_XS " flags=0x%llx",
	pcmk__client_name(client), client->flags);
	}
	return ENOTCONN;
	}

	// \return Standard Pacemaker return code
	int
	lrmd_server_send_notify(pcmk__client_t client, xmlNode msg)
	{
	crm_trace("Sending notification to client (%s)", client->id);
	switch (PCMK__CLIENT_TYPE(client)) {
	case pcmk__client_ipc:
	if (client->ipcs == NULL) {
	crm_trace("Could not notify local client: disconnected");
	return ENOTCONN;
	}
	return pcmk__ipc_send_xml(client, 0, msg, crm_ipc_server_event);
	#ifdef PCMK__COMPILE_REMOTE
	case pcmk__client_tls:
	if (client->remote == NULL) {
	crm_trace("Could not notify remote client: disconnected");
	return ENOTCONN;
	} else {
	return lrmd__remote_send_xml(client->remote, msg, 0, "notify");
	}
	#endif
	default:
	crm_err("Could not notify client %s with unknown transport "
	CRM_XS " flags=0x%llx",
	pcmk__client_name(client), client->flags);
	}
	return ENOTCONN;
	}

	/*!
	* \internal
	* \brief Clean up and exit immediately
	*
	* \param[in] data Ignored
	*
	* \return Doesn't return
	* \note This can be used as a timer callback.
	*/
	static gboolean
	lrmd_exit(gpointer data)
	{
	crm_info("Terminating with %d clients", pcmk__ipc_client_count());
	if (stonith_api) {
	stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT);
	stonith_api->cmds->disconnect(stonith_api);
	stonith_api_delete(stonith_api);
	}
	if (ipcs) {
	mainloop_del_ipc_server(ipcs);
	}

	#ifdef PCMK__COMPILE_REMOTE
	lrmd_tls_server_destroy();
	ipc_proxy_cleanup();
	#endif

	pcmk__client_cleanup();
	g_hash_table_destroy(rsc_list);

	if (mainloop) {
	lrmd_drain_alerts(mainloop);
	}

	crm_exit(CRM_EX_OK);
	return FALSE;
	}

	/*!
	* \internal
	* \brief Request cluster shutdown if appropriate, otherwise exit immediately
	*
	* \param[in] nsig Signal that caused invocation (ignored)
	*/
	static void
	lrmd_shutdown(int nsig)
	{
	#ifdef PCMK__COMPILE_REMOTE
	pcmk__client_t *ipc_proxy = ipc_proxy_get_provider();

	/* If there are active proxied IPC providers, then we may be running
	* resources, so notify the cluster that we wish to shut down.
	*/
	if (ipc_proxy) {
	if (shutting_down) {
	crm_notice("Waiting for cluster to stop resources before exiting");
	return;
	}

	crm_info("Sending shutdown request to cluster");
	if (ipc_proxy_shutdown_req(ipc_proxy) < 0) {
	crm_crit("Shutdown request failed, exiting immediately");

	} else {
	/* We requested a shutdown. Now, we need to wait for an
	* acknowledgement from the proxy host (which ensures the proxy host
	* supports shutdown requests), then wait for all proxy hosts to
	* disconnect (which ensures that all resources have been stopped).
	*/
	shutting_down = TRUE;

	/* Stop accepting new proxy connections */
	lrmd_tls_server_destroy();

	/* Older controller versions will never acknowledge our request, so
	* set a fairly short timeout to exit quickly in that case. If we
	* get the ack, we'll defuse this timer.
	*/
	shutdown_ack_timer = g_timeout_add_seconds(20, lrmd_exit, NULL);

	/* Currently, we let the OS kill us if the clients don't disconnect
	* in a reasonable time. We could instead set a long timer here
	* (shorter than what the OS is likely to use) and exit immediately
	* if it pops.
	*/
	return;
	}
	}
	#endif
	lrmd_exit(NULL);
	}

	/*!
	* \internal
	* \brief Defuse short exit timer if shutting down
	*/
	void handle_shutdown_ack()
	{
	#ifdef PCMK__COMPILE_REMOTE
	if (shutting_down) {
	crm_info("Received shutdown ack");
	if (shutdown_ack_timer > 0) {
	g_source_remove(shutdown_ack_timer);
	shutdown_ack_timer = 0;
	}
	return;
	}
	#endif
	crm_debug("Ignoring unexpected shutdown ack");
	}

	/*!
	* \internal
	* \brief Make short exit timer fire immediately
	*/
	void handle_shutdown_nack()
	{
	#ifdef PCMK__COMPILE_REMOTE
	if (shutting_down) {
	crm_info("Received shutdown nack");
	if (shutdown_ack_timer > 0) {
	g_source_remove(shutdown_ack_timer);
	shutdown_ack_timer = g_timeout_add(0, lrmd_exit, NULL);
	}
	return;
	}
	#endif
	crm_debug("Ignoring unexpected shutdown nack");
	}

	static pcmk__cli_option_t long_options[] = {
	// long option, argument type, storage, short option, description, flags
	{
	"help", no_argument, NULL, '?',
	"\tThis text", pcmk__option_default
	},
	{
	"version", no_argument, NULL, '$',
	"\tVersion information", pcmk__option_default
	},
	{
	"verbose", no_argument, NULL, 'V',
	"\tIncrease debug output", pcmk__option_default
	},
	{
	"logfile", required_argument, NULL, 'l',
	"\tSend logs to the additional named logfile", pcmk__option_default
	},
	#ifdef PCMK__COMPILE_REMOTE
	{
	"port", required_argument, NULL, 'p',
	"\tPort to listen on", pcmk__option_default
	},
	#endif
	{ 0, 0, 0, 0 }
	};

	#ifdef PCMK__COMPILE_REMOTE
	# define EXECD_TYPE "remote"
	# define EXECD_NAME "pacemaker-remoted"
	# define EXECD_DESC "resource agent executor daemon for Pacemaker Remote nodes"
	#else
	# define EXECD_TYPE "local"
	# define EXECD_NAME "pacemaker-execd"
	# define EXECD_DESC "resource agent executor daemon for Pacemaker cluster nodes"
	#endif

	int
	main(int argc, char argv, char envp)
	{
	int flag = 0;
	int index = 0;
	int bump_log_num = 0;
	const char *option = NULL;

	#ifdef PCMK__COMPILE_REMOTE
	// If necessary, create PID 1 now before any file descriptors are opened
	remoted_spawn_pidone(argc, argv, envp);
	#endif

	crm_log_preinit(EXECD_NAME, argc, argv);
	pcmk__set_cli_options(NULL, "[options]", long_options, EXECD_DESC);

	while (1) {
	flag = pcmk__next_cli_option(argc, argv, &index, NULL);
	if (flag == -1) {
	break;
	}

	switch (flag) {
	case 'l':
	- pcmk__add_logfile(optarg);
	+ {
	+ int rc = pcmk__add_logfile(optarg);
	+
	+ if (rc != pcmk_rc_ok) {
	+ /* Logging has not yet been initialized, so stderr is
	+ * the only way to get information out
	+ */
	+ fprintf(stderr, "Logging to %s is disabled: %s\n",
	+ optarg, pcmk_rc_str(rc));
	+ }
	+ }
	break;
	case 'p':
	setenv("PCMK_remote_port", optarg, 1);
	break;
	case 'V':
	bump_log_num++;
	break;
	case '?':
	case '$':
	pcmk__cli_help(flag, CRM_EX_OK);
	break;
	default:
	pcmk__cli_help('?', CRM_EX_USAGE);
	break;
	}
	}

	crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);

	while (bump_log_num > 0) {
	crm_bump_log_level(argc, argv);
	bump_log_num--;
	}

	option = pcmk__env_option("logfacility");
	if (option && !pcmk__strcase_any_of(option, "none", "/dev/null", NULL)) {
	setenv("HA_LOGFACILITY", option, 1); /* Used by the ocf_log/ha_log OCF macro */
	}

	option = pcmk__env_option("logfile");
	if(option && !pcmk__str_eq(option, "none", pcmk__str_casei)) {
	setenv("HA_LOGFILE", option, 1); /* Used by the ocf_log/ha_log OCF macro */

	if (pcmk__env_option_enabled(crm_system_name, "debug")) {
	setenv("HA_DEBUGLOG", option, 1); /* Used by the ocf_log/ha_debug OCF macro */
	}
	}

	crm_notice("Starting Pacemaker " EXECD_TYPE " executor");

	/* The presence of this variable allegedly controls whether child
	* processes like httpd will try and use Systemd's sd_notify
	* API
	*/
	unsetenv("NOTIFY_SOCKET");

	{
	// Temporary directory for resource agent use (leave owned by root)
	int rc = pcmk__build_path(CRM_RSCTMP_DIR, 0755);

	if (rc != pcmk_rc_ok) {
	crm_warn("Could not create resource agent temporary directory "
	CRM_RSCTMP_DIR ": %s", pcmk_rc_str(rc));
	}
	}

	rsc_list = pcmk__strkey_table(NULL, free_rsc);
	ipcs = mainloop_add_ipc_server(CRM_SYSTEM_LRMD, QB_IPC_SHM, &lrmd_ipc_callbacks);
	if (ipcs == NULL) {
	crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
	crm_exit(CRM_EX_FATAL);
	}

	#ifdef PCMK__COMPILE_REMOTE
	if (lrmd_init_remote_tls_server() < 0) {
	crm_err("Failed to create TLS listener: shutting down and staying down");
	crm_exit(CRM_EX_FATAL);
	}
	ipc_proxy_init();
	#endif

	mainloop_add_signal(SIGTERM, lrmd_shutdown);
	mainloop = g_main_loop_new(NULL, FALSE);
	crm_notice("Pacemaker " EXECD_TYPE " executor successfully started and accepting connections");
	g_main_loop_run(mainloop);

	/* should never get here */
	lrmd_exit(NULL);
	return CRM_EX_OK;
	}
	diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c
	index f4ab84fd7c..39738d8be9 100644
	--- a/daemons/fenced/pacemaker-fenced.c
	+++ b/daemons/fenced/pacemaker-fenced.c
	@@ -1,1607 +1,1617 @@
	/*
	* Copyright 2009-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <inttypes.h> // PRIu32, PRIx32

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/cluster/internal.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <crm/common/mainloop.h>

	#include <crm/cib/internal.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	#include <pacemaker-fenced.h>

	char *stonith_our_uname = NULL;
	long stonith_watchdog_timeout_ms = 0;

	static GMainLoop *mainloop = NULL;

	gboolean stand_alone = FALSE;
	static gboolean no_cib_connect = FALSE;
	static gboolean stonith_shutdown_flag = FALSE;

	static qb_ipcs_service_t *ipcs = NULL;
	static xmlNode *local_cib = NULL;
	static pe_working_set_t *fenced_data_set = NULL;

	static cib_t *cib_api = NULL;

	static pcmk__output_t *out = NULL;

	pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_LOG,
	PCMK__SUPPORTED_FORMAT_NONE,
	PCMK__SUPPORTED_FORMAT_TEXT,
	{ NULL, NULL, NULL }
	};

	static void stonith_shutdown(int nsig);
	static void stonith_cleanup(void);

	static int32_t
	st_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	if (stonith_shutdown_flag) {
	crm_info("Ignoring new client [%d] during shutdown",
	pcmk__client_pid(c));
	return -EPERM;
	}

	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	/* Exit code means? */
	static int32_t
	st_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	int call_options = 0;
	xmlNode *request = NULL;
	pcmk__client_t *c = pcmk__find_client(qbc);
	const char *op = NULL;

	if (c == NULL) {
	crm_info("Invalid client: %p", qbc);
	return 0;
	}

	request = pcmk__client_data2xml(c, data, &id, &flags);
	if (request == NULL) {
	pcmk__ipc_send_ack(c, id, flags, "nack", CRM_EX_PROTOCOL);
	return 0;
	}


	op = crm_element_value(request, F_CRM_TASK);
	if(pcmk__str_eq(op, CRM_OP_RM_NODE_CACHE, pcmk__str_casei)) {
	crm_xml_add(request, F_TYPE, T_STONITH_NG);
	crm_xml_add(request, F_STONITH_OPERATION, op);
	crm_xml_add(request, F_STONITH_CLIENTID, c->id);
	crm_xml_add(request, F_STONITH_CLIENTNAME, pcmk__client_name(c));
	crm_xml_add(request, F_STONITH_CLIENTNODE, stonith_our_uname);

	send_cluster_message(NULL, crm_msg_stonith_ng, request, FALSE);
	free_xml(request);
	return 0;
	}

	if (c->name == NULL) {
	const char *value = crm_element_value(request, F_STONITH_CLIENTNAME);

	if (value == NULL) {
	value = "unknown";
	}
	c->name = crm_strdup_printf("%s.%u", value, c->pid);
	}

	crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
	crm_trace("Flags 0x%08" PRIx32 "/0x%08x for command %" PRIu32
	" from client %s", flags, call_options, id, pcmk__client_name(c));

	if (pcmk_is_set(call_options, st_opt_sync_call)) {
	CRM_ASSERT(flags & crm_ipc_client_response);
	CRM_LOG_ASSERT(c->request_id == 0); /* This means the client has two synchronous events in-flight */
	c->request_id = id; /* Reply only to the last one */
	}

	crm_xml_add(request, F_STONITH_CLIENTID, c->id);
	crm_xml_add(request, F_STONITH_CLIENTNAME, pcmk__client_name(c));
	crm_xml_add(request, F_STONITH_CLIENTNODE, stonith_our_uname);

	stonith_command(c, id, flags, request, NULL);

	free_xml(request);
	return 0;
	}

	/* Error code means? */
	static int32_t
	st_ipc_closed(qb_ipcs_connection_t * c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client == NULL) {
	return 0;
	}

	crm_trace("Connection %p closed", c);
	pcmk__free_client(client);

	/* 0 means: yes, go ahead and destroy the connection */
	return 0;
	}

	static void
	st_ipc_destroy(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p destroyed", c);
	st_ipc_closed(c);
	}

	static void
	stonith_peer_callback(xmlNode * msg, void *private_data)
	{
	const char *remote_peer = crm_element_value(msg, F_ORIG);
	const char *op = crm_element_value(msg, F_STONITH_OPERATION);

	if (pcmk__str_eq(op, "poke", pcmk__str_none)) {
	return;
	}

	crm_log_xml_trace(msg, "Peer[inbound]");
	stonith_command(NULL, 0, 0, msg, remote_peer);
	}

	#if SUPPORT_COROSYNC
	static void
	stonith_peer_ais_callback(cpg_handle_t handle,
	const struct cpg_name *groupName,
	uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
	{
	uint32_t kind = 0;
	xmlNode *xml = NULL;
	const char *from = NULL;
	char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);

	if(data == NULL) {
	return;
	}
	if (kind == crm_class_cluster) {
	xml = string2xml(data);
	if (xml == NULL) {
	crm_err("Invalid XML: '%.120s'", data);
	free(data);
	return;
	}
	crm_xml_add(xml, F_ORIG, from);
	/* crm_xml_add_int(xml, F_SEQ, wrapper->id); */
	stonith_peer_callback(xml, NULL);
	}

	free_xml(xml);
	free(data);
	return;
	}

	static void
	stonith_peer_cs_destroy(gpointer user_data)
	{
	crm_crit("Lost connection to cluster layer, shutting down");
	stonith_shutdown(0);
	}
	#endif

	void
	do_local_reply(xmlNode * notify_src, const char *client_id, gboolean sync_reply, gboolean from_peer)
	{
	/* send callback to originating child */
	pcmk__client_t *client_obj = NULL;
	int local_rc = pcmk_rc_ok;

	crm_trace("Sending response");
	client_obj = pcmk__find_client_by_id(client_id);

	crm_trace("Sending callback to request originator");
	if (client_obj == NULL) {
	local_rc = EPROTO;
	crm_trace("No client to sent the response to. F_STONITH_CLIENTID not set.");

	} else {
	int rid = 0;

	if (sync_reply) {
	CRM_LOG_ASSERT(client_obj->request_id);

	rid = client_obj->request_id;
	client_obj->request_id = 0;

	crm_trace("Sending response %d to client %s%s",
	rid, pcmk__client_name(client_obj),
	(from_peer? " (originator of delegated request)" : ""));

	} else {
	crm_trace("Sending an event to client %s%s",
	pcmk__client_name(client_obj),
	(from_peer? " (originator of delegated request)" : ""));
	}

	local_rc = pcmk__ipc_send_xml(client_obj, rid, notify_src,
	(sync_reply? crm_ipc_flags_none
	: crm_ipc_server_event));
	}

	if ((local_rc != pcmk_rc_ok) && (client_obj != NULL)) {
	crm_warn("%s reply to client %s failed: %s",
	(sync_reply? "Synchronous" : "Asynchronous"),
	pcmk__client_name(client_obj), pcmk_rc_str(local_rc));
	}
	}

	uint64_t
	get_stonith_flag(const char *name)
	{
	if (pcmk__str_eq(name, T_STONITH_NOTIFY_FENCE, pcmk__str_casei)) {
	return st_callback_notify_fence;

	} else if (pcmk__str_eq(name, STONITH_OP_DEVICE_ADD, pcmk__str_casei)) {
	return st_callback_device_add;

	} else if (pcmk__str_eq(name, STONITH_OP_DEVICE_DEL, pcmk__str_casei)) {
	return st_callback_device_del;

	} else if (pcmk__str_eq(name, T_STONITH_NOTIFY_HISTORY, pcmk__str_casei)) {
	return st_callback_notify_history;

	} else if (pcmk__str_eq(name, T_STONITH_NOTIFY_HISTORY_SYNCED, pcmk__str_casei)) {
	return st_callback_notify_history_synced;

	}
	return st_callback_unknown;
	}

	static void
	stonith_notify_client(gpointer key, gpointer value, gpointer user_data)
	{

	xmlNode *update_msg = user_data;
	pcmk__client_t *client = value;
	const char *type = NULL;

	CRM_CHECK(client != NULL, return);
	CRM_CHECK(update_msg != NULL, return);

	type = crm_element_value(update_msg, F_SUBTYPE);
	CRM_CHECK(type != NULL, crm_log_xml_err(update_msg, "notify"); return);

	if (client->ipcs == NULL) {
	crm_trace("Skipping client with NULL channel");
	return;
	}

	if (pcmk_is_set(client->flags, get_stonith_flag(type))) {
	int rc = pcmk__ipc_send_xml(client, 0, update_msg,
	crm_ipc_server_event\|crm_ipc_server_error);

	if (rc != pcmk_rc_ok) {
	crm_warn("%s notification of client %s failed: %s "
	CRM_XS " id=%.8s rc=%d", type, pcmk__client_name(client),
	pcmk_rc_str(rc), client->id, rc);
	} else {
	crm_trace("Sent %s notification to client %s",
	type, pcmk__client_name(client));
	}
	}
	}

	void
	do_stonith_async_timeout_update(const char client_id, const char call_id, int timeout)
	{
	pcmk__client_t *client = NULL;
	xmlNode *notify_data = NULL;

	if (!timeout \|\| !call_id \|\| !client_id) {
	return;
	}

	client = pcmk__find_client_by_id(client_id);
	if (!client) {
	return;
	}

	notify_data = create_xml_node(NULL, T_STONITH_TIMEOUT_VALUE);
	crm_xml_add(notify_data, F_TYPE, T_STONITH_TIMEOUT_VALUE);
	crm_xml_add(notify_data, F_STONITH_CALLID, call_id);
	crm_xml_add_int(notify_data, F_STONITH_TIMEOUT, timeout);

	crm_trace("timeout update is %d for client %s and call id %s", timeout, client_id, call_id);

	if (client) {
	pcmk__ipc_send_xml(client, 0, notify_data, crm_ipc_server_event);
	}

	free_xml(notify_data);
	}

	void
	do_stonith_notify(int options, const char type, int result, xmlNode data)
	{
	/* TODO: Standardize the contents of data */
	xmlNode *update_msg = create_xml_node(NULL, "notify");

	CRM_CHECK(type != NULL,;);

	crm_xml_add(update_msg, F_TYPE, T_STONITH_NOTIFY);
	crm_xml_add(update_msg, F_SUBTYPE, type);
	crm_xml_add(update_msg, F_STONITH_OPERATION, type);
	crm_xml_add_int(update_msg, F_STONITH_RC, result);

	if (data != NULL) {
	add_message_xml(update_msg, F_STONITH_CALLDATA, data);
	}

	crm_trace("Notifying clients");
	pcmk__foreach_ipc_client(stonith_notify_client, update_msg);
	free_xml(update_msg);
	crm_trace("Notify complete");
	}

	static void
	do_stonith_notify_config(int options, const char *op, int rc,
	const char *desc, int active)
	{
	xmlNode *notify_data = create_xml_node(NULL, op);

	CRM_CHECK(notify_data != NULL, return);

	crm_xml_add(notify_data, F_STONITH_DEVICE, desc);
	crm_xml_add_int(notify_data, F_STONITH_ACTIVE, active);

	do_stonith_notify(options, op, rc, notify_data);
	free_xml(notify_data);
	}

	void
	do_stonith_notify_device(int options, const char op, int rc, const char desc)
	{
	do_stonith_notify_config(options, op, rc, desc, g_hash_table_size(device_list));
	}

	void
	do_stonith_notify_level(int options, const char op, int rc, const char desc)
	{
	do_stonith_notify_config(options, op, rc, desc, g_hash_table_size(topology));
	}

	static void
	topology_remove_helper(const char *node, int level)
	{
	int rc;
	char *desc = NULL;
	xmlNode *data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL);

	crm_xml_add(data, F_STONITH_ORIGIN, __func__);
	crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level);
	crm_xml_add(data, XML_ATTR_STONITH_TARGET, node);

	rc = stonith_level_remove(data, &desc);
	do_stonith_notify_level(0, STONITH_OP_LEVEL_DEL, rc, desc);

	free_xml(data);
	free(desc);
	}

	static void
	remove_cib_device(xmlXPathObjectPtr xpathObj)
	{
	int max = numXpathResults(xpathObj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	const char *rsc_id = NULL;
	const char *standard = NULL;
	xmlNode *match = getXpathResult(xpathObj, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if(match != NULL) {
	standard = crm_element_value(match, XML_AGENT_ATTR_CLASS);
	}

	if (!pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	continue;
	}

	rsc_id = crm_element_value(match, XML_ATTR_ID);

	stonith_device_remove(rsc_id, TRUE);
	}
	}

	static void
	handle_topology_change(xmlNode *match, bool remove)
	{
	int rc;
	char *desc = NULL;

	CRM_CHECK(match != NULL, return);
	crm_trace("Updating %s", ID(match));

	if(remove) {
	int index = 0;
	char *key = stonith_level_key(match, -1);

	crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index);
	topology_remove_helper(key, index);
	free(key);
	}

	rc = stonith_level_register(match, &desc);
	do_stonith_notify_level(0, STONITH_OP_LEVEL_ADD, rc, desc);

	free(desc);
	}

	static void
	remove_fencing_topology(xmlXPathObjectPtr xpathObj)
	{
	int max = numXpathResults(xpathObj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if (match && crm_element_value(match, XML_DIFF_MARKER)) {
	/* Deletion */
	int index = 0;
	char *target = stonith_level_key(match, -1);

	crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index);
	if (target == NULL) {
	crm_err("Invalid fencing target in element %s", ID(match));

	} else if (index <= 0) {
	crm_err("Invalid level for %s in element %s", target, ID(match));

	} else {
	topology_remove_helper(target, index);
	}
	/* } else { Deal with modifications during the 'addition' stage */
	}
	}
	}

	static void
	register_fencing_topology(xmlXPathObjectPtr xpathObj)
	{
	int max = numXpathResults(xpathObj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);

	handle_topology_change(match, TRUE);
	}
	}

	/* Fencing
	<diff crm_feature_set="3.0.6">
	<diff-removed>
	<fencing-topology>
	<fencing-level id="f-p1.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="removed:top"/>
	<fencing-level id="f-p1.2" target="pcmk-1" index="2" devices="power" __crm_diff_marker__="removed:top"/>
	<fencing-level devices="disk,network" id="f-p2.1"/>
	</fencing-topology>
	</diff-removed>
	<diff-added>
	<fencing-topology>
	<fencing-level id="f-p.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="added:top"/>
	<fencing-level id="f-p2.1" target="pcmk-2" index="1" devices="disk,something"/>
	<fencing-level id="f-p3.1" target="pcmk-2" index="2" devices="power" __crm_diff_marker__="added:top"/>
	</fencing-topology>
	</diff-added>
	</diff>
	*/

	static void
	fencing_topology_init(void)
	{
	xmlXPathObjectPtr xpathObj = NULL;
	const char *xpath = "//" XML_TAG_FENCING_LEVEL;

	crm_trace("Full topology refresh");
	free_topology_list();
	init_topology_list();

	/* Grab everything */
	xpathObj = xpath_search(local_cib, xpath);
	register_fencing_topology(xpathObj);

	freeXpathObject(xpathObj);
	}

	#define rsc_name(x) x->clone_name?x->clone_name:x->id

	/*!
	* \internal
	* \brief Check whether our uname is in a resource's allowed node list
	*
	* \param[in] rsc Resource to check
	*
	* \return Pointer to node object if found, NULL otherwise
	*/
	static pe_node_t *
	our_node_allowed_for(pe_resource_t *rsc)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;

	if (rsc && stonith_our_uname) {
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node && strcmp(node->details->uname, stonith_our_uname) == 0) {
	break;
	}
	node = NULL;
	}
	}
	return node;
	}

	static void
	watchdog_device_update(xmlNode *cib)
	{
	xmlNode *stonith_enabled_xml = NULL;
	const char *stonith_enabled_s = NULL;
	long timeout_ms = 0;

	stonith_enabled_xml = get_xpath_object("//nvpair[@name='stonith-enabled']",
	cib, LOG_NEVER);
	if (stonith_enabled_xml) {
	stonith_enabled_s = crm_element_value(stonith_enabled_xml, XML_NVPAIR_ATTR_VALUE);
	}

	if (stonith_enabled_s == NULL \|\| crm_is_true(stonith_enabled_s)) {
	xmlNode *stonith_watchdog_xml = NULL;
	const char *value = NULL;

	stonith_watchdog_xml = get_xpath_object("//nvpair[@name='stonith-watchdog-timeout']",
	cib, LOG_NEVER);
	if (stonith_watchdog_xml) {
	value = crm_element_value(stonith_watchdog_xml, XML_NVPAIR_ATTR_VALUE);
	}
	if (value) {
	timeout_ms = crm_get_msec(value);
	}

	if (timeout_ms < 0) {
	timeout_ms = pcmk__auto_watchdog_timeout();
	}
	}

	if (timeout_ms != stonith_watchdog_timeout_ms) {
	crm_notice("New watchdog timeout %lds (was %lds)", timeout_ms/1000, stonith_watchdog_timeout_ms/1000);
	stonith_watchdog_timeout_ms = timeout_ms;

	if (stonith_watchdog_timeout_ms > 0) {
	int rc;
	xmlNode *xml;
	stonith_key_value_t *params = NULL;

	params = stonith_key_value_add(params, PCMK_STONITH_HOST_LIST,
	stonith_our_uname);

	xml = create_device_registration_xml("watchdog", st_namespace_internal,
	STONITH_WATCHDOG_AGENT, params,
	NULL);
	stonith_key_value_freeall(params, 1, 1);
	rc = stonith_device_register(xml, NULL, FALSE);
	free_xml(xml);
	if (rc != pcmk_ok) {
	crm_crit("Cannot register watchdog pseudo fence agent");
	crm_exit(CRM_EX_FATAL);
	}

	} else {
	stonith_device_remove("watchdog", FALSE);
	}
	}
	}

	/*!
	* \internal
	* \brief If a resource or any of its children are STONITH devices, update their
	* definitions given a cluster working set.
	*
	* \param[in] rsc Resource to check
	* \param[in] data_set Cluster working set with device information
	*/
	static void cib_device_update(pe_resource_t rsc, pe_working_set_t data_set)
	{
	pe_node_t *node = NULL;
	const char *value = NULL;
	const char *rclass = NULL;
	pe_node_t *parent = NULL;

	/* If this is a complex resource, check children rather than this resource itself. */
	if(rsc->children) {
	GList *gIter = NULL;
	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	cib_device_update(gIter->data, data_set);
	if(pe_rsc_is_clone(rsc)) {
	crm_trace("Only processing one copy of the clone %s", rsc->id);
	break;
	}
	}
	return;
	}

	/* We only care about STONITH resources. */
	rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	if (!pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	return;
	}

	/* If this STONITH resource is disabled, remove it. */
	if (pe__resource_is_disabled(rsc)) {
	crm_info("Device %s has been disabled", rsc->id);
	return;
	}

	/* Check whether our node is allowed for this resource (and its parent if in a group) */
	node = our_node_allowed_for(rsc);
	if (rsc->parent && (rsc->parent->variant == pe_group)) {
	parent = our_node_allowed_for(rsc->parent);
	}

	if(node == NULL) {
	/* Our node is disallowed, so remove the device */
	GHashTableIter iter;

	crm_info("Device %s has been disabled on %s: unknown", rsc->id, stonith_our_uname);
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	crm_trace("Available: %s = %d", node->details->uname, node->weight);
	}

	return;

	} else if(node->weight < 0 \|\| (parent && parent->weight < 0)) {
	/* Our node (or its group) is disallowed by score, so remove the device */
	char *score = score2char((node->weight < 0) ? node->weight : parent->weight);

	crm_info("Device %s has been disabled on %s: score=%s", rsc->id, stonith_our_uname, score);
	free(score);

	return;

	} else {
	/* Our node is allowed, so update the device information */
	int rc;
	xmlNode *data;
	GHashTable *rsc_params = NULL;
	GHashTableIter gIter;
	stonith_key_value_t *params = NULL;

	const char *name = NULL;
	const char *agent = crm_element_value(rsc->xml, XML_EXPR_ATTR_TYPE);
	const char *rsc_provides = NULL;

	crm_debug("Device %s is allowed on %s: score=%d", rsc->id, stonith_our_uname, node->weight);
	rsc_params = pe_rsc_params(rsc, node, data_set);
	get_meta_attributes(rsc->meta, rsc, node, data_set);

	rsc_provides = g_hash_table_lookup(rsc->meta, PCMK_STONITH_PROVIDES);

	g_hash_table_iter_init(&gIter, rsc_params);
	while (g_hash_table_iter_next(&gIter, (gpointer ) & name, (gpointer ) & value)) {
	if (!name \|\| !value) {
	continue;
	}
	params = stonith_key_value_add(params, name, value);
	crm_trace(" %s=%s", name, value);
	}

	data = create_device_registration_xml(rsc_name(rsc), st_namespace_any,
	agent, params, rsc_provides);
	stonith_key_value_freeall(params, 1, 1);
	rc = stonith_device_register(data, NULL, TRUE);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(data);
	}
	}

	/*!
	* \internal
	* \brief Update all STONITH device definitions based on current CIB
	*/
	static void
	cib_devices_update(void)
	{
	GList *gIter = NULL;
	GHashTableIter iter;
	stonith_device_t *device = NULL;

	crm_info("Updating devices to version %s.%s.%s",
	crm_element_value(local_cib, XML_ATTR_GENERATION_ADMIN),
	crm_element_value(local_cib, XML_ATTR_GENERATION),
	crm_element_value(local_cib, XML_ATTR_NUMUPDATES));

	CRM_ASSERT(fenced_data_set != NULL);
	fenced_data_set->input = local_cib;
	fenced_data_set->now = crm_time_new(NULL);
	fenced_data_set->localhost = stonith_our_uname;
	pe__set_working_set_flags(fenced_data_set, pe_flag_quick_location);

	cluster_status(fenced_data_set);
	pcmk__schedule_actions(fenced_data_set, NULL, NULL);

	g_hash_table_iter_init(&iter, device_list);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&device)) {
	if (device->cib_registered) {
	device->dirty = TRUE;
	}
	}

	for (gIter = fenced_data_set->resources; gIter != NULL; gIter = gIter->next) {
	cib_device_update(gIter->data, fenced_data_set);
	}

	g_hash_table_iter_init(&iter, device_list);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&device)) {
	if (device->dirty) {
	g_hash_table_iter_remove(&iter);
	}
	}

	fenced_data_set->input = NULL; // Wasn't a copy, so don't let API free it
	pe_reset_working_set(fenced_data_set);
	}

	static void
	update_cib_stonith_devices_v2(const char event, xmlNode msg)
	{
	xmlNode *change = NULL;
	char *reason = NULL;
	bool needs_update = FALSE;
	xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	for (change = pcmk__xml_first_child(patchset); change != NULL;
	change = pcmk__xml_next(change)) {
	const char *op = crm_element_value(change, XML_DIFF_OP);
	const char *xpath = crm_element_value(change, XML_DIFF_PATH);
	const char *shortpath = NULL;

	if ((op == NULL) \|\|
	(strcmp(op, "move") == 0) \|\|
	strstr(xpath, "/"XML_CIB_TAG_STATUS)) {
	continue;
	} else if (pcmk__str_eq(op, "delete", pcmk__str_casei) && strstr(xpath, "/"XML_CIB_TAG_RESOURCE)) {
	const char *rsc_id = NULL;
	char *search = NULL;
	char *mutable = NULL;

	if (strstr(xpath, XML_TAG_ATTR_SETS) \|\|
	strstr(xpath, XML_TAG_META_SETS)) {
	needs_update = TRUE;
	reason = strdup("(meta) attribute deleted from resource");
	break;
	}
	mutable = strdup(xpath);
	rsc_id = strstr(mutable, "primitive[@id=\'");
	if (rsc_id != NULL) {
	rsc_id += strlen("primitive[@id=\'");
	search = strchr(rsc_id, '\'');
	}
	if (search != NULL) {
	*search = 0;
	stonith_device_remove(rsc_id, TRUE);
	} else {
	crm_warn("Ignoring malformed CIB update (resource deletion)");
	}
	free(mutable);

	} else if (strstr(xpath, "/"XML_CIB_TAG_RESOURCES) \|\|
	strstr(xpath, "/"XML_CIB_TAG_CONSTRAINTS) \|\|
	strstr(xpath, "/"XML_CIB_TAG_RSCCONFIG)) {
	shortpath = strrchr(xpath, '/'); CRM_ASSERT(shortpath);
	reason = crm_strdup_printf("%s %s", op, shortpath+1);
	needs_update = TRUE;
	break;
	}
	}

	if(needs_update) {
	crm_info("Updating device list from CIB: %s", reason);
	cib_devices_update();
	} else {
	crm_trace("No updates for device list found in CIB");
	}
	free(reason);
	}


	static void
	update_cib_stonith_devices_v1(const char event, xmlNode msg)
	{
	const char *reason = "none";
	gboolean needs_update = FALSE;
	xmlXPathObjectPtr xpath_obj = NULL;

	/* process new constraints */
	xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_CONS_TAG_RSC_LOCATION);
	if (numXpathResults(xpath_obj) > 0) {
	int max = numXpathResults(xpath_obj), lpc = 0;

	/* Safest and simplest to always recompute */
	needs_update = TRUE;
	reason = "new location constraint";

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpath_obj, lpc);

	crm_log_xml_trace(match, "new constraint");
	}
	}
	freeXpathObject(xpath_obj);

	/* process deletions */
	xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_RESOURCE);
	if (numXpathResults(xpath_obj) > 0) {
	remove_cib_device(xpath_obj);
	}
	freeXpathObject(xpath_obj);

	/* process additions */
	xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_RESOURCE);
	if (numXpathResults(xpath_obj) > 0) {
	int max = numXpathResults(xpath_obj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	const char *rsc_id = NULL;
	const char *standard = NULL;
	xmlNode *match = getXpathResult(xpath_obj, lpc);

	rsc_id = crm_element_value(match, XML_ATTR_ID);
	standard = crm_element_value(match, XML_AGENT_ATTR_CLASS);

	if (!pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	continue;
	}

	crm_trace("Fencing resource %s was added or modified", rsc_id);
	reason = "new resource";
	needs_update = TRUE;
	}
	}
	freeXpathObject(xpath_obj);

	if(needs_update) {
	crm_info("Updating device list from CIB: %s", reason);
	cib_devices_update();
	}
	}

	static void
	update_cib_stonith_devices(const char event, xmlNode msg)
	{
	int format = 1;
	xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	CRM_ASSERT(patchset);
	crm_element_value_int(patchset, "format", &format);
	switch(format) {
	case 1:
	update_cib_stonith_devices_v1(event, msg);
	break;
	case 2:
	update_cib_stonith_devices_v2(event, msg);
	break;
	default:
	crm_warn("Unknown patch format: %d", format);
	}
	}

	/* Needs to hold node name + attribute name + attribute value + 75 */
	#define XPATH_MAX 512

	/*!
	* \internal
	* \brief Check whether a node has a specific attribute name/value
	*
	* \param[in] node Name of node to check
	* \param[in] name Name of an attribute to look for
	* \param[in] value The value the named attribute needs to be set to in order to be considered a match
	*
	* \return TRUE if the locally cached CIB has the specified node attribute
	*/
	gboolean
	node_has_attr(const char node, const char name, const char *value)
	{
	char xpath[XPATH_MAX];
	xmlNode *match;
	int n;

	CRM_CHECK(local_cib != NULL, return FALSE);

	/* Search for the node's attributes in the CIB. While the schema allows
	* multiple sets of instance attributes, and allows instance attributes to
	* use id-ref to reference values elsewhere, that is intended for resources,
	* so we ignore that here.
	*/
	n = snprintf(xpath, XPATH_MAX, "//" XML_CIB_TAG_NODES
	"/" XML_CIB_TAG_NODE "[@uname='%s']/" XML_TAG_ATTR_SETS
	"/" XML_CIB_TAG_NVPAIR "[@name='%s' and @value='%s']",
	node, name, value);
	match = get_xpath_object(xpath, local_cib, LOG_NEVER);

	CRM_CHECK(n < XPATH_MAX, return FALSE);
	return (match != NULL);
	}

	static void
	update_fencing_topology(const char event, xmlNode msg)
	{
	int format = 1;
	const char *xpath;
	xmlXPathObjectPtr xpathObj = NULL;
	xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	CRM_ASSERT(patchset);
	crm_element_value_int(patchset, "format", &format);

	if(format == 1) {
	/* Process deletions (only) */
	xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_TAG_FENCING_LEVEL;
	xpathObj = xpath_search(msg, xpath);

	remove_fencing_topology(xpathObj);
	freeXpathObject(xpathObj);

	/* Process additions and changes */
	xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_TAG_FENCING_LEVEL;
	xpathObj = xpath_search(msg, xpath);

	register_fencing_topology(xpathObj);
	freeXpathObject(xpathObj);

	} else if(format == 2) {
	xmlNode *change = NULL;
	int add[] = { 0, 0, 0 };
	int del[] = { 0, 0, 0 };

	xml_patch_versions(patchset, add, del);

	for (change = pcmk__xml_first_child(patchset); change != NULL;
	change = pcmk__xml_next(change)) {
	const char *op = crm_element_value(change, XML_DIFF_OP);
	const char *xpath = crm_element_value(change, XML_DIFF_PATH);

	if(op == NULL) {
	continue;

	} else if(strstr(xpath, "/" XML_TAG_FENCING_LEVEL) != NULL) {
	/* Change to a specific entry */

	crm_trace("Handling %s operation %d.%d.%d for %s", op, add[0], add[1], add[2], xpath);
	if(strcmp(op, "move") == 0) {
	continue;

	} else if(strcmp(op, "create") == 0) {
	handle_topology_change(change->children, FALSE);

	} else if(strcmp(op, "modify") == 0) {
	xmlNode *match = first_named_child(change, XML_DIFF_RESULT);

	if(match) {
	handle_topology_change(match->children, TRUE);
	}

	} else if(strcmp(op, "delete") == 0) {
	/* Nuclear option, all we have is the path and an id... not enough to remove a specific entry */
	crm_info("Re-initializing fencing topology after %s operation %d.%d.%d for %s",
	op, add[0], add[1], add[2], xpath);
	fencing_topology_init();
	return;
	}

	} else if (strstr(xpath, "/" XML_TAG_FENCING_TOPOLOGY) != NULL) {
	/* Change to the topology in general */
	crm_info("Re-initializing fencing topology after top-level %s operation %d.%d.%d for %s",
	op, add[0], add[1], add[2], xpath);
	fencing_topology_init();
	return;

	} else if (strstr(xpath, "/" XML_CIB_TAG_CONFIGURATION)) {
	/* Changes to the whole config section, possibly including the topology as a whild */
	if(first_named_child(change, XML_TAG_FENCING_TOPOLOGY) == NULL) {
	crm_trace("Nothing for us in %s operation %d.%d.%d for %s.",
	op, add[0], add[1], add[2], xpath);

	} else if(strcmp(op, "delete") == 0 \|\| strcmp(op, "create") == 0) {
	crm_info("Re-initializing fencing topology after top-level %s operation %d.%d.%d for %s.",
	op, add[0], add[1], add[2], xpath);
	fencing_topology_init();
	return;
	}

	} else {
	crm_trace("Nothing for us in %s operation %d.%d.%d for %s",
	op, add[0], add[1], add[2], xpath);
	}
	}

	} else {
	crm_warn("Unknown patch format: %d", format);
	}
	}
	static bool have_cib_devices = FALSE;

	static void
	update_cib_cache_cb(const char event, xmlNode msg)
	{
	int rc = pcmk_ok;
	xmlNode *stonith_enabled_xml = NULL;
	const char *stonith_enabled_s = NULL;
	static gboolean stonith_enabled_saved = TRUE;

	if(!have_cib_devices) {
	crm_trace("Skipping updates until we get a full dump");
	return;

	} else if(msg == NULL) {
	crm_trace("Missing %s update", event);
	return;
	}

	/* Maintain a local copy of the CIB so that we have full access
	* to device definitions, location constraints, and node attributes
	*/
	if (local_cib != NULL) {
	int rc = pcmk_ok;
	xmlNode *patchset = NULL;

	crm_element_value_int(msg, F_CIB_RC, &rc);
	if (rc != pcmk_ok) {
	return;
	}

	patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
	xml_log_patchset(LOG_TRACE, "Config update", patchset);
	rc = xml_apply_patchset(local_cib, patchset, TRUE);
	switch (rc) {
	case pcmk_ok:
	case -pcmk_err_old_data:
	break;
	case -pcmk_err_diff_resync:
	case -pcmk_err_diff_failed:
	crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
	free_xml(local_cib);
	local_cib = NULL;
	break;
	default:
	crm_warn("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
	free_xml(local_cib);
	local_cib = NULL;
	}
	}

	if (local_cib == NULL) {
	crm_trace("Re-requesting full CIB");
	rc = cib_api->cmds->query(cib_api, NULL, &local_cib, cib_scope_local \| cib_sync_call);
	if(rc != pcmk_ok) {
	crm_err("Couldn't retrieve the CIB: %s (%d)", pcmk_strerror(rc), rc);
	return;
	}
	CRM_ASSERT(local_cib != NULL);
	stonith_enabled_saved = FALSE; /* Trigger a full refresh below */
	}

	pcmk__refresh_node_caches_from_cib(local_cib);

	stonith_enabled_xml = get_xpath_object("//nvpair[@name='stonith-enabled']",
	local_cib, LOG_NEVER);
	if (stonith_enabled_xml) {
	stonith_enabled_s = crm_element_value(stonith_enabled_xml, XML_NVPAIR_ATTR_VALUE);
	}

	watchdog_device_update(local_cib);

	if (stonith_enabled_s && crm_is_true(stonith_enabled_s) == FALSE) {
	crm_trace("Ignoring CIB updates while fencing is disabled");
	stonith_enabled_saved = FALSE;
	return;

	} else if (stonith_enabled_saved == FALSE) {
	crm_info("Updating fencing device and topology lists "
	"now that fencing is enabled");
	stonith_enabled_saved = TRUE;
	fencing_topology_init();
	cib_devices_update();

	} else {
	update_fencing_topology(event, msg);
	update_cib_stonith_devices(event, msg);
	}
	}

	static void
	init_cib_cache_cb(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	crm_info("Updating device list from CIB");
	have_cib_devices = TRUE;
	local_cib = copy_xml(output);

	pcmk__refresh_node_caches_from_cib(local_cib);

	fencing_topology_init();
	watchdog_device_update(local_cib);
	cib_devices_update();
	}

	static void
	stonith_shutdown(int nsig)
	{
	crm_info("Terminating with %d clients", pcmk__ipc_client_count());
	stonith_shutdown_flag = TRUE;
	if (mainloop != NULL && g_main_loop_is_running(mainloop)) {
	g_main_loop_quit(mainloop);
	} else {
	stonith_cleanup();
	crm_exit(CRM_EX_OK);
	}
	}

	static void
	cib_connection_destroy(gpointer user_data)
	{
	if (stonith_shutdown_flag) {
	crm_info("Connection to the CIB manager closed");
	return;
	} else {
	crm_crit("Lost connection to the CIB manager, shutting down");
	}
	if (cib_api) {
	cib_api->cmds->signoff(cib_api);
	}
	stonith_shutdown(0);
	}

	static void
	stonith_cleanup(void)
	{
	if (cib_api) {
	cib_api->cmds->del_notify_callback(cib_api, T_CIB_DIFF_NOTIFY, update_cib_cache_cb);
	cib_api->cmds->signoff(cib_api);
	}

	if (ipcs) {
	qb_ipcs_destroy(ipcs);
	}

	crm_peer_destroy();
	pcmk__client_cleanup();
	free_stonith_remote_op_list();
	free_topology_list();
	free_device_list();
	free_metadata_cache();

	free(stonith_our_uname);
	stonith_our_uname = NULL;

	free_xml(local_cib);
	local_cib = NULL;
	}

	static pcmk__cli_option_t long_options[] = {
	// long option, argument type, storage, short option, description, flags
	{
	"stand-alone", no_argument, 0, 's',
	NULL, pcmk__option_default
	},
	{
	"stand-alone-w-cpg", no_argument, 0, 'c',
	NULL, pcmk__option_default
	},
	{
	"logfile", required_argument, 0, 'l',
	NULL, pcmk__option_default
	},
	{
	"verbose", no_argument, 0, 'V',
	NULL, pcmk__option_default
	},
	{
	"version", no_argument, 0, '$',
	NULL, pcmk__option_default
	},
	{
	"help", no_argument, 0, '?',
	NULL, pcmk__option_default
	},
	{ 0, 0, 0, 0 }
	};

	static void
	setup_cib(void)
	{
	int rc, retries = 0;

	cib_api = cib_new();
	if (cib_api == NULL) {
	crm_err("No connection to the CIB manager");
	return;
	}

	do {
	sleep(retries);
	rc = cib_api->cmds->signon(cib_api, CRM_SYSTEM_STONITHD, cib_command);
	} while (rc == -ENOTCONN && ++retries < 5);

	if (rc != pcmk_ok) {
	crm_err("Could not connect to the CIB manager: %s (%d)", pcmk_strerror(rc), rc);

	} else if (pcmk_ok !=
	cib_api->cmds->add_notify_callback(cib_api, T_CIB_DIFF_NOTIFY, update_cib_cache_cb)) {
	crm_err("Could not set CIB notification callback");

	} else {
	rc = cib_api->cmds->query(cib_api, NULL, NULL, cib_scope_local);
	cib_api->cmds->register_callback(cib_api, rc, 120, FALSE, NULL, "init_cib_cache_cb",
	init_cib_cache_cb);
	cib_api->cmds->set_connection_dnotify(cib_api, cib_connection_destroy);
	crm_info("Watching for fencing topology changes");
	}
	}

	struct qb_ipcs_service_handlers ipc_callbacks = {
	.connection_accept = st_ipc_accept,
	.connection_created = NULL,
	.msg_process = st_ipc_dispatch,
	.connection_closed = st_ipc_closed,
	.connection_destroyed = st_ipc_destroy
	};

	/*!
	* \internal
	* \brief Callback for peer status changes
	*
	* \param[in] type What changed
	* \param[in] node What peer had the change
	* \param[in] data Previous value of what changed
	*/
	static void
	st_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
	{
	if ((type != crm_status_processes)
	&& !pcmk_is_set(node->flags, crm_remote_node)) {
	/*
	* This is a hack until we can send to a nodeid and/or we fix node name lookups
	* These messages are ignored in stonith_peer_callback()
	*/
	xmlNode *query = create_xml_node(NULL, "stonith_command");

	crm_xml_add(query, F_XML_TAGNAME, "stonith_command");
	crm_xml_add(query, F_TYPE, T_STONITH_NG);
	crm_xml_add(query, F_STONITH_OPERATION, "poke");

	crm_debug("Broadcasting our uname because of node %u", node->id);
	send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE);

	free_xml(query);
	}
	}

	int
	main(int argc, char **argv)
	{
	int flag;
	int lpc = 0;
	int argerr = 0;
	int option_index = 0;
	crm_cluster_t *cluster = NULL;
	const char *actions[] = { "reboot", "off", "on", "list", "monitor", "status" };
	crm_ipc_t *old_instance = NULL;
	int rc = pcmk_rc_ok;

	crm_log_preinit(NULL, argc, argv);
	pcmk__set_cli_options(NULL, "[options]", long_options,
	"daemon for executing fencing devices in a "
	"Pacemaker cluster");

	while (1) {
	flag = pcmk__next_cli_option(argc, argv, &option_index, NULL);
	if (flag == -1) {
	break;
	}

	switch (flag) {
	case 'V':
	crm_bump_log_level(argc, argv);
	break;
	case 'l':
	- pcmk__add_logfile(optarg);
	+ {
	+ int rc = pcmk__add_logfile(optarg);
	+
	+ if (rc != pcmk_rc_ok) {
	+ /* Logging has not yet been initialized, so stderr is
	+ * the only way to get information out
	+ */
	+ fprintf(stderr, "Logging to %s is disabled: %s\n",
	+ optarg, pcmk_rc_str(rc));
	+ }
	+ }
	break;
	case 's':
	stand_alone = TRUE;
	break;
	case 'c':
	stand_alone = FALSE;
	no_cib_connect = TRUE;
	break;
	case '$':
	case '?':
	pcmk__cli_help(flag, CRM_EX_OK);
	break;
	default:
	++argerr;
	break;
	}
	}

	if (argc - optind == 1 && pcmk__str_eq("metadata", argv[optind], pcmk__str_casei)) {
	printf("<?xml version=\"1.0\"?><!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n");
	printf("<resource-agent name=\"pacemaker-fenced\">\n");
	printf(" <version>1.0</version>\n");
	printf(" <longdesc lang=\"en\">Instance attributes available for all \"stonith\"-class resources"
	" and used by Pacemaker's fence daemon, formerly known as stonithd</longdesc>\n");
	printf(" <shortdesc lang=\"en\">Instance attributes available for all \"stonith\"-class resources</shortdesc>\n");
	printf(" <parameters>\n");

	#if 0
	// priority is not implemented yet
	printf(" <parameter name=\"priority\" unique=\"0\">\n");
	printf(" <shortdesc lang=\"en\">Devices that are not in a topology "
	"are tried in order of highest to lowest integer priority</shortdesc>\n");
	printf(" <content type=\"integer\" default=\"0\"/>\n");
	printf(" </parameter>\n");
	#endif

	printf(" <parameter name=\"%s\" unique=\"0\">\n",
	PCMK_STONITH_HOST_ARGUMENT);
	printf
	(" <shortdesc lang=\"en\">Advanced use only: An alternate parameter to supply instead of 'port'</shortdesc>\n");
	printf
	(" <longdesc lang=\"en\">Some devices do not support the standard 'port' parameter or may provide additional ones.\n"
	"Use this to specify an alternate, device-specific, parameter that should indicate the machine to be fenced.\n"
	"A value of 'none' can be used to tell the cluster not to supply any additional parameters.\n"
	" </longdesc>\n");
	printf(" <content type=\"string\" default=\"port\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n",
	PCMK_STONITH_HOST_MAP);
	printf
	(" <shortdesc lang=\"en\">A mapping of host names to ports numbers for devices that do not support host names.</shortdesc>\n");
	printf
	(" <longdesc lang=\"en\">Eg. node1:1;node2:2,3 would tell the cluster to use port 1 for node1 and ports 2 and 3 for node2</longdesc>\n");
	printf(" <content type=\"string\" default=\"\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n",
	PCMK_STONITH_HOST_LIST);
	printf(" <shortdesc lang=\"en\">A list of machines controlled by "
	"this device (Optional unless %s=static-list).</shortdesc>\n",
	PCMK_STONITH_HOST_CHECK);
	printf(" <content type=\"string\" default=\"\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n",
	PCMK_STONITH_HOST_CHECK);
	printf
	(" <shortdesc lang=\"en\">How to determine which machines are controlled by the device.</shortdesc>\n");
	printf(" <longdesc lang=\"en\">Allowed values: dynamic-list "
	"(query the device via the 'list' command), static-list "
	"(check the " PCMK_STONITH_HOST_LIST " attribute), status "
	"(query the device via the 'status' command), none (assume "
	"every device can fence every machine)</longdesc>\n");
	printf(" <content type=\"string\" default=\"dynamic-list\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n",
	PCMK_STONITH_DELAY_MAX);
	printf(" <shortdesc lang=\"en\">Enable a delay of no more than the "
	"time specified before executing fencing actions. Pacemaker "
	"derives the overall delay by taking the value of "
	PCMK_STONITH_DELAY_BASE " and adding a random delay value such "
	"that the sum is kept below this maximum.</shortdesc>\n");
	printf(" <longdesc lang=\"en\">This prevents double fencing when "
	"using slow devices such as sbd.\nUse this to enable a random "
	"delay for fencing actions.\nThe overall delay is derived from "
	"this random delay value adding a static delay so that the sum "
	"is kept below the maximum delay.</longdesc>\n");
	printf(" <content type=\"time\" default=\"0s\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n",
	PCMK_STONITH_DELAY_BASE);
	printf(" <shortdesc lang=\"en\">Enable a base delay for "
	"fencing actions and specify base delay value.</shortdesc>\n");
	printf(" <longdesc lang=\"en\">This prevents double fencing when "
	"different delays are configured on the nodes.\nUse this to "
	"enable a static delay for fencing actions.\nThe overall delay "
	"is derived from a random delay value adding this static delay "
	"so that the sum is kept below the maximum delay.</longdesc>\n");
	printf(" <content type=\"time\" default=\"0s\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n",
	PCMK_STONITH_ACTION_LIMIT);
	printf
	(" <shortdesc lang=\"en\">The maximum number of actions can be performed in parallel on this device</shortdesc>\n");
	printf
	(" <longdesc lang=\"en\">Cluster property concurrent-fencing=true needs to be configured first.\n"
	"Then use this to specify the maximum number of actions can be performed in parallel on this device. -1 is unlimited.</longdesc>\n");
	printf(" <content type=\"integer\" default=\"1\"/>\n");
	printf(" </parameter>\n");


	for (lpc = 0; lpc < PCMK__NELEM(actions); lpc++) {
	printf(" <parameter name=\"pcmk_%s_action\" unique=\"0\">\n", actions[lpc]);
	printf
	(" <shortdesc lang=\"en\">Advanced use only: An alternate command to run instead of '%s'</shortdesc>\n",
	actions[lpc]);
	printf
	(" <longdesc lang=\"en\">Some devices do not support the standard commands or may provide additional ones.\n"
	"Use this to specify an alternate, device-specific, command that implements the '%s' action.</longdesc>\n",
	actions[lpc]);
	printf(" <content type=\"string\" default=\"%s\"/>\n", actions[lpc]);
	printf(" </parameter>\n");

	printf(" <parameter name=\"pcmk_%s_timeout\" unique=\"0\">\n", actions[lpc]);
	printf
	(" <shortdesc lang=\"en\">Advanced use only: Specify an alternate timeout to use for %s actions instead of stonith-timeout</shortdesc>\n",
	actions[lpc]);
	printf
	(" <longdesc lang=\"en\">Some devices need much more/less time to complete than normal.\n"
	"Use this to specify an alternate, device-specific, timeout for '%s' actions.</longdesc>\n",
	actions[lpc]);
	printf(" <content type=\"time\" default=\"60s\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"pcmk_%s_retries\" unique=\"0\">\n", actions[lpc]);
	printf
	(" <shortdesc lang=\"en\">Advanced use only: The maximum number of times to retry the '%s' command within the timeout period</shortdesc>\n",
	actions[lpc]);
	printf(" <longdesc lang=\"en\">Some devices do not support multiple connections."
	" Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining."
	" Use this option to alter the number of times Pacemaker retries '%s' actions before giving up."
	"</longdesc>\n", actions[lpc]);
	printf(" <content type=\"integer\" default=\"2\"/>\n");
	printf(" </parameter>\n");
	}

	printf(" </parameters>\n");
	printf("</resource-agent>\n");
	return CRM_EX_OK;
	}

	if (optind != argc) {
	++argerr;
	}

	if (argerr) {
	pcmk__cli_help('?', CRM_EX_USAGE);
	}

	crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);

	crm_notice("Starting Pacemaker fencer");

	old_instance = crm_ipc_new("stonith-ng", 0);
	if (crm_ipc_connect(old_instance)) {
	/* IPC end-point already up */
	crm_ipc_close(old_instance);
	crm_ipc_destroy(old_instance);
	crm_err("pacemaker-fenced is already active, aborting startup");
	crm_exit(CRM_EX_OK);
	} else {
	/* not up or not authentic, we'll proceed either way */
	crm_ipc_destroy(old_instance);
	old_instance = NULL;
	}

	mainloop_add_signal(SIGTERM, stonith_shutdown);

	crm_peer_init();

	fenced_data_set = pe_new_working_set();
	CRM_ASSERT(fenced_data_set != NULL);
	pe__set_working_set_flags(fenced_data_set,
	pe_flag_no_counts\|pe_flag_no_compat);
	pe__set_working_set_flags(fenced_data_set, pe_flag_show_utilization);

	cluster = calloc(1, sizeof(crm_cluster_t));
	CRM_ASSERT(cluster != NULL);

	if (stand_alone == FALSE) {

	if (is_corosync_cluster()) {
	#if SUPPORT_COROSYNC
	cluster->destroy = stonith_peer_cs_destroy;
	cluster->cpg.cpg_deliver_fn = stonith_peer_ais_callback;
	cluster->cpg.cpg_confchg_fn = pcmk_cpg_membership;
	#endif
	}

	crm_set_status_callback(&st_peer_update_callback);

	if (crm_cluster_connect(cluster) == FALSE) {
	crm_crit("Cannot sign in to the cluster... terminating");
	crm_exit(CRM_EX_FATAL);
	}
	stonith_our_uname = strdup(cluster->uname);

	if (no_cib_connect == FALSE) {
	setup_cib();
	}

	} else {
	stonith_our_uname = strdup("localhost");
	}

	init_device_list();
	init_topology_list();

	pcmk__serve_fenced_ipc(&ipcs, &ipc_callbacks);

	pcmk__register_formats(NULL, formats);
	rc = pcmk__output_new(&out, "log", NULL, argv);
	if ((rc != pcmk_rc_ok) \|\| (out == NULL)) {
	crm_err("Can't log resource details due to internal error: %s\n",
	pcmk_rc_str(rc));
	crm_exit(CRM_EX_FATAL);
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);

	pcmk__output_set_log_level(out, LOG_TRACE);
	fenced_data_set->priv = out;

	/* Create the mainloop and run it... */
	mainloop = g_main_loop_new(NULL, FALSE);
	crm_notice("Pacemaker fencer successfully started and accepting connections");
	g_main_loop_run(mainloop);

	stonith_cleanup();
	free(cluster->uuid);
	free(cluster->uname);
	free(cluster);
	pe_free_working_set(fenced_data_set);

	pcmk__unregister_formats();
	out->finish(out, CRM_EX_OK, true, NULL);
	pcmk__output_free(out);

	crm_exit(CRM_EX_OK);
	}

File Metadata

Mime Type: text/x-diff
Expires: Sat, Nov 23, 10:01 AM (1 d, 13 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1018525
Default Alt Text: (70 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions