No OneTemporary
Actions

Size

212 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/TODO.markdown b/TODO.markdown
	index 6c234c9185..4ea7f353cf 100644
	--- a/TODO.markdown
	+++ b/TODO.markdown
	@@ -1,59 +1,58 @@
	# Semi-random collection of tasks we'd like to get done

	## Targeted for 1.2
	-- Get fencing/test.c working again (with and without stonithd -s)
	- Avoid the use of xmlNode in fencing register_callback() call types
	-- Replace (delegated)_variant_op in CIB API with a private function

	- Need a way to indicate when unfencing operations need to be initiated from the host to be unfenced
	+
	- Test startup fencing with node lists
	- Figure out how to distinguish between "down" and "down
	but we've never seen them before so they need to be shot"
	- Attempt to obtain a node list from corosync if one exists

	## Targeted for 1.2.x

	- Check for uppercase letters in node names, warn if found
	- Imply startup-failure-is-fatal from on-fail="restart"
	- Show an english version of the config with crm_resource --rules
	- Convert cts/CIB.py into a supported Python API for the CIB
	- Use crm_log_tag() in the PE to allow per-resource trace logging
	- Reduce the amount of stonith-ng logging
	- Reduce the amount of attrd logging
	- Use dlopen for snmp in crm_mon

	## Targeted for 1.4

	- Support A colocated with (B \|\| C \|\| D)
	- Implement a truely atomic version of attrd
	- Support rolling average values in attrd
	- Support heartbeat with the mcp
	- Freeze/Thaw
	- Create Pacemaker plugin for snmpd - http://www.net-snmp.org/
	- Investigate using a DB as the back-end for the CIB
	- Decide whether to fully support or drop failover domains

	# Testing
	- Write a regression test for Stonith-NG
	- Convert BandwidthTest CTS test into a Scenario wrapper
	- find_operations() is not covered by PE regression tests
	- Some node states in determine_online_status_fencing() are untested by PE regression tests
	- no_quorum_policy==suicide is not covered by PE regression tests
	- parse_xml_duration() is not covered by PE regression tests
	- phase_of_the_moon() is not covered by PE regression tests
	- test_role_expression() is not covered by PE regression tests
	- native_parameter() is not covered by PE regression tests
	- clone_resource_state() is not covered by PE regression tests
	- clone_active() is not covered by PE regression tests
	- convert_non_atomic_task() in native.c is not covered by PE regression tests
	- clone_rsc_colocation_lh() is not covered by PE regression tests
	- group_rsc_colocation_lh() is not covered by PE regression tests
	- Test on-fail=standby

	# Documentation
	- Clusters from Scratch: Mail
	- Clusters from Scratch: MySQL
	- Document reload in Pacemaker Explained
	- Document advanced fencing logic in Pacemaker Explained
	- Use ann:defaultValue="..." instead of <optional> in the schema more often
	- Allow Clusters from Scratch to be built in two flavors - pcs and crm shell
	diff --git a/crmd/crmd_utils.h b/crmd/crmd_utils.h
	index 1293982a30..3320e721d5 100644
	--- a/crmd/crmd_utils.h
	+++ b/crmd/crmd_utils.h
	@@ -1,94 +1,94 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/
	#ifndef CRMD_UTILS__H
	# define CRMD_UTILS__H

	# include <crm/crm.h>
	# include <crm/common/xml.h>
	# include <crm/cib/internal.h> /* For CIB_OP_MODIFY */

	# define CLIENT_EXIT_WAIT 30
	# define FAKE_TE_ID "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"

	extern void process_client_disconnect(crmd_client_t * curr_client);

	# define fsa_cib_update(section, data, options, call_id, user_name) \
	if(fsa_cib_conn != NULL) { \
	- call_id = fsa_cib_conn->cmds->delegated_variant_op( \
	+ call_id = cib_internal_op( \
	fsa_cib_conn, CIB_OP_MODIFY, NULL, section, data, \
	NULL, options, user_name); \
	\
	} else { \
	crm_err("No CIB connection available"); \
	}

	# define fsa_cib_anon_update(section, data, options) \
	if(fsa_cib_conn != NULL) { \
	fsa_cib_conn->cmds->modify( \
	fsa_cib_conn, section, data, options); \
	\
	} else { \
	crm_err("No CIB connection available"); \
	}

	extern gboolean fsa_has_quorum;
	extern int last_peer_update;

	extern gboolean crm_timer_stop(fsa_timer_t * timer);
	extern gboolean crm_timer_start(fsa_timer_t * timer);
	extern gboolean crm_timer_popped(gpointer data);
	extern gboolean is_timer_started(fsa_timer_t *timer);

	extern xmlNode create_node_state(const char uname, const char *in_cluster,
	const char is_peer, const char join_state,
	const char exp_state, gboolean clear_shutdown, const char src);

	extern gboolean stop_subsystem(struct crm_subsystem_s *centry, gboolean force_quit);
	extern gboolean start_subsystem(struct crm_subsystem_s *centry);

	extern void fsa_dump_actions(long long action, const char *text);
	extern void fsa_dump_inputs(int log_level, const char *text, long long input_register);

	extern gboolean update_dc(xmlNode * msg);
	extern void erase_node_from_join(const char *node);
	extern void populate_cib_nodes(gboolean with_client_status);
	extern void crm_update_quorum(gboolean quorum, gboolean force_update);
	extern void erase_status_tag(const char uname, const char tag, int options);
	extern void update_attrd(const char host, const char name, const char *value,
	const char *user_name);

	extern const char get_timer_desc(fsa_timer_t timer);
	gboolean too_many_st_failures(void);

	# define start_transition(state) do { \
	switch(state) { \
	case S_TRANSITION_ENGINE: \
	register_fsa_action(A_TE_CANCEL); \
	break; \
	case S_POLICY_ENGINE: \
	case S_IDLE: \
	register_fsa_input(C_FSA_INTERNAL, I_PE_CALC, NULL); \
	break; \
	default: \
	crm_debug("NOT starting a new transition in state %s", \
	fsa_state2string(fsa_state)); \
	break; \
	} \
	} while(0)

	#endif
	diff --git a/crmd/lrm.c b/crmd/lrm.c
	index fec90996e7..78523f28d4 100644
	--- a/crmd/lrm.c
	+++ b/crmd/lrm.c
	@@ -1,2002 +1,2002 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/wait.h>

	#include <crm/crm.h>
	#include <crm/services.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crmd.h>
	#include <crmd_fsa.h>
	#include <crmd_messages.h>
	#include <crmd_callbacks.h>
	#include <crmd_lrm.h>

	#define START_DELAY_THRESHOLD 5 * 60 * 1000

	typedef struct resource_history_s {
	char *id;
	lrmd_rsc_info_t rsc;
	lrmd_event_data_t *last;
	lrmd_event_data_t *failed;
	GList *recurring_op_list;

	/* Resources must be stopped using the same
	* parameters they were started with. This hashtable
	* holds the parameters that should be used for the next stop
	* cmd on this resource. */
	GHashTable *stop_params;
	} rsc_history_t;

	struct recurring_op_s {
	char *rsc_id;
	char *op_type;
	char *op_key;
	int call_id;
	int interval;
	int last_rc;
	int last_op_status;
	gboolean remove;
	gboolean cancelled;
	};

	struct pending_deletion_op_s {
	char *rsc;
	ha_msg_input_t *input;
	};

	struct delete_event_s {
	int rc;
	const char *rsc;
	};

	GHashTable *resource_history = NULL;
	GHashTable *pending_ops = NULL;
	GHashTable *deletion_ops = NULL;

	int num_lrm_register_fails = 0;
	int max_lrm_register_fails = 30;

	gboolean process_lrm_event(lrmd_event_data_t * op);
	gboolean is_rsc_active(const char *rsc_id);
	gboolean build_active_RAs(xmlNode * rsc_list);
	static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
	static int delete_rsc_status(const char rsc_id, int call_options, const char user_name);

	static lrmd_event_data_t construct_op(xmlNode rsc_op, const char rsc_id, const char operation);
	void do_lrm_rsc_op(lrmd_rsc_info_t * rsc, const char operation, xmlNode msg, xmlNode * request);

	void send_direct_ack(const char to_host, const char to_sys,
	lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id);

	static void
	lrm_connection_destroy(void)
	{
	if (is_set(fsa_input_register, R_LRM_CONNECTED)) {
	crm_crit("LRM Connection failed");
	register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
	clear_bit(fsa_input_register, R_LRM_CONNECTED);

	} else {
	crm_info("LRM Connection disconnected");
	}

	}

	static void
	free_deletion_op(gpointer value)
	{
	struct pending_deletion_op_s *op = value;

	free(op->rsc);
	delete_ha_msg_input(op->input);
	free(op);
	}

	static void
	free_recurring_op(gpointer value)
	{
	struct recurring_op_s op = (struct recurring_op_s )value;

	free(op->rsc_id);
	free(op->op_type);
	free(op->op_key);
	free(op);
	}

	static char *
	make_stop_id(const char *rsc, int call_id)
	{
	char *op_id = NULL;

	op_id = calloc(1, strlen(rsc) + 34);
	if (op_id != NULL) {
	snprintf(op_id, strlen(rsc) + 34, "%s:%d", rsc, call_id);
	}
	return op_id;
	}

	static void
	copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
	{
	if (strstr(key, CRM_META "_") == NULL) {
	g_hash_table_replace(user_data, strdup((const char )key), strdup((const char )value));
	}
	}

	static void
	copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
	{
	if (strstr(key, CRM_META "_") != NULL) {
	g_hash_table_replace(user_data, strdup((const char )key), strdup((const char )value));
	}
	}

	static void
	history_cache_destroy(gpointer data)
	{
	rsc_history_t *entry = data;

	if (entry->stop_params) {
	g_hash_table_destroy(entry->stop_params);
	}

	free(entry->rsc.type);
	free(entry->rsc.class);
	free(entry->rsc.provider);

	lrmd_free_event(entry->failed);
	lrmd_free_event(entry->last);
	free(entry->id);
	free(entry);
	}

	static void
	update_history_cache(lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
	{
	int target_rc = 0;
	rsc_history_t *entry = NULL;

	if (op->rsc_deleted) {
	crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
	delete_rsc_status(op->rsc_id, cib_quorum_override, NULL);
	return;
	}

	if (safe_str_eq(op->op_type, RSC_NOTIFY)) {
	return;
	}

	crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);

	entry = g_hash_table_lookup(resource_history, op->rsc_id);
	if (entry == NULL && rsc) {
	entry = calloc(1, sizeof(rsc_history_t));
	entry->id = strdup(op->rsc_id);
	g_hash_table_insert(resource_history, entry->id, entry);

	entry->rsc.id = entry->id;
	entry->rsc.type = strdup(rsc->type);
	entry->rsc.class = strdup(rsc->class);
	if (rsc->provider) {
	entry->rsc.provider = strdup(rsc->provider);
	} else {
	entry->rsc.provider = NULL;
	}

	} else if (entry == NULL) {
	crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
	return;
	}

	target_rc = rsc_op_expected_rc(op);
	if (op->op_status == PCMK_LRM_OP_CANCELLED) {
	if (op->interval > 0) {
	GList gIter, gIterNext;

	crm_trace("Removing cancelled recurring op: %s_%s_%d", op->rsc_id, op->op_type, op->interval);

	for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIterNext) {
	lrmd_event_data_t *existing = gIter->data;
	gIterNext = gIter->next;

	if (safe_str_eq(op->rsc_id, existing->rsc_id)
	&& safe_str_eq(op->op_type, existing->op_type)
	&& op->interval == existing->interval) {
	lrmd_free_event(existing);
	entry->recurring_op_list = g_list_delete_link(entry->recurring_op_list, gIter);
	}
	}
	return;

	} else {
	crm_trace("Skipping %s_%s_%d rc=%d, status=%d", op->rsc_id, op->op_type, op->interval,
	op->rc, op->op_status);
	}

	} else if (did_rsc_op_fail(op, target_rc)) {
	/* We must store failed monitors here
	* - otherwise the block below will cause them to be forgetten them when a stop happens
	*/
	if (entry->failed) {
	lrmd_free_event(entry->failed);
	}
	entry->failed = lrmd_copy_event(op);

	} else if (op->interval == 0) {
	if (entry->last) {
	lrmd_free_event(entry->last);
	}
	entry->last = lrmd_copy_event(op);

	if (op->params &&
	(safe_str_eq(CRMD_ACTION_START, op->op_type) \|\|
	safe_str_eq(CRMD_ACTION_STATUS, op->op_type))) {

	if (entry->stop_params) {
	g_hash_table_destroy(entry->stop_params);
	}
	entry->stop_params = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
	}
	}

	if (op->interval > 0) {
	crm_trace("Adding recurring op: %s_%s_%d", op->rsc_id, op->op_type, op->interval);
	entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));

	} else if (entry->recurring_op_list && safe_str_eq(op->op_type, RSC_STATUS) == FALSE) {
	GList *gIter = entry->recurring_op_list;

	crm_trace("Dropping %d recurring ops because of: %s_%s_%d",
	g_list_length(gIter), op->rsc_id, op->op_type, op->interval);
	for (; gIter != NULL; gIter = gIter->next) {
	lrmd_free_event(gIter->data);
	}
	g_list_free(entry->recurring_op_list);
	entry->recurring_op_list = NULL;
	}
	}

	/* A_LRM_CONNECT */
	void
	do_lrm_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	if (fsa_lrm_conn == NULL) {
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	if (action & A_LRM_DISCONNECT) {
	if (verify_stopped(cur_state, LOG_INFO) == FALSE) {
	if(action == A_LRM_DISCONNECT) {
	crmd_fsa_stall(NULL);
	return;
	}
	}

	if (is_set(fsa_input_register, R_LRM_CONNECTED)) {
	clear_bit(fsa_input_register, R_LRM_CONNECTED);

	fsa_lrm_conn->cmds->disconnect(fsa_lrm_conn);
	crm_info("Disconnected from the LRM");
	}

	g_hash_table_destroy(resource_history);
	resource_history = NULL;
	g_hash_table_destroy(deletion_ops);
	deletion_ops = NULL;
	g_hash_table_destroy(pending_ops);
	pending_ops = NULL;
	}

	if (action & A_LRM_CONNECT) {
	int ret = pcmk_ok;

	deletion_ops = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, free_deletion_op);

	pending_ops = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, free_recurring_op);

	resource_history = g_hash_table_new_full(crm_str_hash, g_str_equal,
	NULL, history_cache_destroy);

	crm_debug("Connecting to the LRM");
	ret = fsa_lrm_conn->cmds->connect(fsa_lrm_conn, CRM_SYSTEM_CRMD, NULL);

	if (ret != pcmk_ok) {
	if (++num_lrm_register_fails < max_lrm_register_fails) {
	crm_warn("Failed to sign on to the LRM %d"
	" (%d max) times", num_lrm_register_fails, max_lrm_register_fails);

	crm_timer_start(wait_timer);
	crmd_fsa_stall(NULL);
	return;
	}
	}

	if (ret == pcmk_ok) {
	crm_trace("LRM: set_lrm_callback...");
	fsa_lrm_conn->cmds->set_callback(fsa_lrm_conn, lrm_op_callback);
	}

	if (ret != pcmk_ok) {
	crm_err("Failed to sign on to the LRM %d" " (max) times", num_lrm_register_fails);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	set_bit(fsa_input_register, R_LRM_CONNECTED);
	crm_debug("LRM connection established");
	}

	if (action & ~(A_LRM_CONNECT \| A_LRM_DISCONNECT)) {
	crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__);
	}
	}

	static void
	ghash_print_pending(gpointer key, gpointer value, gpointer user_data)
	{
	const char *stop_id = key;
	int *log_level = user_data;
	struct recurring_op_s *pending = value;

	do_crm_log(*log_level, "Pending action: %s (%s)", stop_id, pending->op_key);
	}

	static void
	ghash_print_pending_for_rsc(gpointer key, gpointer value, gpointer user_data)
	{
	const char *stop_id = key;
	char *rsc = user_data;
	struct recurring_op_s *pending = value;

	if (safe_str_eq(rsc, pending->rsc_id)) {
	crm_notice("%sction %s (%s) incomplete at shutdown",
	pending->interval == 0 ? "A" : "Recurring a", stop_id, pending->op_key);
	}
	}

	static void
	ghash_count_pending(gpointer key, gpointer value, gpointer user_data)
	{
	int *counter = user_data;
	struct recurring_op_s *pending = value;

	if (pending->interval > 0) {
	/* Ignore recurring actions in the shutdown calculations */
	return;
	}

	(*counter)++;
	}

	gboolean
	verify_stopped(enum crmd_fsa_state cur_state, int log_level)
	{
	int counter = 0;
	gboolean rc = TRUE;

	GHashTableIter gIter;
	rsc_history_t *entry = NULL;

	crm_debug("Checking for active resources before exit");

	if (cur_state == S_TERMINATE) {
	log_level = LOG_ERR;
	}

	if (pending_ops) {
	if (is_set(fsa_input_register, R_LRM_CONNECTED)) {
	/* Only log/complain about non-recurring actions */
	g_hash_table_foreach_remove(pending_ops, stop_recurring_actions, NULL);
	}
	g_hash_table_foreach(pending_ops, ghash_count_pending, &counter);
	}

	if (counter > 0) {
	rc = FALSE;
	do_crm_log(log_level,
	"%d pending LRM operations at shutdown%s",
	counter, cur_state == S_TERMINATE ? "" : "... waiting");

	if (cur_state == S_TERMINATE \|\| !is_set(fsa_input_register, R_SENT_RSC_STOP)) {
	g_hash_table_foreach(pending_ops, ghash_print_pending, &log_level);
	}
	goto bail;
	}

	if (resource_history == NULL) {
	goto bail;
	}

	g_hash_table_iter_init(&gIter, resource_history);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	if (is_rsc_active(entry->id) == FALSE) {
	continue;
	}

	crm_err("Resource %s was active at shutdown."
	" You may ignore this error if it is unmanaged.", entry->id);

	g_hash_table_foreach(pending_ops, ghash_print_pending_for_rsc, entry->id);
	}

	bail:
	set_bit(fsa_input_register, R_SENT_RSC_STOP);

	if (cur_state == S_TERMINATE) {
	rc = TRUE;
	}

	return rc;
	}

	static char *
	get_rsc_metadata(const char type, const char class, const char *provider)
	{
	char *metadata = NULL;

	CRM_CHECK(type != NULL, return NULL);
	CRM_CHECK(class != NULL, return NULL);
	if (provider == NULL) {
	provider = "heartbeat";
	}

	crm_trace("Retreiving metadata for %s::%s:%s", type, class, provider);
	fsa_lrm_conn->cmds->get_metadata(fsa_lrm_conn, class, provider, type, &metadata, 0);

	if (metadata) {
	/* copy the metadata because the LRM likes using
	* g_alloc instead of cl_malloc
	*/
	char *m_copy = strdup(metadata);

	g_free(metadata);
	metadata = m_copy;

	} else {
	crm_warn("No metadata found for %s::%s:%s", type, class, provider);
	}

	return metadata;
	}

	typedef struct reload_data_s {
	char *key;
	char *metadata;
	time_t last_query;
	gboolean can_reload;
	GListPtr restart_list;
	} reload_data_t;

	static void
	g_hash_destroy_reload(gpointer data)
	{
	reload_data_t *reload = data;

	free(reload->key);
	free(reload->metadata);
	g_list_free_full(reload->restart_list, free);
	free(reload);
	}

	GHashTable *reload_hash = NULL;
	static GListPtr
	get_rsc_restart_list(lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
	{
	int len = 0;
	char *key = NULL;
	char *copy = NULL;
	const char *value = NULL;
	const char *provider = NULL;

	xmlNode *param = NULL;
	xmlNode *params = NULL;
	xmlNode *actions = NULL;
	xmlNode *metadata = NULL;

	time_t now = time(NULL);
	reload_data_t *reload = NULL;

	if (reload_hash == NULL) {
	reload_hash = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, g_hash_destroy_reload);
	}

	provider = rsc->provider;
	if (provider == NULL) {
	provider = "heartbeat";
	}

	len = strlen(rsc->type) + strlen(rsc->class) + strlen(provider) + 4;
	key = malloc( len);
	snprintf(key, len, "%s::%s:%s", rsc->type, rsc->class, provider);

	reload = g_hash_table_lookup(reload_hash, key);

	if (reload && ((now - 9) > reload->last_query)
	&& safe_str_eq(op->op_type, RSC_START)) {
	reload = NULL; /* re-query */
	}

	if (reload == NULL) {
	xmlNode *action = NULL;

	reload = calloc(1, sizeof(reload_data_t));
	g_hash_table_replace(reload_hash, key, reload);

	reload->last_query = now;
	reload->key = key;
	key = NULL;
	reload->metadata = get_rsc_metadata(rsc->type, rsc->class, provider);

	metadata = string2xml(reload->metadata);
	if (metadata == NULL) {
	crm_err("Metadata for %s::%s:%s is not valid XML",
	rsc->provider, rsc->class, rsc->type);
	goto cleanup;
	}

	actions = find_xml_node(metadata, "actions", TRUE);

	for (action = __xml_first_child(actions); action != NULL; action = __xml_next(action)) {
	if (crm_str_eq((const char *)action->name, "action", TRUE)) {
	value = crm_element_value(action, "name");
	if (safe_str_eq("reload", value)) {
	reload->can_reload = TRUE;
	break;
	}
	}
	}

	if (reload->can_reload == FALSE) {
	goto cleanup;
	}

	params = find_xml_node(metadata, "parameters", TRUE);
	for (param = __xml_first_child(params); param != NULL; param = __xml_next(param)) {
	if (crm_str_eq((const char *)param->name, "parameter", TRUE)) {
	value = crm_element_value(param, "unique");
	if (crm_is_true(value)) {
	value = crm_element_value(param, "name");
	if (value == NULL) {
	crm_err("%s: NULL param", key);
	continue;
	}
	crm_debug("Attr %s is not reloadable", value);
	copy = strdup(value);
	CRM_CHECK(copy != NULL, continue);
	reload->restart_list = g_list_append(reload->restart_list, copy);
	}
	}
	}
	}

	cleanup:
	free(key);
	free_xml(metadata);
	return reload->restart_list;
	}

	static void
	append_restart_list(lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, xmlNode * update, const char *version)
	{
	int len = 0;
	char *list = NULL;
	char *digest = NULL;
	const char *value = NULL;
	gboolean non_empty = FALSE;
	xmlNode *restart = NULL;
	GListPtr restart_list = NULL;
	GListPtr lpc = NULL;

	if (op->interval > 0) {
	/* monitors are not reloadable */
	return;

	} else if (op->params == NULL) {
	crm_debug("%s has no parameters", ID(update));
	return;

	} else if (rsc == NULL) {
	return;

	} else if (crm_str_eq(CRMD_ACTION_STOP, op->op_type, TRUE)) {
	/* Stopped resources don't need to be reloaded */
	return;

	} else if (compare_version("1.0.8", version) > 0) {
	/* Caller version does not support reloads */
	return;
	}

	restart_list = get_rsc_restart_list(rsc, op);
	if (restart_list == NULL) {
	/* Resource does not support reloads */
	return;
	}

	restart = create_xml_node(NULL, XML_TAG_PARAMS);
	for (lpc = restart_list; lpc != NULL; lpc = lpc->next) {
	const char param = (const char )lpc->data;

	int start = len;

	CRM_CHECK(param != NULL, continue);
	value = g_hash_table_lookup(op->params, param);
	if (value != NULL) {
	non_empty = TRUE;
	crm_xml_add(restart, param, value);
	}
	len += strlen(param) + 2;
	list = realloc(list, len + 1);
	sprintf(list + start, " %s ", param);
	}

	digest = calculate_operation_digest(restart, version);
	crm_xml_add(update, XML_LRM_ATTR_OP_RESTART, list);
	crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);

	#if 0
	crm_debug("%s: %s, %s", rsc->id, digest, list);
	if (non_empty) {
	crm_log_xml_debug(restart, "restart digest source");
	}
	#endif

	free_xml(restart);
	free(digest);
	free(list);
	}

	static gboolean
	build_operation_update(xmlNode * parent, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *src)
	{
	int target_rc = 0;
	xmlNode *xml_op = NULL;
	const char *caller_version = CRM_FEATURE_SET;

	if (op == NULL) {
	return FALSE;

	} else if (AM_I_DC) {

	} else if (fsa_our_dc_version != NULL) {
	caller_version = fsa_our_dc_version;
	} else if (op->params == NULL) {
	caller_version = fsa_our_dc_version;
	} else {
	/* there is a small risk in formerly mixed clusters that
	* it will be sub-optimal.
	* however with our upgrade policy, the update we send
	* should still be completely supported anyway
	*/
	caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION);
	crm_debug("Falling back to operation originator version: %s", caller_version);
	}

	target_rc = rsc_op_expected_rc(op);
	xml_op = create_operation_update(parent, op, caller_version, target_rc, src, LOG_DEBUG);

	if (xml_op) {
	append_restart_list(rsc, op, xml_op, caller_version);
	}
	return TRUE;
	}

	gboolean
	is_rsc_active(const char *rsc_id)
	{
	rsc_history_t *entry = NULL;

	crm_trace("Processing lrmd_rsc_info_t entry %s", rsc_id);

	entry = g_hash_table_lookup(resource_history, rsc_id);
	if (entry == NULL \|\| entry->last == NULL) {
	return FALSE;
	}

	if (entry->last->rc == PCMK_EXECRA_OK && safe_str_eq(entry->last->op_type, CRMD_ACTION_STOP)) {
	return FALSE;

	} else if (entry->last->rc == PCMK_EXECRA_OK
	&& safe_str_eq(entry->last->op_type, CRMD_ACTION_MIGRATE)) {
	/* a stricter check is too complex...
	* leave that to the PE
	*/
	return FALSE;

	} else if (entry->last->rc == PCMK_EXECRA_NOT_RUNNING) {
	return FALSE;

	} else if (entry->last->interval == 0 && entry->last->rc == PCMK_EXECRA_NOT_CONFIGURED) {
	/* Badly configured resources can't be reliably stopped */
	return FALSE;
	}

	return TRUE;
	}

	gboolean
	build_active_RAs(xmlNode * rsc_list)
	{
	GHashTableIter iter;
	rsc_history_t *entry = NULL;

	g_hash_table_iter_init(&iter, resource_history);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {

	GList *gIter = NULL;
	xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE);

	crm_xml_add(xml_rsc, XML_ATTR_ID, entry->id);
	crm_xml_add(xml_rsc, XML_ATTR_TYPE, entry->rsc.type);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.class);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, entry->rsc.provider);

	build_operation_update(xml_rsc, &(entry->rsc), entry->last, __FUNCTION__);
	build_operation_update(xml_rsc, &(entry->rsc), entry->failed, __FUNCTION__);
	for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
	build_operation_update(xml_rsc, &(entry->rsc), gIter->data, __FUNCTION__);
	}
	}

	return FALSE;
	}

	xmlNode *
	do_lrm_query(gboolean is_replace)
	{
	gboolean shut_down = FALSE;
	xmlNode *xml_result = NULL;
	xmlNode *xml_state = NULL;
	xmlNode *xml_data = NULL;
	xmlNode *rsc_list = NULL;
	const char *exp_state = CRMD_STATE_ACTIVE;

	if (is_set(fsa_input_register, R_SHUTDOWN)) {
	exp_state = CRMD_STATE_INACTIVE;
	shut_down = TRUE;
	}

	xml_state = create_node_state(fsa_our_uname, XML_BOOLEAN_TRUE,
	ONLINESTATUS, CRMD_JOINSTATE_MEMBER, exp_state,
	!shut_down, __FUNCTION__);

	xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM);
	crm_xml_add(xml_data, XML_ATTR_ID, fsa_our_uuid);
	rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);

	/* Build a list of active (not always running) resources */
	build_active_RAs(rsc_list);

	xml_result = create_cib_fragment(xml_state, XML_CIB_TAG_STATUS);
	crm_log_xml_trace(xml_state, "Current state of the LRM");
	free_xml(xml_state);

	return xml_result;
	}

	static void
	notify_deleted(ha_msg_input_t * input, const char *rsc_id, int rc)
	{
	lrmd_event_data_t *op = NULL;
	const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
	const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);

	crm_info("Notifying %s on %s that %s was%s deleted",
	from_sys, from_host, rsc_id, rc == pcmk_ok ? "" : " not");

	op = construct_op(input->xml, rsc_id, CRMD_ACTION_DELETE);
	CRM_ASSERT(op != NULL);

	if (rc == pcmk_ok) {
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_EXECRA_OK;
	} else {
	op->op_status = PCMK_LRM_OP_ERROR;
	op->rc = PCMK_EXECRA_UNKNOWN_ERROR;
	}

	send_direct_ack(from_host, from_sys, NULL, op, rsc_id);
	lrmd_free_event(op);

	if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
	/* this isn't expected - trigger a new transition */
	time_t now = time(NULL);
	char *now_s = crm_itoa(now);

	crm_debug("Triggering a refresh after %s deleted %s from the LRM", from_sys, rsc_id);

	update_attr_delegate(
	fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	"last-lrm-refresh", now_s, FALSE, NULL);

	free(now_s);
	}
	}

	static gboolean
	lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
	{
	struct delete_event_s *event = user_data;
	struct pending_deletion_op_s *op = value;

	if (safe_str_eq(event->rsc, op->rsc)) {
	notify_deleted(op->input, event->rsc, event->rc);
	return TRUE;
	}
	return FALSE;
	}

	static gboolean
	lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
	{
	const char *rsc = user_data;
	struct recurring_op_s *pending = value;

	if (safe_str_eq(rsc, pending->rsc_id)) {
	crm_info("Removing op %s:%d for deleted resource %s",
	pending->op_key, pending->call_id, rsc);
	return TRUE;
	}
	return FALSE;
	}

	/*
	* Remove the rsc from the CIB
	*
	* Avoids refreshing the entire LRM section of this host
	*/
	#define rsc_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"

	static int
	delete_rsc_status(const char rsc_id, int call_options, const char user_name)
	{
	char *rsc_xpath = NULL;
	int max = 0;
	int rc = pcmk_ok;

	CRM_CHECK(rsc_id != NULL, return -ENXIO);

	max = strlen(rsc_template) + strlen(rsc_id) + strlen(fsa_our_uname) + 1;
	rsc_xpath = calloc(1, max);
	snprintf(rsc_xpath, max, rsc_template, fsa_our_uname, rsc_id);

	- rc = fsa_cib_conn->cmds->delegated_variant_op(fsa_cib_conn, CIB_OP_DELETE, NULL, rsc_xpath,
	- NULL, NULL, call_options \| cib_xpath, user_name);
	+ rc = cib_internal_op(fsa_cib_conn, CIB_OP_DELETE, NULL, rsc_xpath,
	+ NULL, NULL, call_options \| cib_xpath, user_name);

	free(rsc_xpath);
	return rc;
	}

	static void
	delete_rsc_entry(ha_msg_input_t * input, const char rsc_id, GHashTableIter rsc_gIter, int rc, const char *user_name)
	{
	struct delete_event_s event;

	CRM_CHECK(rsc_id != NULL, return);

	if (rc == pcmk_ok) {
	char *rsc_id_copy = strdup(rsc_id);

	if (rsc_gIter)
	g_hash_table_iter_remove(rsc_gIter);
	else
	g_hash_table_remove(resource_history, rsc_id_copy);
	crm_debug("sync: Sending delete op for %s", rsc_id_copy);
	delete_rsc_status(rsc_id_copy, cib_quorum_override, user_name);

	g_hash_table_foreach_remove(pending_ops, lrm_remove_deleted_op, rsc_id_copy);

	free(rsc_id_copy);
	}

	if (input) {
	notify_deleted(input, rsc_id, rc);
	}

	event.rc = rc;
	event.rsc = rsc_id;
	g_hash_table_foreach_remove(deletion_ops, lrm_remove_deleted_rsc, &event);
	}

	/*
	* Remove the op from the CIB
	*
	* Avoids refreshing the entire LRM section of this host
	*/

	#define op_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s']"
	#define op_call_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s' and @"XML_LRM_ATTR_CALLID"='%d']"

	static void
	delete_op_entry(lrmd_event_data_t * op, const char rsc_id, const char key, int call_id)
	{
	xmlNode *xml_top = NULL;

	if (op != NULL) {
	xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
	crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
	crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);

	if(op->interval > 0) {
	char *op_id = generate_op_key(op->rsc_id, op->op_type, op->interval);
	/* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
	crm_xml_add(xml_top, XML_ATTR_ID, op_id);
	free(op_id);
	}

	crm_debug("async: Sending delete op for %s_%s_%d (call=%d)",
	op->rsc_id, op->op_type, op->interval, op->call_id);

	fsa_cib_conn->cmds->delete(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top, cib_quorum_override);

	} else if (rsc_id != NULL && key != NULL) {
	int max = 0;
	char *op_xpath = NULL;

	if (call_id > 0) {
	max =
	strlen(op_call_template) + strlen(rsc_id) + strlen(fsa_our_uname) + strlen(key) +
	10;
	op_xpath = calloc(1, max);
	snprintf(op_xpath, max, op_call_template, fsa_our_uname, rsc_id, key, call_id);

	} else {
	max = strlen(op_template) + strlen(rsc_id) + strlen(fsa_our_uname) + strlen(key) + 1;
	op_xpath = calloc(1, max);
	snprintf(op_xpath, max, op_template, fsa_our_uname, rsc_id, key);
	}

	crm_debug("sync: Sending delete op for %s (call=%d)", rsc_id, call_id);
	fsa_cib_conn->cmds->delete(fsa_cib_conn, op_xpath, NULL, cib_quorum_override \| cib_xpath);

	free(op_xpath);

	} else {
	crm_err("Not enough information to delete op entry: rsc=%p key=%p", rsc_id, key);
	return;
	}

	crm_log_xml_trace(xml_top, "op:cancel");
	free_xml(xml_top);
	}

	void lrm_clear_last_failure(const char *rsc_id)
	{
	char *attr = NULL;
	GHashTableIter iter;
	rsc_history_t *entry = NULL;

	attr = generate_op_key(rsc_id, "last_failure", 0);
	delete_op_entry(NULL, rsc_id, attr, 0);
	free(attr);

	if (!resource_history) {
	return;
	}

	g_hash_table_iter_init(&iter, resource_history);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
	if (safe_str_eq(rsc_id, entry->id)) {
	lrmd_free_event(entry->failed);
	entry->failed = NULL;
	}
	}
	}

	static gboolean
	cancel_op(const char rsc_id, const char key, int op, gboolean remove)
	{
	int rc = pcmk_ok;
	struct recurring_op_s *pending = NULL;

	CRM_CHECK(op != 0, return FALSE);
	CRM_CHECK(rsc_id != NULL, return FALSE);
	if (key == NULL) {
	key = make_stop_id(rsc_id, op);
	}
	pending = g_hash_table_lookup(pending_ops, key);

	if (pending) {
	if (remove && pending->remove == FALSE) {
	pending->remove = TRUE;
	crm_debug("Scheduling %s for removal", key);
	}

	if (pending->cancelled) {
	crm_debug("Operation %s already cancelled", key);
	return TRUE;
	}

	pending->cancelled = TRUE;

	} else {
	crm_info("No pending op found for %s", key);
	return TRUE;
	}

	crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);

	rc = fsa_lrm_conn->cmds->cancel(fsa_lrm_conn,
	pending->rsc_id,
	pending->op_type,
	pending->interval);

	if (rc == pcmk_ok) {
	crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
	} else {
	crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
	/* The caller needs to make sure the entry is
	* removed from the pending_ops list
	*
	* Usually by returning TRUE inside the worker function
	* supplied to g_hash_table_foreach_remove()
	*
	* Not removing the entry from pending_ops will block
	* the node from shutting down
	*/
	return FALSE;
	}

	return TRUE;
	}

	struct cancel_data {
	gboolean done;
	gboolean remove;
	const char *key;
	lrmd_rsc_info_t *rsc;
	};

	static gboolean
	cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
	{
	struct cancel_data *data = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;

	if (safe_str_eq(op->op_key, data->key)) {
	data->done = TRUE;
	if (cancel_op(data->rsc->id, key, op->call_id, data->remove) == FALSE) {
	return TRUE;
	}
	}
	return FALSE;
	}

	static gboolean
	cancel_op_key(lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
	{
	struct cancel_data data;

	CRM_CHECK(rsc != NULL, return FALSE);
	CRM_CHECK(key != NULL, return FALSE);

	data.key = key;
	data.rsc = rsc;
	data.done = FALSE;
	data.remove = remove;

	g_hash_table_foreach_remove(pending_ops, cancel_action_by_key, &data);
	return data.done;
	}

	static lrmd_rsc_info_t *
	get_lrm_resource(xmlNode * resource, xmlNode * op_msg, gboolean do_create)
	{
	lrmd_rsc_info_t *rsc = NULL;
	const char *id = ID(resource);
	const char *type = crm_element_value(resource, XML_ATTR_TYPE);
	const char *class = crm_element_value(resource, XML_AGENT_ATTR_CLASS);
	const char *provider = crm_element_value(resource, XML_AGENT_ATTR_PROVIDER);
	const char *long_id = crm_element_value(resource, XML_ATTR_ID_LONG);

	crm_trace("Retrieving %s from the LRM.", id);
	CRM_CHECK(id != NULL, return NULL);

	rsc = fsa_lrm_conn->cmds->get_rsc_info(fsa_lrm_conn, id, 0);
	if (!rsc && long_id) {
	rsc = fsa_lrm_conn->cmds->get_rsc_info(fsa_lrm_conn, long_id, 0);
	}

	if (!rsc && do_create) {
	CRM_CHECK(class != NULL, return NULL);
	CRM_CHECK(type != NULL, return NULL);

	crm_trace("Adding rsc %s before operation", id);

	fsa_lrm_conn->cmds->register_rsc(fsa_lrm_conn,
	id, class, provider, type, 0);

	rsc = fsa_lrm_conn->cmds->get_rsc_info(fsa_lrm_conn, id, 0);

	if (!rsc) {
	fsa_data_t *msg_data = NULL;

	crm_err("Could not add resource %s to LRM", id);
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
	}
	}

	return rsc;
	}

	static void
	delete_resource(const char id, lrmd_rsc_info_t rsc, GHashTableIter *gIter,
	const char sys, const char host, const char user, ha_msg_input_t request)
	{
	int rc = pcmk_ok;

	crm_info("Removing resource %s for %s (%s) on %s", id, sys, user ? user : "internal", host);

	if (rsc) {
	rc = fsa_lrm_conn->cmds->unregister_rsc(fsa_lrm_conn, id, 0);
	}

	if (rc == pcmk_ok) {
	crm_trace("Resource '%s' deleted", id);
	} else if (rc == -EINPROGRESS) {
	crm_info("Deletion of resource '%s' pending", id);
	if (request) {
	struct pending_deletion_op_s *op = NULL;
	char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE);

	op = calloc(1, sizeof(struct pending_deletion_op_s));
	op->rsc = strdup(rsc->id);
	op->input = copy_ha_msg_input(request);
	g_hash_table_insert(deletion_ops, ref, op);
	}
	return;
	} else {
	crm_warn("Deletion of resource '%s' for %s (%s) on %s failed: %d",
	id, sys, user ? user : "internal", host, rc);
	}

	delete_rsc_entry(request, id, gIter, rc, user);
	}

	/* A_LRM_INVOKE */
	void
	do_lrm_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	gboolean done = FALSE;
	gboolean create_rsc = TRUE;
	const char *crm_op = NULL;
	const char *from_sys = NULL;
	const char *from_host = NULL;
	const char *operation = NULL;
	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
	const char *user_name = NULL;

	#if ENABLE_ACL
	user_name = crm_element_value(input->msg, F_CRM_USER);
	crm_trace("LRM command from user '%s'", user_name);
	#endif

	crm_op = crm_element_value(input->msg, F_CRM_TASK);
	from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
	if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
	from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
	}

	crm_trace("LRM command from: %s", from_sys);

	if (safe_str_eq(crm_op, CRM_OP_LRM_DELETE)) {
	operation = CRMD_ACTION_DELETE;

	} else if (safe_str_eq(operation, CRM_OP_LRM_REFRESH)) {
	crm_op = CRM_OP_LRM_REFRESH;

	} else if (safe_str_eq(crm_op, CRM_OP_LRM_FAIL)) {
	rsc_history_t *entry = NULL;
	lrmd_event_data_t *op = NULL;
	lrmd_rsc_info_t *rsc = NULL;
	xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);

	CRM_CHECK(xml_rsc != NULL, return);

	/* The lrmd can not fail a resource, it does not understand the
	* concept of success or failure in relation to a resource, it simply
	* executes operations and reports the results. We determine what a failure is.
	* Becaues of this, if we want to fail a resource we have to fake what we
	* understand a failure to look like.
	*
	* To do this we create a fake lrmd operation event for the resource
	* we want to fail. We then pass that event to the lrmd client callback
	* so it will be processed as if it actually came from the lrmd. */
	op = construct_op(input->xml, ID(xml_rsc), "asyncmon");
	free((char *) op->user_data);
	op->user_data = NULL;
	entry = g_hash_table_lookup(resource_history, op->rsc_id);
	/* Make sure the call id is greater than the last successful operation,
	* otherwise the failure will not result in a possible recovery of the resource
	* as it could appear the failure occurred before the successful start */
	if (entry && entry->last) {
	op->call_id = entry->last->call_id + 1;
	if (op->call_id < 0) {
	op->call_id = 1;
	}
	}
	op->interval = 0;
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_EXECRA_UNKNOWN_ERROR;
	CRM_ASSERT(op != NULL);

	# if ENABLE_ACL
	if (user_name && is_privileged(user_name) == FALSE) {
	crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc));
	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);
	return;
	}
	# endif

	rsc = get_lrm_resource(xml_rsc, input->xml, create_rsc);
	if (rsc) {
	crm_info("Failing resource %s...", rsc->id);
	process_lrm_event(op);
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_EXECRA_OK;
	lrmd_free_rsc_info(rsc);
	} else {
	crm_info("Cannot find/create resource in order to fail it...");
	crm_log_xml_warn(input->msg, "bad input");
	}

	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);
	return;

	} else if (input->xml != NULL) {
	operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
	}

	if (safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) {
	int rc = pcmk_ok;
	xmlNode *fragment = do_lrm_query(TRUE);

	crm_info("Forcing a local LRM refresh");

	fsa_cib_update(XML_CIB_TAG_STATUS, fragment, cib_quorum_override, rc, user_name);
	free_xml(fragment);

	} else if (safe_str_eq(crm_op, CRM_OP_LRM_QUERY)) {
	xmlNode *data = do_lrm_query(FALSE);
	xmlNode *reply = create_reply(input->msg, data);

	if (relay_message(reply, TRUE) == FALSE) {
	crm_err("Unable to route reply");
	crm_log_xml_err(reply, "reply");
	}
	free_xml(reply);
	free_xml(data);

	} else if (safe_str_eq(operation, CRM_OP_PROBED)) {
	update_attrd(NULL, CRM_OP_PROBED, XML_BOOLEAN_TRUE, user_name);

	} else if (safe_str_eq(crm_op, CRM_OP_REPROBE)) {
	GHashTableIter gIter;
	rsc_history_t *entry = NULL;

	crm_notice("Forcing the status of all resources to be redetected");

	g_hash_table_iter_init(&gIter, resource_history);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	delete_resource(entry->id, &entry->rsc, &gIter, from_sys, from_host, user_name, NULL);
	}

	/* Now delete the copy in the CIB */
	erase_status_tag(fsa_our_uname, XML_CIB_TAG_LRM, cib_scope_local);

	/* And finally, _delete_ the value in attrd
	* Setting it to FALSE results in the PE sending us back here again
	*/
	update_attrd(NULL, CRM_OP_PROBED, NULL, user_name);

	} else if (operation != NULL) {
	lrmd_rsc_info_t *rsc = NULL;
	xmlNode *params = NULL;
	xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);

	CRM_CHECK(xml_rsc != NULL, return);

	/* only the first 16 chars are used by the LRM */
	params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);

	if (safe_str_eq(operation, CRMD_ACTION_DELETE)) {
	create_rsc = FALSE;
	}

	rsc = get_lrm_resource(xml_rsc, input->xml, create_rsc);

	if (rsc == NULL && create_rsc) {
	crm_err("Invalid resource definition");
	crm_log_xml_warn(input->msg, "bad input");

	} else if (rsc == NULL) {
	lrmd_event_data_t *op = NULL;

	crm_notice("Not creating resource for a %s event: %s", operation, ID(input->xml));
	delete_rsc_entry(input, ID(xml_rsc), NULL, pcmk_ok, user_name);

	op = construct_op(input->xml, ID(xml_rsc), operation);
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_EXECRA_OK;
	CRM_ASSERT(op != NULL);
	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);

	} else if (safe_str_eq(operation, CRMD_ACTION_CANCEL)) {
	lrmd_event_data_t *op = NULL;
	char *op_key = NULL;
	char *meta_key = NULL;
	int call = 0;
	const char *call_id = NULL;
	const char *op_task = NULL;
	const char *op_interval = NULL;

	CRM_CHECK(params != NULL, crm_log_xml_warn(input->xml, "Bad command");
	return);

	meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
	op_interval = crm_element_value(params, meta_key);
	free(meta_key);

	meta_key = crm_meta_name(XML_LRM_ATTR_TASK);
	op_task = crm_element_value(params, meta_key);
	free(meta_key);

	meta_key = crm_meta_name(XML_LRM_ATTR_CALLID);
	call_id = crm_element_value(params, meta_key);
	free(meta_key);

	CRM_CHECK(op_task != NULL, crm_log_xml_warn(input->xml, "Bad command");
	return);
	CRM_CHECK(op_interval != NULL, crm_log_xml_warn(input->xml, "Bad command");
	return);

	op = construct_op(input->xml, rsc->id, op_task);
	CRM_ASSERT(op != NULL);
	op_key = generate_op_key(rsc->id, op_task, crm_parse_int(op_interval, "0"));

	crm_debug("PE requested op %s (call=%s) be cancelled",
	op_key, call_id ? call_id : "NA");
	call = crm_parse_int(call_id, "0");
	if (call == 0) {
	/* the normal case when the PE cancels a recurring op */
	done = cancel_op_key(rsc, op_key, TRUE);

	} else {
	/* the normal case when the PE cancels an orphan op */
	done = cancel_op(rsc->id, NULL, call, TRUE);
	}

	if (done == FALSE) {
	crm_debug("Nothing known about operation %d for %s", call, op_key);
	delete_op_entry(NULL, rsc->id, op_key, call);

	/* needed?? surely not otherwise the cancel_op_(_key) wouldn't
	* have failed in the first place
	*/
	g_hash_table_remove(pending_ops, op_key);
	}

	op->rc = PCMK_EXECRA_OK;
	op->op_status = PCMK_LRM_OP_DONE;
	send_direct_ack(from_host, from_sys, rsc, op, rsc->id);

	free(op_key);
	lrmd_free_event(op);

	} else if (safe_str_eq(operation, CRMD_ACTION_DELETE)) {
	int cib_rc = pcmk_ok;

	CRM_ASSERT(rsc != NULL);

	cib_rc = delete_rsc_status(rsc->id, cib_dryrun \| cib_sync_call, user_name);
	if (cib_rc != pcmk_ok) {
	lrmd_event_data_t *op = NULL;

	crm_err
	("Attempt of deleting resource status '%s' from CIB for %s (user=%s) on %s failed: (rc=%d) %s",
	rsc->id, from_sys, user_name ? user_name : "unknown", from_host, cib_rc,
	pcmk_strerror(cib_rc));

	op = construct_op(input->xml, rsc->id, operation);
	op->op_status = PCMK_LRM_OP_ERROR;

	if (cib_rc == -EACCES) {
	op->rc = PCMK_EXECRA_INSUFFICIENT_PRIV;
	} else {
	op->rc = PCMK_EXECRA_UNKNOWN_ERROR;
	}
	send_direct_ack(from_host, from_sys, NULL, op, rsc->id);
	lrmd_free_event(op);
	return;
	}

	delete_resource(rsc->id, rsc, NULL, from_sys, from_host, user_name, input);

	} else if (rsc != NULL) {
	do_lrm_rsc_op(rsc, operation, input->xml, input->msg);
	}

	lrmd_free_rsc_info(rsc);

	} else {
	crm_err("Operation was neither a lrm_query, nor a rsc op. %s", crm_str(crm_op));
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	static lrmd_event_data_t *
	construct_op(xmlNode * rsc_op, const char rsc_id, const char operation)
	{
	lrmd_event_data_t *op = NULL;
	const char *op_delay = NULL;
	const char *op_timeout = NULL;
	const char *op_interval = NULL;
	GHashTable *params = NULL;

	const char *transition = NULL;

	CRM_LOG_ASSERT(rsc_id != NULL);

	op = calloc(1, sizeof(lrmd_event_data_t));
	op->type = lrmd_event_exec_complete;
	op->op_type = strdup(operation);
	op->op_status = PCMK_LRM_OP_PENDING;
	op->rc = -1;
	op->rsc_id = strdup(rsc_id);
	op->interval = 0;
	op->timeout = 0;
	op->start_delay = 0;

	if (rsc_op == NULL) {
	CRM_LOG_ASSERT(safe_str_eq(CRMD_ACTION_STOP, operation));
	op->user_data = NULL;
	/* the stop_all_resources() case
	* by definition there is no DC (or they'd be shutting
	* us down).
	* So we should put our version here.
	*/
	op->params = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	g_hash_table_insert(op->params,
	strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));

	crm_trace("Constructed %s op for %s", operation, rsc_id);
	return op;
	}

	params = xml2list(rsc_op);
	g_hash_table_remove(params, CRM_META "_op_target_rc");

	op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY);
	op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT);
	op_interval = crm_meta_value(params, XML_LRM_ATTR_INTERVAL);

	op->interval = crm_parse_int(op_interval, "0");
	op->timeout = crm_parse_int(op_timeout, "0");
	op->start_delay = crm_parse_int(op_delay, "0");

	if (safe_str_neq(operation, RSC_STOP)) {
	op->params = params;

	} else {
	rsc_history_t *entry = g_hash_table_lookup(resource_history, rsc_id);

	/* If we do not have stop parameters cached, use
	* whatever we are given */
	if (!entry \|\| !entry->stop_params) {
	op->params = params;
	} else {
	/* Copy the cached parameter list so that we stop the resource
	* with the old attributes, not the new ones */
	op->params = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	g_hash_table_foreach(params, copy_meta_keys, op->params);
	g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
	g_hash_table_destroy(params);
	params = NULL;
	}
	}

	/* sanity */
	if (op->interval < 0) {
	op->interval = 0;
	}
	if (op->timeout <= 0) {
	op->timeout = op->interval;
	}
	if (op->start_delay < 0) {
	op->start_delay = 0;
	}

	transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
	CRM_CHECK(transition != NULL, return op);

	op->user_data = strdup(transition);

	if (op->interval != 0) {
	if (safe_str_eq(operation, CRMD_ACTION_START)
	\|\| safe_str_eq(operation, CRMD_ACTION_STOP)) {
	crm_err("Start and Stop actions cannot have an interval: %d", op->interval);
	op->interval = 0;
	}
	}

	crm_trace("Constructed %s op for %s: interval=%d", operation, rsc_id, op->interval);

	return op;
	}

	void
	send_direct_ack(const char to_host, const char to_sys,
	lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id)
	{
	xmlNode *reply = NULL;
	xmlNode update, iter;
	xmlNode *fragment;

	CRM_CHECK(op != NULL, return);
	if (op->rsc_id == NULL) {
	CRM_LOG_ASSERT(rsc_id != NULL);
	op->rsc_id = strdup(rsc_id);
	}
	if (to_sys == NULL) {
	to_sys = CRM_SYSTEM_TENGINE;
	}
	update = create_node_state(fsa_our_uname, NULL, NULL, NULL, NULL, FALSE, __FUNCTION__);

	iter = create_xml_node(update, XML_CIB_TAG_LRM);
	crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);

	crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);

	build_operation_update(iter, rsc, op, __FUNCTION__);
	fragment = create_cib_fragment(update, XML_CIB_TAG_STATUS);

	reply = create_request(CRM_OP_INVOKE_LRM, fragment, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);

	crm_log_xml_trace(update, "ACK Update");

	crm_debug("ACK'ing resource op %s_%s_%d from %s: %s",
	op->rsc_id, op->op_type, op->interval, op->user_data,
	crm_element_value(reply, XML_ATTR_REFERENCE));

	if (relay_message(reply, TRUE) == FALSE) {
	crm_log_xml_err(reply, "Unable to route reply");
	}

	free_xml(fragment);
	free_xml(update);
	free_xml(reply);
	}

	static gboolean
	stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
	{
	lrmd_rsc_info_t *rsc = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;

	if (op->interval != 0 && safe_str_eq(op->rsc_id, rsc->id)) {
	if (cancel_op(rsc->id, key, op->call_id, FALSE) == FALSE) {
	return TRUE;
	}
	}

	return FALSE;
	}

	static gboolean
	stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
	{
	gboolean remove = FALSE;
	struct recurring_op_s op = (struct recurring_op_s )value;

	if (op->interval != 0) {
	remove = cancel_op(op->rsc_id, key, op->call_id, FALSE);
	}

	return remove;
	}

	void
	do_lrm_rsc_op(lrmd_rsc_info_t * rsc, const char operation, xmlNode msg, xmlNode * request)
	{
	int call_id = 0;
	char *op_id = NULL;
	lrmd_event_data_t *op = NULL;
	lrmd_key_value_t *params = NULL;
	fsa_data_t *msg_data = NULL;
	const char *transition = NULL;

	CRM_CHECK(rsc != NULL, return);

	if (msg != NULL) {
	transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
	if (transition == NULL) {
	crm_log_xml_err(msg, "Missing transition number");
	}
	}

	op = construct_op(msg, rsc->id, operation);

	/* stop the monitor before stopping the resource */
	if (crm_str_eq(operation, CRMD_ACTION_STOP, TRUE)
	\|\| crm_str_eq(operation, CRMD_ACTION_DEMOTE, TRUE)
	\|\| crm_str_eq(operation, CRMD_ACTION_PROMOTE, TRUE)
	\|\| crm_str_eq(operation, CRMD_ACTION_MIGRATE, TRUE)) {
	g_hash_table_foreach_remove(pending_ops, stop_recurring_action_by_rsc, rsc);
	}

	/* now do the op */
	crm_debug("Performing key=%s op=%s_%s_%d", transition, rsc->id, operation, op->interval);

	if (fsa_state != S_NOT_DC && fsa_state != S_POLICY_ENGINE && fsa_state != S_TRANSITION_ENGINE) {
	if (safe_str_neq(operation, "fail")
	&& safe_str_neq(operation, CRMD_ACTION_STOP)) {
	crm_info("Discarding attempt to perform action %s on %s"
	" in state %s", operation, rsc->id, fsa_state2string(fsa_state));
	op->rc = 99;
	op->op_status = PCMK_LRM_OP_ERROR;
	send_direct_ack(NULL, NULL, rsc, op, rsc->id);
	lrmd_free_event(op);
	free(op_id);
	return;
	}
	}

	op_id = generate_op_key(rsc->id, op->op_type, op->interval);

	if (op->interval > 0) {
	/* cancel it so we can then restart it without conflict */
	cancel_op_key(rsc, op_id, FALSE);
	}

	if (op->params) {
	char *key = NULL;
	char *value = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, op->params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	params = lrmd_key_value_add(params, key, value);
	}
	}

	call_id = fsa_lrm_conn->cmds->exec(fsa_lrm_conn,
	rsc->id,
	op->op_type,
	op->user_data,
	op->interval,
	op->timeout,
	op->start_delay,
	0,
	params);

	if (call_id <= 0) {
	crm_err("Operation %s on %s failed: %d", operation, rsc->id, call_id);
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);

	} else {
	/* record all operations so we can wait
	* for them to complete during shutdown
	*/
	char *call_id_s = make_stop_id(rsc->id, call_id);
	struct recurring_op_s *pending = NULL;

	pending = calloc(1, sizeof(struct recurring_op_s));
	crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);

	pending->call_id = call_id;
	pending->interval = op->interval;
	pending->op_type = strdup(operation);
	pending->op_key = strdup(op_id);
	pending->rsc_id = strdup(rsc->id);
	pending->last_rc = -1; /* All rc are positive, -1 indicates the last rc has not been set. */
	pending->last_op_status = -2;
	g_hash_table_replace(pending_ops, call_id_s, pending);

	if (op->interval > 0 && op->start_delay > START_DELAY_THRESHOLD) {
	char *uuid = NULL;
	int dummy = 0, target_rc = 0;

	crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);

	decode_transition_key(op->user_data, &uuid, &dummy, &dummy, &target_rc);
	free(uuid);

	op->rc = target_rc;
	op->op_status = PCMK_LRM_OP_DONE;
	send_direct_ack(NULL, NULL, rsc, op, rsc->id);
	}
	}

	free(op_id);
	lrmd_free_event(op);
	return;
	}

	static void
	cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	switch (rc) {
	case pcmk_ok:
	case -pcmk_err_diff_failed:
	case -pcmk_err_diff_resync:
	crm_trace("Resource update %d complete: rc=%d", call_id, rc);
	break;
	default:
	crm_warn("Resource update %d failed: (rc=%d) %s", call_id, rc, pcmk_strerror(rc));
	}
	}

	static int
	do_update_resource(lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
	{
	/*
	<status>
	<nodes_status id=uname>
	<lrm>
	<lrm_resources>
	<lrm_resource id=...>
	</...>
	*/
	int rc = pcmk_ok;
	xmlNode update, iter = NULL;
	int call_opt = cib_quorum_override;

	CRM_CHECK(op != NULL, return 0);

	if (fsa_state == S_ELECTION \|\| fsa_state == S_PENDING) {
	crm_info("Sending update to local CIB in state: %s", fsa_state2string(fsa_state));
	call_opt \|= cib_scope_local;
	}

	iter = create_xml_node(iter, XML_CIB_TAG_STATUS);
	update = iter;
	iter = create_xml_node(iter, XML_CIB_TAG_STATE);

	set_uuid(iter, XML_ATTR_UUID, fsa_our_uname);
	crm_xml_add(iter, XML_ATTR_UNAME, fsa_our_uname);
	crm_xml_add(iter, XML_ATTR_ORIGIN, __FUNCTION__);

	iter = create_xml_node(iter, XML_CIB_TAG_LRM);
	crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);

	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
	crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);

	build_operation_update(iter, rsc, op, __FUNCTION__);

	if (rsc) {
	crm_xml_add(iter, XML_ATTR_TYPE, rsc->type);
	crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->class);
	crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER, rsc->provider);

	CRM_CHECK(rsc->type != NULL, crm_err("Resource %s has no value for type", op->rsc_id));
	CRM_CHECK(rsc->class != NULL, crm_err("Resource %s has no value for class", op->rsc_id));

	} else {
	crm_warn("Resource %s no longer exists in the lrmd", op->rsc_id);
	goto cleanup;
	}

	/* make it an asyncronous call and be done with it
	*
	* Best case:
	* the resource state will be discovered during
	* the next signup or election.
	*
	* Bad case:
	* we are shutting down and there is no DC at the time,
	* but then why were we shutting down then anyway?
	* (probably because of an internal error)
	*
	* Worst case:
	* we get shot for having resources "running" when the really weren't
	*
	* the alternative however means blocking here for too long, which
	* isnt acceptable
	*/
	fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, rc, NULL);

	/* the return code is a call number, not an error code */
	crm_trace("Sent resource state update message: %d", rc);
	fsa_cib_conn->cmds->register_callback(fsa_cib_conn, rc, 60, FALSE, NULL, "cib_rsc_callback",
	cib_rsc_callback);

	cleanup:
	free_xml(update);
	return rc;
	}

	void
	do_lrm_event(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
	{
	CRM_CHECK(FALSE, return);
	}

	gboolean
	process_lrm_event(lrmd_event_data_t * op)
	{
	char *op_id = NULL;
	char *op_key = NULL;

	int update_id = 0;
	int log_level = LOG_ERR;
	gboolean removed = FALSE;
	lrmd_rsc_info_t *rsc = NULL;

	struct recurring_op_s *pending = NULL;

	CRM_CHECK(op != NULL, return FALSE);

	if (op->type == lrmd_event_disconnect) {
	lrm_connection_destroy();
	return TRUE;
	} else if (op->type != lrmd_event_exec_complete) {
	return TRUE;
	}

	CRM_CHECK(op->rsc_id != NULL, return FALSE);
	op_id = make_stop_id(op->rsc_id, op->call_id);
	pending = g_hash_table_lookup(pending_ops, op_id);

	/* ignore recurring ops that have not changed. */
	if (op->interval &&
	pending &&
	(pending->last_rc == op->rc) &&
	(pending->last_op_status == op->op_status)) {
	free(op_id);
	return TRUE;
	}

	if (pending) {
	pending->last_rc = op->rc;
	pending->last_op_status = op->op_status;
	}

	op_key = generate_op_key(op->rsc_id, op->op_type, op->interval);
	rsc = fsa_lrm_conn->cmds->get_rsc_info(fsa_lrm_conn, op->rsc_id, 0);

	switch (op->op_status) {
	case PCMK_LRM_OP_ERROR:
	case PCMK_LRM_OP_PENDING:
	case PCMK_LRM_OP_NOTSUPPORTED:
	break;
	case PCMK_LRM_OP_CANCELLED:
	log_level = LOG_INFO;
	break;
	case PCMK_LRM_OP_DONE:
	log_level = LOG_NOTICE;
	break;
	case PCMK_LRM_OP_TIMEOUT:
	log_level = LOG_DEBUG_3;
	crm_err("LRM operation %s (%d) %s (timeout=%dms)",
	op_key, op->call_id, services_lrm_status_str(op->op_status), op->timeout);
	break;
	default:
	crm_err("Mapping unknown status (%d) to ERROR", op->op_status);
	op->op_status = PCMK_LRM_OP_ERROR;
	}

	if (op->op_status == PCMK_LRM_OP_ERROR
	&& (op->rc == PCMK_EXECRA_RUNNING_MASTER \|\| op->rc == PCMK_EXECRA_NOT_RUNNING)) {
	/* Leave it up to the TE/PE to decide if this is an error */
	op->op_status = PCMK_LRM_OP_DONE;
	log_level = LOG_INFO;
	}

	if (op->op_status != PCMK_LRM_OP_CANCELLED) {
	if (safe_str_eq(op->op_type, RSC_NOTIFY)) {
	/* Keep notify ops out of the CIB */
	send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);
	} else {
	update_id = do_update_resource(rsc, op);
	}
	} else if (op->interval == 0) {
	/* This will occur when "crm resource cleanup" is called while actions are in-flight */
	crm_err("Op %s (call=%d): Cancelled", op_key, op->call_id);
	send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);

	} else if (pending == NULL) {
	/* Operations that are cancelled may safely be removed
	* from the pending op list before the lrmd completion event
	* is received. Only report non-cancelled ops here. */
	if (op->op_status != PCMK_LRM_OP_CANCELLED) {
	crm_err("Op %s (call=%d): No 'pending' entry", op_key, op->call_id);
	}
	} else if (op->user_data == NULL) {
	crm_err("Op %s (call=%d): No user data", op_key, op->call_id);

	} else if (pending->remove) {
	delete_op_entry(op, op->rsc_id, op_key, op->call_id);

	} else {
	/* Before a stop is called, no need to direct ack */
	crm_trace("Op %s (call=%d): no delete event required", op_key, op->call_id);
	}

	if ((op->interval == 0) && g_hash_table_remove(pending_ops, op_id)) {
	removed = TRUE;
	crm_trace("Op %s (call=%d, stop-id=%s): Confirmed", op_key, op->call_id, op_id);
	}

	if (op->op_status == PCMK_LRM_OP_DONE) {
	do_crm_log(log_level,
	"LRM operation %s (call=%d, rc=%d, cib-update=%d, confirmed=%s) %s",
	op_key, op->call_id, op->rc, update_id, removed ? "true" : "false",
	lrmd_event_rc2str(op->rc));
	} else {
	do_crm_log(log_level,
	"LRM operation %s (call=%d, status=%d, cib-update=%d, confirmed=%s) %s",
	op_key, op->call_id, op->op_status, update_id, removed ? "true" : "false",
	services_lrm_status_str(op->op_status));
	}

	if (op->rc != 0 && op->output != NULL) {
	crm_info("Result: %s", op->output);
	} else if (op->output != NULL) {
	crm_debug("Result: %s", op->output);
	}

	if (op->rsc_deleted) {
	crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
	delete_rsc_entry(NULL, op->rsc_id, NULL, pcmk_ok, NULL);
	}

	/* If a shutdown was escalated while operations were pending,
	* then the FSA will be stalled right now... allow it to continue
	*/
	mainloop_set_trigger(fsa_source);
	update_history_cache(rsc, op);

	lrmd_free_rsc_info(rsc);
	free(op_key);
	free(op_id);

	return TRUE;
	}
	diff --git a/include/crm/cib.h b/include/crm/cib.h
	index 2edcac81b1..9a9829ddee 100644
	--- a/include/crm/cib.h
	+++ b/include/crm/cib.h
	@@ -1,176 +1,171 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/
	#ifndef CIB__H
	# define CIB__H

	# include <crm/common/ipc.h>
	# include <crm/common/xml.h>

	# define CIB_FEATURE_SET "2.0"

	/* use compare_version() for doing comparisons */

	enum cib_variant {
	cib_undefined,
	cib_native,
	cib_file,
	cib_remote,
	cib_database,
	};

	enum cib_state {
	cib_connected_command,
	cib_connected_query,
	cib_disconnected
	};

	enum cib_conn_type {
	cib_command,
	cib_query,
	cib_no_connection,
	cib_command_nonblocking,
	};

	/* INDENT-OFF */
	enum cib_call_options {
	cib_none = 0x00000000,
	cib_verbose = 0x00000001,
	cib_xpath = 0x00000002,
	cib_multiple = 0x00000004,
	cib_can_create = 0x00000008,
	cib_discard_reply = 0x00000010,
	cib_no_children = 0x00000020,
	cib_scope_local = 0x00000100,
	cib_dryrun = 0x00000200,
	cib_sync_call = 0x00001000,
	cib_inhibit_notify = 0x00010000,
	cib_quorum_override = 0x00100000,
	cib_inhibit_bcast = 0x01000000,
	cib_force_diff = 0x10000000
	};

	#define cib_default_options = cib_none

	/* INDENT-ON */

	typedef struct cib_s cib_t;

	typedef struct cib_api_operations_s {
	- int (variant_op) (cib_t cib, const char op, const char host,
	- const char section, xmlNode data,
	- xmlNode ** output_data, int call_options);
	-
	int (signon) (cib_t cib, const char *name, enum cib_conn_type type);
	int (signon_raw) (cib_t cib, const char name, enum cib_conn_type type, int async_fd,
	int *unused);
	int (signoff) (cib_t cib);
	int (free) (cib_t cib);

	int (set_op_callback) (cib_t cib, void (callback) (const xmlNode msg, int callid,
	int rc, xmlNode * output));

	int (add_notify_callback) (cib_t cib, const char *event,
	void (callback) (const char event, xmlNode * msg));

	int (del_notify_callback) (cib_t cib, const char *event,
	void (callback) (const char event, xmlNode * msg));

	int (set_connection_dnotify) (cib_t cib, void (*dnotify) (gpointer user_data));

	int (inputfd) (cib_t cib);

	int (noop) (cib_t cib, int call_options);
	int (ping) (cib_t cib, xmlNode ** output_data, int call_options);

	int (query) (cib_t cib, const char section, xmlNode * output_data, int call_options);
	int (query_from) (cib_t cib, const char host, const char section,
	xmlNode ** output_data, int call_options);

	int (is_master) (cib_t cib);
	int (set_master) (cib_t cib, int call_options);
	int (set_slave) (cib_t cib, int call_options);
	int (set_slave_all) (cib_t cib, int call_options);

	int (sync) (cib_t cib, const char *section, int call_options);
	int (sync_from) (cib_t cib, const char host, const char section, int call_options);

	int (upgrade) (cib_t cib, int call_options);
	int (bump_epoch) (cib_t cib, int call_options);

	int (create) (cib_t cib, const char section, xmlNode data, int call_options);
	int (modify) (cib_t cib, const char section, xmlNode data, int call_options);
	int (update) (cib_t cib, const char section, xmlNode data, int call_options);
	int (replace) (cib_t cib, const char section, xmlNode data, int call_options);
	int (delete) (cib_t cib, const char section, xmlNode data, int call_options);

	int (erase) (cib_t cib, xmlNode ** output_data, int call_options);
	int (delete_absolute) (cib_t cib, const char section, xmlNode data, int call_options);

	int (quit) (cib_t cib, int call_options);

	int (register_notification) (cib_t cib, const char *callback, int enabled);

	- gboolean(register_callback) (cib_t cib, int call_id, int timeout, gboolean only_success,
	+ gboolean(register_callback) (cib_t cib, int call_id, int timeout, gboolean only_success,
	void user_data, const char callback_name,
	void (callback) (xmlNode , int, int, xmlNode , void ));

	- int (delegated_variant_op) (cib_t cib, const char op, const char host,
	- const char section, xmlNode data,
	- xmlNode ** output_data, int call_options, const char *user_name);
	+

	} cib_api_operations_t;

	struct cib_s {
	enum cib_state state;
	enum cib_conn_type type;
	enum cib_variant variant;

	int call_id;
	int call_timeout;
	void *variant_opaque;
	+ void *delegate_fn;

	GList *notify_list;
	void (op_callback) (const xmlNode msg, int call_id, int rc, xmlNode * output);

	cib_api_operations_t *cmds;
	};

	/* Core functions */
	cib_t *cib_new(void);
	cib_t *cib_native_new(void);
	cib_t cib_file_new(const char filename);
	cib_t cib_remote_new(const char server, const char user, const char passwd, int port,
	gboolean encrypted);

	cib_t *cib_new_no_shadow(void);
	char get_shadow_file(const char name);
	cib_t cib_shadow_new(const char name);

	void cib_delete(cib_t * cib);

	void cib_dump_pending_callbacks(void);
	int num_cib_op_callbacks(void);
	void remove_cib_op_callback(int call_id, gboolean all_callbacks);

	# define add_cib_op_callback(cib, id, flag, data, fn) cib->cmds->register_callback(cib, id, 120, flag, data, #fn, fn)

	# include <crm/cib/util.h>

	# define CIB_LIBRARY "libcib.so.1"

	#endif
	diff --git a/include/crm/cib/internal.h b/include/crm/cib/internal.h
	index 1c8d1aaa62..957ffdc95d 100644
	--- a/include/crm/cib/internal.h
	+++ b/include/crm/cib/internal.h
	@@ -1,177 +1,181 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/
	#ifndef CIB_INTERNAL__H
	# define CIB_INTERNAL__H
	#include <crm/cib.h>

	# define CIB_OP_SLAVE "cib_slave"
	# define CIB_OP_SLAVEALL "cib_slave_all"
	# define CIB_OP_MASTER "cib_master"
	# define CIB_OP_SYNC "cib_sync"
	# define CIB_OP_SYNC_ONE "cib_sync_one"
	# define CIB_OP_ISMASTER "cib_ismaster"
	# define CIB_OP_BUMP "cib_bump"
	# define CIB_OP_QUERY "cib_query"
	# define CIB_OP_CREATE "cib_create"
	# define CIB_OP_UPDATE "cib_update"
	# define CIB_OP_MODIFY "cib_modify"
	# define CIB_OP_DELETE "cib_delete"
	# define CIB_OP_ERASE "cib_erase"
	# define CIB_OP_REPLACE "cib_replace"
	# define CIB_OP_NOTIFY "cib_notify"
	# define CIB_OP_APPLY_DIFF "cib_apply_diff"
	# define CIB_OP_UPGRADE "cib_upgrade"
	# define CIB_OP_DELETE_ALT "cib_delete_alt"

	# define F_CIB_CLIENTID "cib_clientid"
	# define F_CIB_CALLOPTS "cib_callopt"
	# define F_CIB_CALLID "cib_callid"
	# define F_CIB_CALLDATA "cib_calldata"
	# define F_CIB_OPERATION "cib_op"
	# define F_CIB_ISREPLY "cib_isreplyto"
	# define F_CIB_SECTION "cib_section"
	# define F_CIB_HOST "cib_host"
	# define F_CIB_RC "cib_rc"
	# define F_CIB_DELEGATED "cib_delegated_from"
	# define F_CIB_OBJID "cib_object"
	# define F_CIB_OBJTYPE "cib_object_type"
	# define F_CIB_EXISTING "cib_existing_object"
	# define F_CIB_SEENCOUNT "cib_seen"
	# define F_CIB_TIMEOUT "cib_timeout"
	# define F_CIB_UPDATE "cib_update"
	# define F_CIB_CALLBACK_TOKEN "cib_async_id"
	# define F_CIB_GLOBAL_UPDATE "cib_update"
	# define F_CIB_UPDATE_RESULT "cib_update_result"
	# define F_CIB_CLIENTNAME "cib_clientname"
	# define F_CIB_NOTIFY_TYPE "cib_notify_type"
	# define F_CIB_NOTIFY_ACTIVATE "cib_notify_activate"
	# define F_CIB_UPDATE_DIFF "cib_update_diff"
	# define F_CIB_USER "cib_user"
	# define F_CIB_LOCAL_NOTIFY_ID "cib_local_notify_id"

	# define T_CIB "cib"
	# define T_CIB_NOTIFY "cib_notify"
	/* notify sub-types */
	# define T_CIB_PRE_NOTIFY "cib_pre_notify"
	# define T_CIB_POST_NOTIFY "cib_post_notify"
	# define T_CIB_UPDATE_CONFIRM "cib_update_confirmation"
	# define T_CIB_DIFF_NOTIFY "cib_diff_notify"
	# define T_CIB_REPLACE_NOTIFY "cib_refresh_notify"

	# define cib_channel_ro "cib_ro"
	# define cib_channel_rw "cib_rw"
	# define cib_channel_shm "cib_shm"

	void fix_cib_diff(xmlNode * last, xmlNode * next, xmlNode * local_diff, gboolean changed);
	gboolean cib_diff_version_details(xmlNode * diff, int admin_epoch, int epoch, int *updates,
	int _admin_epoch, int _epoch, int *_updates);

	gboolean startCib(const char *filename);
	int cib_compare_generation(xmlNode * left, xmlNode * right);
	gboolean cib_read_config(GHashTable * options, xmlNode * current_cib);
	void verify_cib_options(GHashTable * options);
	gboolean cib_internal_config_changed(xmlNode * diff);

	extern GHashTable *cib_op_callback_table;
	typedef struct cib_notify_client_s {
	const char *event;
	const char obj_id; / implement one day */
	const char obj_type; / implement one day */
	void (callback) (const char event, xmlNode * msg);

	} cib_notify_client_t;

	typedef struct cib_callback_client_s {
	void (callback) (xmlNode , int, int, xmlNode , void );
	const char *id;
	void *user_data;
	gboolean only_success;
	struct timer_rec_s *timer;

	} cib_callback_client_t;

	struct timer_rec_s {
	int call_id;
	int timeout;
	guint ref;
	cib_t *cib;
	};

	typedef int (cib_op_t) (const char , int, const char , xmlNode ,
	xmlNode , xmlNode , xmlNode , xmlNode );

	cib_t *cib_new_variant(void);

	int cib_perform_op(const char op, int call_options, cib_op_t fn, gboolean is_query,
	const char section, xmlNode req, xmlNode * input,
	gboolean manage_counters, gboolean * config_changed,
	xmlNode * current_cib, xmlNode result_cib, xmlNode diff, xmlNode ** output);

	xmlNode cib_create_op(int call_id, const char token, const char op, const char host,
	const char section, xmlNode data, int call_options,
	const char *user_name);

	void cib_native_callback(cib_t * cib, xmlNode * msg, int call_id, int rc);
	void cib_native_notify(gpointer data, gpointer user_data);
	int cib_native_register_notification(cib_t * cib, const char *callback, int enabled);
	gboolean cib_client_register_callback(cib_t * cib, int call_id, int timeout, gboolean only_success,
	void user_data, const char callback_name,
	void (callback) (xmlNode , int, int, xmlNode , void ));

	gboolean acl_enabled(GHashTable * config_hash);
	gboolean acl_filter_cib(xmlNode * request, xmlNode * current_cib, xmlNode * orig_cib, xmlNode ** filtered_cib);
	gboolean acl_check_diff(xmlNode * request, xmlNode * current_cib, xmlNode * result_cib, xmlNode * diff);

	int cib_process_query(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer);

	int cib_process_erase(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer);

	int cib_process_bump(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer);

	int cib_process_replace(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_create(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer);

	int cib_process_modify(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer);

	int cib_process_delete(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer);

	int cib_process_diff(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer);

	int cib_process_upgrade(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_xpath(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer);

	gboolean cib_config_changed(xmlNode * last, xmlNode * next, xmlNode ** diff);
	gboolean update_results(xmlNode * failed, xmlNode * target, const char *operation,
	int return_code);
	int cib_update_counter(xmlNode * xml_obj, const char *field, gboolean reset);

	+int cib_internal_op(cib_t * cib, const char op, const char host,
	+ const char section, xmlNode data,
	+ xmlNode ** output_data, int call_options, const char *user_name);
	+
	#endif
	diff --git a/lib/cib/cib_attrs.c b/lib/cib/cib_attrs.c
	index 4a703b3776..1edbc02416 100644
	--- a/lib/cib/cib_attrs.c
	+++ b/lib/cib/cib_attrs.c
	@@ -1,545 +1,541 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cib/internal.h>

	#define attr_msg(level, fmt, args...) do { \
	if(to_console) { \
	printf(fmt"\n", ##args); \
	} else { \
	do_crm_log(level, fmt , ##args); \
	} \
	} while(0)

	extern int
	find_nvpair_attr_delegate(cib_t * the_cib, const char attr, const char section,
	const char node_uuid, const char attr_set_type, const char *set_name,
	const char attr_id, const char attr_name, gboolean to_console,
	char *value, const char user_name)
	{
	int offset = 0;
	static int xpath_max = 1024;
	int rc = pcmk_ok;

	char *xpath_string = NULL;
	xmlNode *xml_search = NULL;
	const char *set_type = NULL;
	const char *node_type = NULL;

	if (attr_set_type) {
	set_type = attr_set_type;
	} else {
	set_type = XML_TAG_ATTR_SETS;
	}

	CRM_ASSERT(value != NULL);
	*value = NULL;

	if (safe_str_eq(section, XML_CIB_TAG_CRMCONFIG)) {
	node_uuid = NULL;
	set_type = XML_CIB_TAG_PROPSET;

	} else if (safe_str_eq(section, XML_CIB_TAG_OPCONFIG)
	\|\| safe_str_eq(section, XML_CIB_TAG_RSCCONFIG)) {
	node_uuid = NULL;
	set_type = XML_TAG_META_SETS;

	} else if (safe_str_eq(section, XML_CIB_TAG_TICKETS)) {
	node_uuid = NULL;
	section = XML_CIB_TAG_STATUS;
	node_type = XML_CIB_TAG_TICKETS;

	} else if (node_uuid == NULL) {
	return -EINVAL;
	}

	xpath_string = calloc(1, xpath_max);
	offset += snprintf(xpath_string + offset, xpath_max - offset, "%s", get_object_path(section));

	if (safe_str_eq(node_type, XML_CIB_TAG_TICKETS)) {
	offset += snprintf(xpath_string + offset, xpath_max - offset, "//%s", node_type);

	} else if (node_uuid) {
	const char *node_type = XML_CIB_TAG_NODE;

	if (safe_str_eq(section, XML_CIB_TAG_STATUS)) {
	node_type = XML_CIB_TAG_STATE;
	set_type = XML_TAG_TRANSIENT_NODEATTRS;
	}
	offset +=
	snprintf(xpath_string + offset, xpath_max - offset, "//%s[@id='%s']", node_type,
	node_uuid);
	}

	if (set_name) {
	offset +=
	snprintf(xpath_string + offset, xpath_max - offset, "//%s[@id='%s']", set_type,
	set_name);
	} else {
	offset += snprintf(xpath_string + offset, xpath_max - offset, "//%s", set_type);
	}

	offset += snprintf(xpath_string + offset, xpath_max - offset, "//nvpair[");
	if (attr_id) {
	offset += snprintf(xpath_string + offset, xpath_max - offset, "@id='%s'", attr_id);
	}

	if (attr_name) {
	if (attr_id) {
	offset += snprintf(xpath_string + offset, xpath_max - offset, " and ");
	}
	offset += snprintf(xpath_string + offset, xpath_max - offset, "@name='%s'", attr_name);
	}
	offset += snprintf(xpath_string + offset, xpath_max - offset, "]");

	- rc = the_cib->cmds->delegated_variant_op(the_cib, CIB_OP_QUERY, NULL, xpath_string, NULL,
	- &xml_search,
	- cib_sync_call \| cib_scope_local \| cib_xpath,
	- user_name);
	+ rc = cib_internal_op(the_cib, CIB_OP_QUERY, NULL, xpath_string, NULL, &xml_search,
	+ cib_sync_call \| cib_scope_local \| cib_xpath, user_name);

	if (rc != pcmk_ok) {
	crm_trace("Query failed for attribute %s (section=%s, node=%s, set=%s, xpath=%s): %s",
	attr_name, section, crm_str(node_uuid), crm_str(set_name), xpath_string,
	pcmk_strerror(rc));
	goto done;
	}

	crm_log_xml_debug(xml_search, "Match");
	if (xml_has_children(xml_search)) {
	xmlNode *child = NULL;

	rc = -EINVAL;
	attr_msg(LOG_WARNING, "Multiple attributes match name=%s", attr_name);

	for (child = __xml_first_child(xml_search); child != NULL; child = __xml_next(child)) {
	attr_msg(LOG_INFO, " Value: %s \t(id=%s)",
	crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
	}

	} else {
	const char *tmp = crm_element_value(xml_search, attr);

	if (tmp) {
	*value = strdup(tmp);
	}
	}

	done:
	free(xpath_string);
	free_xml(xml_search);
	return rc;
	}

	int
	update_attr_delegate(cib_t * the_cib, int call_options,
	const char section, const char node_uuid, const char *set_type,
	const char set_name, const char attr_id, const char *attr_name,
	const char attr_value, gboolean to_console, const char user_name)
	{
	const char *tag = NULL;
	int rc = pcmk_ok;
	xmlNode *xml_top = NULL;
	xmlNode *xml_obj = NULL;

	char *local_attr_id = NULL;
	char *local_set_name = NULL;
	gboolean use_attributes_tag = FALSE;

	CRM_CHECK(section != NULL, return -EINVAL);
	CRM_CHECK(attr_value != NULL, return -EINVAL);
	CRM_CHECK(attr_name != NULL \|\| attr_id != NULL, return -EINVAL);

	rc = find_nvpair_attr_delegate(the_cib, XML_ATTR_ID, section, node_uuid, set_type, set_name,
	attr_id, attr_name, FALSE, &local_attr_id, user_name);
	if (rc == pcmk_ok) {
	attr_id = local_attr_id;
	goto do_modify;

	} else if (rc != -ENXIO) {
	return rc;

	/* } else if(attr_id == NULL) { */
	/* return -EINVAL; */

	} else {
	const char *value = NULL;
	const char *node_type = NULL;
	xmlNode *cib_top = NULL;

	- rc = the_cib->cmds->delegated_variant_op(the_cib, CIB_OP_QUERY, NULL, "/cib", NULL,
	- &cib_top,
	- cib_sync_call \| cib_scope_local \| cib_xpath \|
	- cib_no_children, user_name);
	+ rc = cib_internal_op(the_cib, CIB_OP_QUERY, NULL, "/cib", NULL, &cib_top,
	+ cib_sync_call \| cib_scope_local \| cib_xpath \| cib_no_children, user_name);

	value = crm_element_value(cib_top, "ignore_dtd");
	if (value != NULL) {
	use_attributes_tag = TRUE;

	} else {
	value = crm_element_value(cib_top, XML_ATTR_VALIDATION);
	if (value && strstr(value, "-0.6")) {
	use_attributes_tag = TRUE;
	}
	}
	free_xml(cib_top);

	if (safe_str_eq(section, XML_CIB_TAG_TICKETS)) {
	node_uuid = NULL;
	section = XML_CIB_TAG_STATUS;
	node_type = XML_CIB_TAG_TICKETS;

	xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_STATUS);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_TICKETS);

	} else if (safe_str_eq(section, XML_CIB_TAG_NODES)) {
	tag = XML_CIB_TAG_NODE;
	if (node_uuid == NULL) {
	return -EINVAL;
	}

	} else if (safe_str_eq(section, XML_CIB_TAG_STATUS)) {
	tag = XML_TAG_TRANSIENT_NODEATTRS;
	if (node_uuid == NULL) {
	return -EINVAL;
	}

	xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_STATE);
	crm_xml_add(xml_obj, XML_ATTR_ID, node_uuid);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	} else {
	tag = section;
	node_uuid = NULL;
	}

	if (set_name == NULL) {
	if (safe_str_eq(section, XML_CIB_TAG_CRMCONFIG)) {
	local_set_name = strdup(CIB_OPTIONS_FIRST);

	} else if (safe_str_eq(node_type, XML_CIB_TAG_TICKETS)) {
	local_set_name = crm_concat(section, XML_CIB_TAG_TICKETS, '-');

	} else if (node_uuid) {
	local_set_name = crm_concat(section, node_uuid, '-');

	if (set_type) {
	char *tmp_set_name = local_set_name;

	local_set_name = crm_concat(tmp_set_name, set_type, '-');
	free(tmp_set_name);
	}
	} else {
	local_set_name = crm_concat(section, "options", '-');
	}
	set_name = local_set_name;
	}

	if (attr_id == NULL) {
	int lpc = 0;

	local_attr_id = crm_concat(set_name, attr_name, '-');
	attr_id = local_attr_id;

	/* Minimal attempt at sanitizing automatic IDs */
	for (lpc = 0; local_attr_id[lpc] != 0; lpc++) {
	switch (local_attr_id[lpc]) {
	case ':':
	local_attr_id[lpc] = '.';
	}
	}

	} else if (attr_name == NULL) {
	attr_name = attr_id;
	}

	crm_trace("Creating %s/%s", section, tag);
	if (tag != NULL) {
	xml_obj = create_xml_node(xml_obj, tag);
	crm_xml_add(xml_obj, XML_ATTR_ID, node_uuid);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}
	}

	if (node_uuid == NULL && safe_str_neq(node_type, XML_CIB_TAG_TICKETS)) {
	if (safe_str_eq(section, XML_CIB_TAG_CRMCONFIG)) {
	xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_PROPSET);
	} else {
	xml_obj = create_xml_node(xml_obj, XML_TAG_META_SETS);
	}

	} else if (set_type) {
	xml_obj = create_xml_node(xml_obj, set_type);

	} else {
	xml_obj = create_xml_node(xml_obj, XML_TAG_ATTR_SETS);
	}
	crm_xml_add(xml_obj, XML_ATTR_ID, set_name);

	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	if (use_attributes_tag) {
	xml_obj = create_xml_node(xml_obj, XML_TAG_ATTRS);
	}
	}

	do_modify:
	xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_NVPAIR);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	crm_xml_add(xml_obj, XML_ATTR_ID, attr_id);
	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, attr_name);
	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, attr_value);

	crm_log_xml_trace(xml_top, "update_attr");
	- rc = the_cib->cmds->delegated_variant_op(the_cib, CIB_OP_MODIFY, NULL, section, xml_top, NULL,
	- call_options \| cib_quorum_override, user_name);
	+ rc = cib_internal_op(the_cib, CIB_OP_MODIFY, NULL, section, xml_top, NULL,
	+ call_options \| cib_quorum_override, user_name);

	if (rc < pcmk_ok) {
	attr_msg(LOG_ERR, "Error setting %s=%s (section=%s, set=%s): %s",
	attr_name, attr_value, section, crm_str(set_name), pcmk_strerror(rc));
	crm_log_xml_info(xml_top, "Update");
	}

	free(local_set_name);
	free(local_attr_id);
	free_xml(xml_top);

	return rc;
	}

	int
	read_attr_delegate(cib_t * the_cib,
	const char section, const char node_uuid, const char *set_type,
	const char set_name, const char attr_id, const char *attr_name,
	char *attr_value, gboolean to_console, const char user_name)
	{
	int rc = pcmk_ok;

	CRM_ASSERT(attr_value != NULL);
	CRM_CHECK(section != NULL, return -EINVAL);
	CRM_CHECK(attr_name != NULL \|\| attr_id != NULL, return -EINVAL);

	*attr_value = NULL;

	rc = find_nvpair_attr_delegate(the_cib, XML_NVPAIR_ATTR_VALUE, section, node_uuid, set_type,
	set_name, attr_id, attr_name, to_console, attr_value, user_name);
	if (rc != pcmk_ok) {
	crm_trace("Query failed for attribute %s (section=%s, node=%s, set=%s): %s",
	attr_name, section, crm_str(set_name), crm_str(node_uuid), pcmk_strerror(rc));
	}
	return rc;
	}

	int
	delete_attr_delegate(cib_t * the_cib, int options,
	const char section, const char node_uuid, const char *set_type,
	const char set_name, const char attr_id, const char *attr_name,
	const char attr_value, gboolean to_console, const char user_name)
	{
	int rc = pcmk_ok;
	xmlNode *xml_obj = NULL;
	char *local_attr_id = NULL;

	CRM_CHECK(section != NULL, return -EINVAL);
	CRM_CHECK(attr_name != NULL \|\| attr_id != NULL, return -EINVAL);

	if (attr_id == NULL) {
	rc = find_nvpair_attr_delegate(the_cib, XML_ATTR_ID, section, node_uuid, set_type,
	set_name, attr_id, attr_name, to_console, &local_attr_id,
	user_name);
	if (rc != pcmk_ok) {
	return rc;
	}
	attr_id = local_attr_id;
	}

	xml_obj = create_xml_node(NULL, XML_CIB_TAG_NVPAIR);
	crm_xml_add(xml_obj, XML_ATTR_ID, attr_id);
	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, attr_name);
	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, attr_value);

	- rc = the_cib->cmds->delegated_variant_op(the_cib, CIB_OP_DELETE, NULL, section, xml_obj, NULL,
	- options \| cib_quorum_override, user_name);
	+ rc = cib_internal_op(the_cib, CIB_OP_DELETE, NULL, section, xml_obj, NULL,
	+ options \| cib_quorum_override, user_name);

	if (rc == pcmk_ok) {
	attr_msg(LOG_DEBUG, "Deleted %s %s: id=%s%s%s%s%s\n",
	section, node_uuid ? "attribute" : "option", local_attr_id,
	set_name ? " set=" : "", set_name ? set_name : "",
	attr_name ? " name=" : "", attr_name ? attr_name : "");
	}

	free(local_attr_id);
	free_xml(xml_obj);
	return rc;
	}

	int
	query_node_uuid(cib_t * the_cib, const char uname, char *uuid)
	{
	int rc = pcmk_ok;
	xmlNode *a_child = NULL;
	xmlNode *xml_obj = NULL;
	xmlNode *fragment = NULL;
	const char *child_name = NULL;

	CRM_ASSERT(uname != NULL);
	CRM_ASSERT(uuid != NULL);

	rc = the_cib->cmds->query(the_cib, XML_CIB_TAG_NODES, &fragment,
	cib_sync_call \| cib_scope_local);
	if (rc != pcmk_ok) {
	return rc;
	}

	xml_obj = fragment;
	CRM_CHECK(safe_str_eq(crm_element_name(xml_obj), XML_CIB_TAG_NODES), return -ENOMSG);
	CRM_ASSERT(xml_obj != NULL);
	crm_log_xml_debug(xml_obj, "Result section");

	rc = -ENXIO;
	*uuid = NULL;

	for (a_child = __xml_first_child(xml_obj); a_child != NULL; a_child = __xml_next(a_child)) {
	if (crm_str_eq((const char *)a_child->name, XML_CIB_TAG_NODE, TRUE)) {
	child_name = crm_element_value(a_child, XML_ATTR_UNAME);
	if (safe_str_eq(uname, child_name)) {
	child_name = ID(a_child);
	if (child_name != NULL) {
	*uuid = strdup(child_name);
	rc = pcmk_ok;
	}
	break;
	}
	}
	}
	free_xml(fragment);
	return rc;
	}

	int
	query_node_uname(cib_t * the_cib, const char uuid, char *uname)
	{
	int rc = pcmk_ok;
	xmlNode *a_child = NULL;
	xmlNode *xml_obj = NULL;
	xmlNode *fragment = NULL;
	const char *child_name = NULL;

	CRM_ASSERT(uname != NULL);
	CRM_ASSERT(uuid != NULL);

	rc = the_cib->cmds->query(the_cib, XML_CIB_TAG_NODES, &fragment,
	cib_sync_call \| cib_scope_local);
	if (rc != pcmk_ok) {
	return rc;
	}

	xml_obj = fragment;
	CRM_CHECK(safe_str_eq(crm_element_name(xml_obj), XML_CIB_TAG_NODES), return -ENOMSG);
	CRM_ASSERT(xml_obj != NULL);
	crm_log_xml_trace(xml_obj, "Result section");

	rc = -ENXIO;
	*uname = NULL;

	for (a_child = __xml_first_child(xml_obj); a_child != NULL; a_child = __xml_next(a_child)) {
	if (crm_str_eq((const char *)a_child->name, XML_CIB_TAG_NODE, TRUE)) {
	child_name = ID(a_child);
	if (safe_str_eq(uuid, child_name)) {
	child_name = crm_element_value(a_child, XML_ATTR_UNAME);
	if (child_name != NULL) {
	*uname = strdup(child_name);
	rc = pcmk_ok;
	}
	break;
	}
	}
	}

	free_xml(fragment);
	return rc;
	}

	int
	set_standby(cib_t * the_cib, const char uuid, const char scope, const char *standby_value)
	{
	int rc = pcmk_ok;
	int str_length = 3;
	char *attr_id = NULL;
	char *set_name = NULL;
	const char *attr_name = "standby";

	CRM_CHECK(standby_value != NULL, return -EINVAL);
	if (scope == NULL) {
	scope = XML_CIB_TAG_NODES;
	}

	CRM_CHECK(uuid != NULL, return -EINVAL);
	str_length += strlen(attr_name);
	str_length += strlen(uuid);

	if (safe_str_eq(scope, "reboot") \|\| safe_str_eq(scope, XML_CIB_TAG_STATUS)) {
	const char *extra = "transient";

	scope = XML_CIB_TAG_STATUS;

	str_length += strlen(extra);
	attr_id = calloc(1, str_length);
	sprintf(attr_id, "%s-%s-%s", extra, attr_name, uuid);

	} else {
	scope = XML_CIB_TAG_NODES;
	attr_id = calloc(1, str_length);
	sprintf(attr_id, "%s-%s", attr_name, uuid);
	}

	rc = update_attr_delegate(the_cib, cib_sync_call, scope, uuid, NULL, set_name,
	attr_id, attr_name, standby_value, TRUE, NULL);

	free(attr_id);
	free(set_name);
	return rc;
	}
	diff --git a/lib/cib/cib_client.c b/lib/cib/cib_client.c
	index 1ed0c567d3..6c63bcdc14 100644
	--- a/lib/cib/cib_client.c
	+++ b/lib/cib/cib_client.c
	@@ -1,626 +1,626 @@
	/*
	* Copyright (c) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/
	#include <crm_internal.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <pwd.h>

	#include <sys/stat.h>
	#include <sys/types.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	GHashTable *cib_op_callback_table = NULL;

	int cib_client_set_op_callback(cib_t * cib, void (callback) (const xmlNode msg, int call_id,
	int rc, xmlNode * output));

	int cib_client_add_notify_callback(cib_t * cib, const char *event,
	void (callback) (const char event, xmlNode * msg));

	int cib_client_del_notify_callback(cib_t * cib, const char *event,
	void (callback) (const char event, xmlNode * msg));

	gint ciblib_GCompareFunc(gconstpointer a, gconstpointer b);

	#define op_common(cib) do { \
	if(cib == NULL) { \
	return -EINVAL; \
	- } else if(cib->cmds->variant_op == NULL) { \
	- return -EPROTONOSUPPORT; \
	+ } else if(cib->delegate_fn == NULL) { \
	+ return -EPROTONOSUPPORT; \
	} \
	} while(0)

	static int
	cib_client_noop(cib_t * cib, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CRM_OP_NOOP, NULL, NULL, NULL, NULL, call_options);
	+ return cib_internal_op(cib, CRM_OP_NOOP, NULL, NULL, NULL, NULL, call_options, NULL);
	}

	static int
	cib_client_ping(cib_t * cib, xmlNode ** output_data, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CRM_OP_PING, NULL, NULL, NULL, output_data, call_options);
	+ return cib_internal_op(cib, CRM_OP_PING, NULL, NULL, NULL, output_data, call_options, NULL);
	}

	static int
	cib_client_query(cib_t * cib, const char section, xmlNode * output_data, int call_options)
	{
	return cib->cmds->query_from(cib, NULL, section, output_data, call_options);
	}

	static int
	cib_client_query_from(cib_t * cib, const char host, const char section,
	xmlNode ** output_data, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CIB_OP_QUERY, host, section, NULL, output_data, call_options);
	+ return cib_internal_op(cib, CIB_OP_QUERY, host, section, NULL, output_data, call_options, NULL);
	}

	static int
	cib_client_is_master(cib_t * cib)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CIB_OP_ISMASTER, NULL, NULL, NULL, NULL,
	- cib_scope_local \| cib_sync_call);
	+ return cib_internal_op(cib, CIB_OP_ISMASTER, NULL, NULL, NULL, NULL,
	+ cib_scope_local \| cib_sync_call, NULL);
	}

	static int
	cib_client_set_slave(cib_t * cib, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CIB_OP_SLAVE, NULL, NULL, NULL, NULL, call_options);
	+ return cib_internal_op(cib, CIB_OP_SLAVE, NULL, NULL, NULL, NULL, call_options, NULL);
	}

	static int
	cib_client_set_slave_all(cib_t * cib, int call_options)
	{
	return -EPROTONOSUPPORT;
	}

	static int
	cib_client_set_master(cib_t * cib, int call_options)
	{
	op_common(cib);
	crm_trace("Adding cib_scope_local to options");
	- return cib->cmds->variant_op(cib, CIB_OP_MASTER, NULL, NULL, NULL, NULL,
	- call_options \| cib_scope_local);
	+ return cib_internal_op(cib, CIB_OP_MASTER, NULL, NULL, NULL, NULL,
	+ call_options \| cib_scope_local, NULL);
	}

	static int
	cib_client_bump_epoch(cib_t * cib, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CIB_OP_BUMP, NULL, NULL, NULL, NULL, call_options);
	+ return cib_internal_op(cib, CIB_OP_BUMP, NULL, NULL, NULL, NULL, call_options, NULL);
	}

	static int
	cib_client_upgrade(cib_t * cib, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CIB_OP_UPGRADE, NULL, NULL, NULL, NULL, call_options);
	+ return cib_internal_op(cib, CIB_OP_UPGRADE, NULL, NULL, NULL, NULL, call_options, NULL);
	}

	static int
	cib_client_sync(cib_t * cib, const char *section, int call_options)
	{
	return cib->cmds->sync_from(cib, NULL, section, call_options);
	}

	static int
	cib_client_sync_from(cib_t * cib, const char host, const char section, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CIB_OP_SYNC, host, section, NULL, NULL, call_options);
	+ return cib_internal_op(cib, CIB_OP_SYNC, host, section, NULL, NULL, call_options, NULL);
	}

	static int
	cib_client_create(cib_t * cib, const char section, xmlNode data, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CIB_OP_CREATE, NULL, section, data, NULL, call_options);
	+ return cib_internal_op(cib, CIB_OP_CREATE, NULL, section, data, NULL, call_options, NULL);
	}

	static int
	cib_client_modify(cib_t * cib, const char section, xmlNode data, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CIB_OP_MODIFY, NULL, section, data, NULL, call_options);
	+ return cib_internal_op(cib, CIB_OP_MODIFY, NULL, section, data, NULL, call_options, NULL);
	}

	static int
	cib_client_update(cib_t * cib, const char section, xmlNode data, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CIB_OP_MODIFY, NULL, section, data, NULL, call_options);
	+ return cib_internal_op(cib, CIB_OP_MODIFY, NULL, section, data, NULL, call_options, NULL);
	}

	static int
	cib_client_replace(cib_t * cib, const char section, xmlNode data, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CIB_OP_REPLACE, NULL, section, data, NULL, call_options);
	+ return cib_internal_op(cib, CIB_OP_REPLACE, NULL, section, data, NULL, call_options, NULL);
	}

	static int
	cib_client_delete(cib_t * cib, const char section, xmlNode data, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CIB_OP_DELETE, NULL, section, data, NULL, call_options);
	+ return cib_internal_op(cib, CIB_OP_DELETE, NULL, section, data, NULL, call_options, NULL);
	}

	static int
	cib_client_delete_absolute(cib_t * cib, const char section, xmlNode data, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CIB_OP_DELETE_ALT, NULL, section, data, NULL, call_options);
	+ return cib_internal_op(cib, CIB_OP_DELETE_ALT, NULL, section, data, NULL, call_options, NULL);
	}

	static int
	cib_client_erase(cib_t * cib, xmlNode ** output_data, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CIB_OP_ERASE, NULL, NULL, NULL, output_data, call_options);
	+ return cib_internal_op(cib, CIB_OP_ERASE, NULL, NULL, NULL, output_data, call_options, NULL);
	}

	static int
	cib_client_quit(cib_t * cib, int call_options)
	{
	op_common(cib);
	- return cib->cmds->variant_op(cib, CRM_OP_QUIT, NULL, NULL, NULL, NULL, call_options);
	+ return cib_internal_op(cib, CRM_OP_QUIT, NULL, NULL, NULL, NULL, call_options, NULL);
	}

	static void
	cib_destroy_op_callback(gpointer data)
	{
	cib_callback_client_t *blob = data;

	if (blob->timer && blob->timer->ref > 0) {
	g_source_remove(blob->timer->ref);
	}
	free(blob->timer);
	free(blob);
	}

	char *
	get_shadow_file(const char *suffix)
	{
	char *cib_home = NULL;
	char *fullname = NULL;
	char *name = crm_concat("shadow", suffix, '.');
	const char *dir = getenv("CIB_shadow_dir");

	if (dir == NULL) {
	uid_t uid = geteuid();
	struct passwd *pwent = getpwuid(uid);
	const char *user = NULL;

	if (pwent) {
	user = pwent->pw_name;
	} else {
	crm_perror(LOG_ERR, "Cannot get password entry for uid: %d", uid);
	user = getenv("USER");
	}

	if (safe_str_eq(user, "root") \|\| safe_str_eq(user, CRM_DAEMON_USER)) {
	dir = CRM_CONFIG_DIR;

	} else {
	const char *home = NULL;

	if ((home = getenv("HOME")) == NULL) {
	if (pwent) {
	home = pwent->pw_dir;
	}
	}

	if ((dir = getenv("TMPDIR")) == NULL) {
	dir = "/tmp";
	}
	if (home && home[0] == '/') {
	int rc = 0;

	cib_home = crm_concat(home, ".cib", '/');

	rc = mkdir(cib_home, 0700);
	if (rc < 0 && errno != EEXIST) {
	crm_perror(LOG_ERR, "Couldn't create user-specific shadow directory: %s",
	cib_home);
	errno = 0;

	} else {
	dir = cib_home;
	}
	}
	}
	}

	fullname = crm_concat(dir, name, '/');
	free(cib_home);
	free(name);

	return fullname;
	}

	cib_t *
	cib_shadow_new(const char *shadow)
	{
	cib_t *new_cib = NULL;
	char *shadow_file = NULL;

	CRM_CHECK(shadow != NULL, return NULL);

	shadow_file = get_shadow_file(shadow);
	new_cib = cib_file_new(shadow_file);
	free(shadow_file);

	return new_cib;
	}

	cib_t *
	cib_new_no_shadow(void)
	{
	unsetenv("CIB_shadow");
	return cib_new();
	}

	cib_t *
	cib_new(void)
	{
	const char *value = getenv("CIB_shadow");

	if (value) {
	return cib_shadow_new(value);
	}

	value = getenv("CIB_file");
	if (value) {
	return cib_file_new(value);
	}

	value = getenv("CIB_port");
	if (value) {
	gboolean encrypted = TRUE;
	int port = crm_parse_int(value, NULL);
	const char *server = getenv("CIB_server");
	const char *user = getenv("CIB_user");
	const char *pass = getenv("CIB_passwd");

	value = getenv("CIB_encrypted");
	if (value && crm_is_true(value) == FALSE) {
	crm_info("Disabling TLS");
	encrypted = FALSE;
	}

	if (user == NULL) {
	user = CRM_DAEMON_USER;
	crm_info("Defaulting to user: %s", user);
	}

	if (server == NULL) {
	server = "localhost";
	crm_info("Defaulting to localhost");
	}

	return cib_remote_new(server, user, pass, port, encrypted);
	}

	return cib_native_new();
	}

	/* this is backwards...
	cib_*_new should call this not the other way around
	*/

	cib_t *
	cib_new_variant(void)
	{
	cib_t *new_cib = NULL;

	new_cib = calloc(1, sizeof(cib_t));

	if (cib_op_callback_table != NULL) {
	g_hash_table_destroy(cib_op_callback_table);
	cib_op_callback_table = NULL;
	}
	if (cib_op_callback_table == NULL) {
	cib_op_callback_table = g_hash_table_new_full(g_direct_hash, g_direct_equal,
	NULL, cib_destroy_op_callback);
	}

	new_cib->call_id = 1;
	new_cib->variant = cib_undefined;

	new_cib->type = cib_none;
	new_cib->state = cib_disconnected;

	new_cib->op_callback = NULL;
	new_cib->variant_opaque = NULL;
	new_cib->notify_list = NULL;

	/* the rest will get filled in by the variant constructor */
	new_cib->cmds = calloc(1, sizeof(cib_api_operations_t));

	new_cib->cmds->set_op_callback = cib_client_set_op_callback;
	new_cib->cmds->add_notify_callback = cib_client_add_notify_callback;
	new_cib->cmds->del_notify_callback = cib_client_del_notify_callback;
	new_cib->cmds->register_callback = cib_client_register_callback;

	new_cib->cmds->noop = cib_client_noop;
	new_cib->cmds->ping = cib_client_ping;
	new_cib->cmds->query = cib_client_query;
	new_cib->cmds->sync = cib_client_sync;

	new_cib->cmds->query_from = cib_client_query_from;
	new_cib->cmds->sync_from = cib_client_sync_from;

	new_cib->cmds->is_master = cib_client_is_master;
	new_cib->cmds->set_master = cib_client_set_master;
	new_cib->cmds->set_slave = cib_client_set_slave;
	new_cib->cmds->set_slave_all = cib_client_set_slave_all;

	new_cib->cmds->upgrade = cib_client_upgrade;
	new_cib->cmds->bump_epoch = cib_client_bump_epoch;

	new_cib->cmds->create = cib_client_create;
	new_cib->cmds->modify = cib_client_modify;
	new_cib->cmds->update = cib_client_update;
	new_cib->cmds->replace = cib_client_replace;
	new_cib->cmds->delete = cib_client_delete;
	new_cib->cmds->erase = cib_client_erase;
	new_cib->cmds->quit = cib_client_quit;

	new_cib->cmds->delete_absolute = cib_client_delete_absolute;

	return new_cib;
	}

	void
	cib_delete(cib_t * cib)
	{
	GList *list = cib->notify_list;

	while (list != NULL) {
	cib_notify_client_t *client = g_list_nth_data(list, 0);

	list = g_list_remove(list, client);
	free(client);
	}

	g_hash_table_destroy(cib_op_callback_table);
	cib_op_callback_table = NULL;
	cib->cmds->free(cib);
	cib = NULL;
	}

	int
	cib_client_set_op_callback(cib_t * cib, void (callback) (const xmlNode msg, int call_id,
	int rc, xmlNode * output))
	{
	if (callback == NULL) {
	crm_info("Un-Setting operation callback");

	} else {
	crm_trace("Setting operation callback");
	}
	cib->op_callback = callback;
	return pcmk_ok;
	}

	int
	cib_client_add_notify_callback(cib_t * cib, const char *event,
	void (callback) (const char event, xmlNode * msg))
	{
	GList *list_item = NULL;
	cib_notify_client_t *new_client = NULL;

	if (cib->variant != cib_native && cib->variant != cib_remote) {
	return -EPROTONOSUPPORT;
	}

	crm_trace("Adding callback for %s events (%d)", event, g_list_length(cib->notify_list));

	new_client = calloc(1, sizeof(cib_notify_client_t));
	new_client->event = event;
	new_client->callback = callback;

	list_item = g_list_find_custom(cib->notify_list, new_client, ciblib_GCompareFunc);

	if (list_item != NULL) {
	crm_warn("Callback already present");
	free(new_client);
	return -ENOTUNIQ;

	} else {
	cib->notify_list = g_list_append(cib->notify_list, new_client);

	cib->cmds->register_notification(cib, event, 1);

	crm_trace("Callback added (%d)", g_list_length(cib->notify_list));
	}
	return pcmk_ok;
	}

	int
	cib_client_del_notify_callback(cib_t * cib, const char *event,
	void (callback) (const char event, xmlNode * msg))
	{
	GList *list_item = NULL;
	cib_notify_client_t *new_client = NULL;

	if (cib->variant != cib_native && cib->variant != cib_remote) {
	return -EPROTONOSUPPORT;
	}

	crm_debug("Removing callback for %s events", event);

	new_client = calloc(1, sizeof(cib_notify_client_t));
	new_client->event = event;
	new_client->callback = callback;

	list_item = g_list_find_custom(cib->notify_list, new_client, ciblib_GCompareFunc);

	cib->cmds->register_notification(cib, event, 0);

	if (list_item != NULL) {
	cib_notify_client_t *list_client = list_item->data;

	cib->notify_list = g_list_remove(cib->notify_list, list_client);
	free(list_client);

	crm_trace("Removed callback");

	} else {
	crm_trace("Callback not present");
	}
	free(new_client);
	return pcmk_ok;
	}

	gint
	ciblib_GCompareFunc(gconstpointer a, gconstpointer b)
	{
	int rc = 0;
	const cib_notify_client_t *a_client = a;
	const cib_notify_client_t *b_client = b;

	CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0);
	rc = strcmp(a_client->event, b_client->event);
	if (rc == 0) {
	if (a_client->callback == b_client->callback) {
	return 0;
	} else if (((long)a_client->callback) < ((long)b_client->callback)) {
	crm_trace("callbacks for %s are not equal: %p < %p",
	a_client->event, a_client->callback, b_client->callback);
	return -1;
	}
	crm_trace("callbacks for %s are not equal: %p > %p",
	a_client->event, a_client->callback, b_client->callback);
	return 1;
	}
	return rc;
	}

	static gboolean
	cib_async_timeout_handler(gpointer data)
	{
	struct timer_rec_s *timer = data;

	crm_debug("Async call %d timed out after %ds", timer->call_id, timer->timeout);
	cib_native_callback(timer->cib, NULL, timer->call_id, -ETIME);

	/* Always return TRUE, never remove the handler
	* We do that in remove_cib_op_callback()
	*/
	return TRUE;
	}

	gboolean
	cib_client_register_callback(cib_t * cib, int call_id, int timeout, gboolean only_success,
	void user_data, const char callback_name,
	void (callback) (xmlNode , int, int, xmlNode , void ))
	{
	cib_callback_client_t *blob = NULL;

	if (call_id < 0) {
	if (only_success == FALSE) {
	callback(NULL, call_id, call_id, NULL, user_data);
	} else {
	crm_warn("CIB call failed: %s", pcmk_strerror(call_id));
	}
	return FALSE;
	}

	blob = calloc(1, sizeof(cib_callback_client_t));
	blob->id = callback_name;
	blob->only_success = only_success;
	blob->user_data = user_data;
	blob->callback = callback;

	if (timeout > 0) {
	struct timer_rec_s *async_timer = NULL;

	async_timer = calloc(1, sizeof(struct timer_rec_s));
	blob->timer = async_timer;

	async_timer->cib = cib;
	async_timer->call_id = call_id;
	async_timer->timeout = timeout * 1000;
	async_timer->ref =
	g_timeout_add(async_timer->timeout, cib_async_timeout_handler, async_timer);
	}

	crm_trace("Adding callback %s for call %d", callback_name, call_id);
	g_hash_table_insert(cib_op_callback_table, GINT_TO_POINTER(call_id), blob);

	return TRUE;
	}

	void
	remove_cib_op_callback(int call_id, gboolean all_callbacks)
	{
	if (all_callbacks) {
	if (cib_op_callback_table != NULL) {
	g_hash_table_destroy(cib_op_callback_table);
	}

	cib_op_callback_table = g_hash_table_new_full(g_direct_hash, g_direct_equal,
	NULL, cib_destroy_op_callback);

	} else {
	g_hash_table_remove(cib_op_callback_table, GINT_TO_POINTER(call_id));
	}
	}

	int
	num_cib_op_callbacks(void)
	{
	if (cib_op_callback_table == NULL) {
	return 0;
	}
	return g_hash_table_size(cib_op_callback_table);
	}

	static void
	cib_dump_pending_op(gpointer key, gpointer value, gpointer user_data)
	{
	int call = GPOINTER_TO_INT(key);
	cib_callback_client_t *blob = value;

	crm_debug("Call %d (%s): pending", call, crm_str(blob->id));
	}

	void
	cib_dump_pending_callbacks(void)
	{
	if (cib_op_callback_table == NULL) {
	return;
	}
	return g_hash_table_foreach(cib_op_callback_table, cib_dump_pending_op, NULL);
	}
	diff --git a/lib/cib/cib_file.c b/lib/cib/cib_file.c
	index 71f74319be..095edcbd8b 100644
	--- a/lib/cib/cib_file.c
	+++ b/lib/cib/cib_file.c
	@@ -1,328 +1,327 @@
	/*
	* Copyright (c) 2004 International Business Machines
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/
	#include <crm_internal.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>

	#include <sys/stat.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>

	typedef struct cib_file_opaque_s {
	int flags;
	char *filename;

	} cib_file_opaque_t;

	int cib_file_perform_op(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options);

	int cib_file_perform_op_delegate(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options,
	const char *user_name);

	int cib_file_signon(cib_t * cib, const char *name, enum cib_conn_type type);
	int cib_file_signoff(cib_t * cib);
	int cib_file_free(cib_t * cib);

	static int
	cib_file_inputfd(cib_t * cib)
	{
	return -EPROTONOSUPPORT;
	}

	static int
	cib_file_set_connection_dnotify(cib_t * cib, void (*dnotify) (gpointer user_data))
	{
	return -EPROTONOSUPPORT;
	}

	static int
	cib_file_register_notification(cib_t * cib, const char *callback, int enabled)
	{
	return -EPROTONOSUPPORT;
	}

	cib_t *
	cib_file_new(const char *cib_location)
	{
	cib_file_opaque_t *private = NULL;
	cib_t *cib = cib_new_variant();

	private = calloc(1, sizeof(cib_file_opaque_t));

	cib->variant = cib_file;
	cib->variant_opaque = private;

	if (cib_location == NULL) {
	cib_location = getenv("CIB_file");
	}
	private->filename = strdup(cib_location);

	/* assign variant specific ops */
	- cib->cmds->variant_op = cib_file_perform_op;
	- cib->cmds->delegated_variant_op = cib_file_perform_op_delegate;
	+ cib->delegate_fn = cib_file_perform_op_delegate;
	cib->cmds->signon = cib_file_signon;
	cib->cmds->signoff = cib_file_signoff;
	cib->cmds->free = cib_file_free;
	cib->cmds->inputfd = cib_file_inputfd;

	cib->cmds->register_notification = cib_file_register_notification;
	cib->cmds->set_connection_dnotify = cib_file_set_connection_dnotify;

	return cib;
	}

	static xmlNode *in_mem_cib = NULL;
	static int
	load_file_cib(const char *filename)
	{
	int rc = pcmk_ok;
	struct stat buf;
	xmlNode *root = NULL;
	gboolean dtd_ok = TRUE;
	const char *ignore_dtd = NULL;
	xmlNode *status = NULL;

	rc = stat(filename, &buf);
	if (rc == 0) {
	root = filename2xml(filename);
	if (root == NULL) {
	return -pcmk_err_dtd_validation;
	}

	} else {
	return -ENXIO;
	}

	rc = 0;

	status = find_xml_node(root, XML_CIB_TAG_STATUS, FALSE);
	if (status == NULL) {
	create_xml_node(root, XML_CIB_TAG_STATUS);
	}

	ignore_dtd = crm_element_value(root, XML_ATTR_VALIDATION);
	dtd_ok = validate_xml(root, NULL, TRUE);
	if (dtd_ok == FALSE) {
	crm_err("CIB does not validate against %s", ignore_dtd);
	rc = -pcmk_err_dtd_validation;
	goto bail;
	}

	in_mem_cib = root;
	return rc;

	bail:
	free_xml(root);
	root = NULL;
	return rc;
	}

	int
	cib_file_signon(cib_t * cib, const char *name, enum cib_conn_type type)
	{
	int rc = pcmk_ok;
	cib_file_opaque_t *private = cib->variant_opaque;

	if (private->filename == FALSE) {
	rc = -EINVAL;
	} else {
	rc = load_file_cib(private->filename);
	}

	if (rc == pcmk_ok) {
	crm_debug("%s: Opened connection to local file '%s'", name, private->filename);
	cib->state = cib_connected_command;
	cib->type = cib_command;

	} else {
	fprintf(stderr, "%s: Connection to local file '%s' failed: %s\n",
	name, private->filename, pcmk_strerror(rc));
	}

	return rc;
	}

	int
	cib_file_signoff(cib_t * cib)
	{
	int rc = pcmk_ok;
	cib_file_opaque_t *private = cib->variant_opaque;

	crm_debug("Signing out of the CIB Service");

	if (strstr(private->filename, ".bz2") != NULL) {
	rc = write_xml_file(in_mem_cib, private->filename, TRUE);

	} else {
	rc = write_xml_file(in_mem_cib, private->filename, FALSE);
	}

	if (rc > 0) {
	crm_info("Wrote CIB to %s", private->filename);
	rc = pcmk_ok;

	} else {
	crm_err("Could not write CIB to %s", private->filename);
	}
	free_xml(in_mem_cib);

	cib->state = cib_disconnected;
	cib->type = cib_none;

	return rc;
	}

	int
	cib_file_free(cib_t * cib)
	{
	int rc = pcmk_ok;

	if (cib->state != cib_disconnected) {
	rc = cib_file_signoff(cib);
	}

	if (rc == pcmk_ok) {
	cib_file_opaque_t *private = cib->variant_opaque;

	free(private->filename);
	free(cib->cmds);
	free(private);
	free(cib);

	} else {
	fprintf(stderr, "Couldn't sign off: %d\n", rc);
	}

	return rc;
	}

	struct cib_func_entry {
	const char *op;
	gboolean read_only;
	cib_op_t fn;
	};

	/* INDENT-OFF */
	static struct cib_func_entry cib_file_ops[] = {
	{CIB_OP_QUERY, TRUE, cib_process_query},
	{CIB_OP_MODIFY, FALSE, cib_process_modify},
	{CIB_OP_APPLY_DIFF, FALSE, cib_process_diff},
	{CIB_OP_BUMP, FALSE, cib_process_bump},
	{CIB_OP_REPLACE, FALSE, cib_process_replace},
	{CIB_OP_CREATE, FALSE, cib_process_create},
	{CIB_OP_DELETE, FALSE, cib_process_delete},
	{CIB_OP_ERASE, FALSE, cib_process_erase},
	{CIB_OP_UPGRADE, FALSE, cib_process_upgrade},
	};
	/* INDENT-ON */

	int
	cib_file_perform_op(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options)
	{
	return cib_file_perform_op_delegate(cib, op, host, section, data, output_data, call_options,
	NULL);
	}

	int
	cib_file_perform_op_delegate(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options,
	const char *user_name)
	{
	int rc = pcmk_ok;
	gboolean query = FALSE;
	gboolean changed = FALSE;
	xmlNode *output = NULL;
	xmlNode *cib_diff = NULL;
	xmlNode *result_cib = NULL;
	cib_op_t *fn = NULL;
	int lpc = 0;
	static int max_msg_types = DIMOF(cib_file_ops);

	crm_info("%s on %s", op, section);

	if (cib->state == cib_disconnected) {
	return -ENOTCONN;
	}

	if (output_data != NULL) {
	*output_data = NULL;
	}

	if (op == NULL) {
	return -EINVAL;
	}

	for (lpc = 0; lpc < max_msg_types; lpc++) {
	if (safe_str_eq(op, cib_file_ops[lpc].op)) {
	fn = &(cib_file_ops[lpc].fn);
	query = cib_file_ops[lpc].read_only;
	break;
	}
	}

	if (fn == NULL) {
	return -EPROTONOSUPPORT;
	}

	cib->call_id++;
	rc = cib_perform_op(op, call_options \| cib_inhibit_bcast, fn, query,
	section, NULL, data, TRUE, &changed, in_mem_cib, &result_cib, &cib_diff,
	&output);

	if (rc == -pcmk_err_dtd_validation) {
	validate_xml_verbose(result_cib);
	}

	if (rc != pcmk_ok) {
	free_xml(result_cib);

	} else if (query == FALSE) {
	log_xml_diff(LOG_DEBUG, cib_diff, "cib:diff");
	free_xml(in_mem_cib);
	in_mem_cib = result_cib;
	}

	free_xml(cib_diff);

	if (cib->op_callback != NULL) {
	cib->op_callback(NULL, cib->call_id, rc, output);
	}

	if (output_data && output) {
	*output_data = copy_xml(output);
	}

	if (query == FALSE \|\| (call_options & cib_no_children)) {
	free_xml(output);
	} else if (safe_str_eq(crm_element_name(output), "xpath-query")) {
	free_xml(output);
	}

	return rc;
	}
	diff --git a/lib/cib/cib_native.c b/lib/cib/cib_native.c
	index a3c7e4a02f..5eaef32ef7 100644
	--- a/lib/cib/cib_native.c
	+++ b/lib/cib/cib_native.c
	@@ -1,501 +1,500 @@
	/*
	* Copyright (c) 2004 International Business Machines
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/
	#include <crm_internal.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>

	#include <crm/msg_xml.h>
	#include <crm/common/mainloop.h>

	typedef struct cib_native_opaque_s {
	char *token;
	crm_ipc_t *ipc;
	void (*dnotify_fn) (gpointer user_data);
	mainloop_io_t *source;

	} cib_native_opaque_t;

	int cib_native_perform_op(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options);

	int cib_native_perform_op_delegate(cib_t * cib, const char op, const char host,
	const char section, xmlNode data, xmlNode ** output_data,
	int call_options, const char *user_name);

	int cib_native_free(cib_t * cib);
	int cib_native_signoff(cib_t * cib);
	int cib_native_signon(cib_t * cib, const char *name, enum cib_conn_type type);
	int cib_native_signon_raw(cib_t * cib, const char name, enum cib_conn_type type, int async_fd, int *unused);

	bool cib_native_dispatch(cib_t * cib);

	int cib_native_set_connection_dnotify(cib_t * cib, void (*dnotify) (gpointer user_data));

	cib_t *
	cib_native_new(void)
	{
	cib_native_opaque_t *native = NULL;
	cib_t *cib = cib_new_variant();

	native = calloc(1, sizeof(cib_native_opaque_t));

	cib->variant = cib_native;
	cib->variant_opaque = native;

	native->ipc = NULL;
	native->source = NULL;
	native->dnotify_fn = NULL;

	/* assign variant specific ops */
	- cib->cmds->variant_op = cib_native_perform_op;
	- cib->cmds->delegated_variant_op = cib_native_perform_op_delegate;
	+ cib->delegate_fn = cib_native_perform_op_delegate;
	cib->cmds->signon = cib_native_signon;
	cib->cmds->signon_raw = cib_native_signon_raw;
	cib->cmds->signoff = cib_native_signoff;
	cib->cmds->free = cib_native_free;

	cib->cmds->register_notification = cib_native_register_notification;
	cib->cmds->set_connection_dnotify = cib_native_set_connection_dnotify;

	return cib;
	}

	int
	cib_native_signon(cib_t * cib, const char *name, enum cib_conn_type type)
	{
	return cib_native_signon_raw(cib, name, type, NULL, NULL);
	}

	static int
	cib_native_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata)
	{
	const char *type = NULL;
	xmlNode *msg = NULL;

	cib_t * cib = userdata;
	cib_native_opaque_t *native;

	crm_trace("dispatching %p", userdata);

	if (cib == NULL) {
	crm_err("No CIB!");
	return 0;
	}

	native = cib->variant_opaque;
	msg = string2xml(buffer);

	if (msg == NULL) {
	crm_warn("Received a NULL msg from CIB service.");
	return 0;
	}

	/* do callbacks */
	type = crm_element_value(msg, F_TYPE);
	crm_trace("Activating %s callbacks...", type);
	crm_log_xml_trace(msg, "cib-reply");

	if (safe_str_eq(type, T_CIB)) {
	cib_native_callback(cib, msg, 0, 0);

	} else if (safe_str_eq(type, T_CIB_NOTIFY)) {
	g_list_foreach(cib->notify_list, cib_native_notify, msg);

	} else {
	crm_err("Unknown message type: %s", type);
	}

	free_xml(msg);
	return 0;
	}

	bool
	cib_native_dispatch(cib_t * cib)
	{
	gboolean stay_connected = TRUE;
	cib_native_opaque_t *native;

	if (cib == NULL) {
	crm_err("No CIB!");
	return FALSE;
	}

	crm_trace("dispatching %p", cib);
	native = cib->variant_opaque;
	while(crm_ipc_ready(native->ipc)) {

	if(crm_ipc_read(native->ipc) > 0) {
	const char *msg = crm_ipc_buffer(native->ipc);
	cib_native_dispatch_internal(msg, strlen(msg), cib);
	}

	if(crm_ipc_connected(native->ipc) == FALSE) {
	crm_err("Connection closed");
	stay_connected = FALSE;
	}
	}

	return stay_connected;
	}

	static void
	cib_native_destroy(void *userdata)
	{
	cib_t *cib = userdata;
	cib_native_opaque_t *native = cib->variant_opaque;

	crm_trace("destroying %p", userdata);
	cib->state = cib_disconnected;
	native->source = NULL;
	native->ipc = NULL;

	if(native->dnotify_fn) {
	native->dnotify_fn(userdata);
	}
	}

	int
	cib_native_signon_raw(cib_t * cib, const char name, enum cib_conn_type type, int async_fd, int *unused)
	{
	int rc = pcmk_ok;
	const char *channel = NULL;
	cib_native_opaque_t *native = cib->variant_opaque;

	static struct ipc_client_callbacks cib_callbacks =
	{
	.dispatch = cib_native_dispatch_internal,
	.destroy = cib_native_destroy
	};

	cib->call_timeout = MAX_IPC_DELAY;

	if (type == cib_command) {
	cib->state = cib_connected_command;
	channel = cib_channel_rw;

	} else if (type == cib_command_nonblocking) {
	cib->state = cib_connected_command;
	channel = cib_channel_shm;

	} else if (type == cib_query) {
	cib->state = cib_connected_query;
	channel = cib_channel_ro;

	} else {
	return -ENOTCONN;
	}

	crm_trace("Connecting %s channel", channel);

	if (async_fd != NULL) {
	native->ipc = crm_ipc_new(channel, 0);

	if(native->ipc && crm_ipc_connect(native->ipc)) {
	*async_fd = crm_ipc_get_fd(native->ipc);

	} else if(native->ipc) {
	rc = -ENOTCONN;
	}

	} else {
	native->source = mainloop_add_ipc_client(channel, 5121024 / 512k */, cib, &cib_callbacks);
	native->ipc = mainloop_get_ipc_client(native->source);
	}

	if (rc != pcmk_ok \|\| native->ipc == NULL \|\| crm_ipc_connected(native->ipc) == FALSE) {
	crm_debug("Connection unsuccessful (%d %p)", rc, native->ipc);
	rc = -ENOTCONN;
	}

	if (rc == pcmk_ok) {
	xmlNode *reply = NULL;
	xmlNode *hello = create_xml_node(NULL, "stonith_command");

	crm_xml_add(hello, F_TYPE, T_CIB);
	crm_xml_add(hello, F_CIB_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(hello, F_CIB_CLIENTNAME, name);
	crm_xml_add_int(hello, F_CIB_CALLOPTS, cib_sync_call);

	if (crm_ipc_send(native->ipc, hello, &reply, -1) > 0) {
	const char *msg_type = crm_element_value(reply, F_CIB_OPERATION);

	rc = pcmk_ok;
	crm_log_xml_trace(reply, "reg-reply");

	if (safe_str_neq(msg_type, CRM_OP_REGISTER)) {
	crm_err("Invalid registration message: %s", msg_type);
	rc = -EPROTO;

	} else {
	native->token = crm_element_value_copy(reply, F_CIB_CLIENTID);
	if (native->token == NULL) {
	rc = -EPROTO;
	}
	}

	} else {
	rc = -ECOMM;
	}

	free_xml(hello);
	}

	if (rc == pcmk_ok) {
	crm_debug("Connection to CIB successful");
	return pcmk_ok;
	}

	crm_debug("Connection to CIB failed: %s", pcmk_strerror(rc));
	cib_native_signoff(cib);
	return rc;
	}

	int
	cib_native_signoff(cib_t * cib)
	{
	cib_native_opaque_t *native = cib->variant_opaque;

	crm_debug("Signing out of the CIB Service");

	if (native->ipc != NULL) {
	/* If attached to mainloop and it is active, _close() will result in:
	* - the source being removed from mainloop
	* - the dnotify callback being invoked
	* Otherwise, we are at least correctly disconnecting IPC
	*/
	crm_ipc_close(native->ipc);
	crm_ipc_destroy(native->ipc);
	native->source = NULL;
	native->ipc = NULL;
	}

	cib->state = cib_disconnected;
	cib->type = cib_none;

	return pcmk_ok;
	}

	int
	cib_native_free(cib_t * cib)
	{
	int rc = pcmk_ok;

	if (cib->state != cib_disconnected) {
	rc = cib_native_signoff(cib);
	}

	if (cib->state == cib_disconnected) {
	cib_native_opaque_t *native = cib->variant_opaque;

	free(native->token);
	free(cib->variant_opaque);
	free(cib->cmds);
	free(cib);
	}

	return rc;
	}

	int
	cib_native_perform_op(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options)
	{
	return cib_native_perform_op_delegate(cib, op, host, section,
	data, output_data, call_options, NULL);
	}

	int
	cib_native_perform_op_delegate(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options,
	const char *user_name)
	{
	int rc = pcmk_ok;
	int reply_id = 0;

	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	cib_native_opaque_t *native = cib->variant_opaque;

	if (cib->state == cib_disconnected) {
	return -ENOTCONN;
	}

	if (output_data != NULL) {
	*output_data = NULL;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	cib->call_id++;
	/* prevent call_id from being negative (or zero) and conflicting
	* with the cib_errors enum
	* use 2 because we use it as (cib->call_id - 1) below
	*/
	if (cib->call_id < 1) {
	cib->call_id = 1;
	}

	CRM_CHECK(native->token != NULL,;);
	op_msg =
	cib_create_op(cib->call_id, native->token, op, host, section, data, call_options,
	user_name);
	if (op_msg == NULL) {
	return -EPROTO;
	}

	crm_trace("Sending %s message to CIB service (timeout=%ds)", op, cib->call_timeout);
	rc = crm_ipc_send(native->ipc, op_msg, &op_reply, cib->call_timeout * 1000);
	free_xml(op_msg);

	if(rc < 0) {
	crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%ds): %d", op, cib->call_timeout, rc);
	rc = -ECOMM;
	goto done;
	}

	crm_log_xml_trace(op_reply, "Reply");

	if (!(call_options & cib_sync_call)) {
	crm_trace("Async call, returning %d", cib->call_id);
	CRM_CHECK(cib->call_id != 0, return -ENOMSG);
	free_xml(op_reply);
	return cib->call_id;
	}

	rc = pcmk_ok;
	crm_element_value_int(op_reply, F_CIB_CALLID, &reply_id);
	if (reply_id == cib->call_id) {
	xmlNode *tmp = get_message_xml(op_reply, F_CIB_CALLDATA);

	crm_trace("Syncronous reply %d received", reply_id);
	if (crm_element_value_int(op_reply, F_CIB_RC, &rc) != 0) {
	rc = -EPROTO;
	}

	if (output_data == NULL \|\| (call_options & cib_discard_reply)) {
	crm_trace("Discarding reply");

	} else if (tmp != NULL) {
	*output_data = copy_xml(tmp);
	}

	} else if (reply_id <= 0) {
	crm_err("Recieved bad reply: No id set");
	crm_log_xml_err(op_reply, "Bad reply");
	rc = -ENOMSG;
	goto done;

	} else {
	crm_err("Recieved bad reply: %d (wanted %d)", reply_id, cib->call_id);
	crm_log_xml_err(op_reply, "Old reply");
	rc = -ENOMSG;
	goto done;
	}

	if (op_reply == NULL && cib->state == cib_disconnected) {
	rc = -ENOTCONN;

	} else if (rc == pcmk_ok && op_reply == NULL) {
	rc = -ETIME;
	}

	switch (rc) {
	case pcmk_ok:
	case -EPERM:
	break;

	/* This is an internal value that clients do not and should not care about */
	case -pcmk_err_diff_resync:
	rc = pcmk_ok;
	break;

	/* These indicate internal problems */
	case -EPROTO:
	case -ENOMSG:
	crm_err("Call failed: %s", pcmk_strerror(rc));
	if (op_reply) {
	crm_log_xml_err(op_reply, "Invalid reply");
	}
	break;

	default:
	if (safe_str_neq(op, CIB_OP_QUERY)) {
	crm_warn("Call failed: %s", pcmk_strerror(rc));
	}
	}

	done:
	if (crm_ipc_connected(native->ipc) == FALSE) {
	crm_err("CIB disconnected");
	cib->state = cib_disconnected;
	}

	free_xml(op_reply);
	return rc;
	}

	int
	cib_native_set_connection_dnotify(cib_t * cib, void (*dnotify) (gpointer user_data))
	{
	cib_native_opaque_t *native = NULL;

	if (cib == NULL) {
	crm_err("No CIB!");
	return FALSE;
	}

	native = cib->variant_opaque;
	native->dnotify_fn = dnotify;

	return pcmk_ok;
	}

	int
	cib_native_register_notification(cib_t * cib, const char *callback, int enabled)
	{
	int rc = pcmk_ok;
	xmlNode *notify_msg = create_xml_node(NULL, "cib-callback");
	cib_native_opaque_t *native = cib->variant_opaque;

	if (cib->state != cib_disconnected) {
	crm_xml_add(notify_msg, F_CIB_OPERATION, T_CIB_NOTIFY);
	crm_xml_add(notify_msg, F_CIB_NOTIFY_TYPE, callback);
	crm_xml_add_int(notify_msg, F_CIB_NOTIFY_ACTIVATE, enabled);
	rc = crm_ipc_send(native->ipc, notify_msg, NULL, 1000 * cib->call_timeout);
	if(rc <= 0) {
	crm_trace("Notification not registered: %d", rc);
	rc = -ECOMM;
	}
	}

	free_xml(notify_msg);
	return rc;
	}
	diff --git a/lib/cib/cib_remote.c b/lib/cib/cib_remote.c
	index ecf9acf144..44c14c916f 100644
	--- a/lib/cib/cib_remote.c
	+++ b/lib/cib/cib_remote.c
	@@ -1,641 +1,641 @@
	/*
	* Copyright (c) 2008 Andrew Beekhof
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/
	#include <crm_internal.h>

	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <netdb.h>
	#include <termios.h>
	#include <sys/socket.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/mainloop.h>

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# undef KEYFILE
	# include <gnutls/gnutls.h>
	extern gnutls_anon_client_credentials anon_cred_c;
	extern gnutls_session *create_tls_session(int csock, int type);

	const int kx_prio[] = {
	GNUTLS_KX_ANON_DH,
	0
	};

	#else
	typedef void gnutls_session;
	#endif

	#include <arpa/inet.h>
	#include <sgtty.h>

	#define DH_BITS 1024

	struct remote_connection_s {
	int socket;
	gboolean encrypted;
	gnutls_session *session;
	mainloop_io_t *source;
	char *token;
	};

	typedef struct cib_remote_opaque_s {
	int flags;
	int socket;
	int port;
	char *server;
	char *user;
	char *passwd;
	struct remote_connection_s command;
	struct remote_connection_s callback;

	} cib_remote_opaque_t;

	void cib_remote_connection_destroy(gpointer user_data);
	int cib_remote_dispatch(gpointer user_data);
	int cib_remote_signon(cib_t * cib, const char *name, enum cib_conn_type type);
	int cib_remote_signoff(cib_t * cib);
	int cib_remote_free(cib_t * cib);

	int cib_remote_perform_op(cib_t * cib, const char op, const char host, const char *section,
	- xmlNode * data, xmlNode ** output_data, int call_options);
	+ xmlNode * data, xmlNode ** output_data, int call_options, const char *name);

	static int
	cib_remote_inputfd(cib_t * cib)
	{
	cib_remote_opaque_t *private = cib->variant_opaque;

	return private->callback.socket;
	}

	static int
	cib_remote_set_connection_dnotify(cib_t * cib, void (*dnotify) (gpointer user_data))
	{
	return -EPROTONOSUPPORT;
	}

	static int
	cib_remote_register_notification(cib_t * cib, const char *callback, int enabled)
	{
	xmlNode *notify_msg = create_xml_node(NULL, "cib_command");
	cib_remote_opaque_t *private = cib->variant_opaque;

	crm_xml_add(notify_msg, F_CIB_OPERATION, T_CIB_NOTIFY);
	crm_xml_add(notify_msg, F_CIB_NOTIFY_TYPE, callback);
	crm_xml_add_int(notify_msg, F_CIB_NOTIFY_ACTIVATE, enabled);
	crm_send_remote_msg(private->callback.session, notify_msg, private->callback.encrypted);
	free_xml(notify_msg);
	return pcmk_ok;
	}

	cib_t *
	cib_remote_new(const char server, const char user, const char *passwd, int port,
	gboolean encrypted)
	{
	cib_remote_opaque_t *private = NULL;
	cib_t *cib = cib_new_variant();

	private = calloc(1, sizeof(cib_remote_opaque_t));

	cib->variant = cib_remote;
	cib->variant_opaque = private;

	if (server) {
	private->server = strdup(server);
	}

	if (user) {
	private->user = strdup(user);
	}

	if (passwd) {
	private->passwd = strdup(passwd);
	}

	private->port = port;
	private->command.encrypted = encrypted;
	private->callback.encrypted = encrypted;

	/* assign variant specific ops */
	- cib->cmds->variant_op = cib_remote_perform_op;
	+ cib->delegate_fn = cib_remote_perform_op;
	cib->cmds->signon = cib_remote_signon;
	cib->cmds->signoff = cib_remote_signoff;
	cib->cmds->free = cib_remote_free;
	cib->cmds->inputfd = cib_remote_inputfd;

	cib->cmds->register_notification = cib_remote_register_notification;
	cib->cmds->set_connection_dnotify = cib_remote_set_connection_dnotify;

	return cib;
	}

	static int
	cib_tls_close(cib_t * cib)
	{
	cib_remote_opaque_t *private = cib->variant_opaque;

	shutdown(private->command.socket, SHUT_RDWR); /* no more receptions */
	shutdown(private->callback.socket, SHUT_RDWR); /* no more receptions */
	close(private->command.socket);
	close(private->callback.socket);

	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (private->command.encrypted) {
	gnutls_bye(*(private->command.session), GNUTLS_SHUT_RDWR);
	gnutls_deinit(*(private->command.session));
	gnutls_free(private->command.session);

	gnutls_bye(*(private->callback.session), GNUTLS_SHUT_RDWR);
	gnutls_deinit(*(private->callback.session));
	gnutls_free(private->callback.session);

	gnutls_anon_free_client_credentials(anon_cred_c);
	gnutls_global_deinit();
	}
	#endif
	return 0;
	}

	static int
	cib_tls_signon(cib_t * cib, struct remote_connection_s *connection)
	{
	int sock;
	cib_remote_opaque_t *private = cib->variant_opaque;
	struct sockaddr_in addr;
	int rc = 0;
	char *server = private->server;

	int ret_ga;
	struct addrinfo *res;
	struct addrinfo hints;

	xmlNode *answer = NULL;
	xmlNode *login = NULL;

	static struct mainloop_fd_callbacks cib_fd_callbacks =
	{
	.dispatch = cib_remote_dispatch,
	.destroy = cib_remote_connection_destroy,
	};

	connection->socket = 0;
	connection->session = NULL;

	/* create socket */
	sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
	if (sock == -1) {
	crm_perror(LOG_ERR, "Socket creation failed");
	return -1;
	}

	/* getaddrinfo */
	bzero(&hints, sizeof(struct addrinfo));
	hints.ai_flags = AI_CANONNAME;
	hints.ai_family = AF_INET;
	hints.ai_socktype = SOCK_RAW;

	if (hints.ai_family == AF_INET6) {
	hints.ai_protocol = IPPROTO_ICMPV6;
	} else {
	hints.ai_protocol = IPPROTO_ICMP;
	}

	crm_debug("Looking up %s", server);
	ret_ga = getaddrinfo(server, NULL, &hints, &res);
	if (ret_ga) {
	crm_err("getaddrinfo: %s", gai_strerror(ret_ga));
	close(sock);
	return -1;
	}

	if (res->ai_canonname) {
	server = res->ai_canonname;
	}

	crm_debug("Got address %s for %s", server, private->server);

	if (!res->ai_addr) {
	fprintf(stderr, "getaddrinfo failed");
	exit(1);
	}
	#if 1
	memcpy(&addr, res->ai_addr, res->ai_addrlen);
	#else
	/* connect to server */
	memset(&addr, 0, sizeof(addr));
	addr.sin_family = AF_INET;
	addr.sin_addr.s_addr = inet_addr(server);
	#endif
	addr.sin_port = htons(private->port);

	if (connect(sock, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
	crm_perror(LOG_ERR, "Connection to %s:%d failed", server, private->port);
	close(sock);
	return -1;
	}

	if (connection->encrypted) {
	/* initialize GnuTls lib */
	#ifdef HAVE_GNUTLS_GNUTLS_H
	gnutls_global_init();
	gnutls_anon_allocate_client_credentials(&anon_cred_c);

	/* bind the socket to GnuTls lib */
	connection->session = create_tls_session(sock, GNUTLS_CLIENT);
	if (connection->session == NULL) {
	crm_perror(LOG_ERR, "Session creation for %s:%d failed", server, private->port);
	close(sock);
	cib_tls_close(cib);
	return -1;
	}
	#else
	return -EPROTONOSUPPORT;
	#endif
	} else {
	connection->session = GUINT_TO_POINTER(sock);
	}

	/* login to server */
	login = create_xml_node(NULL, "cib_command");
	crm_xml_add(login, "op", "authenticate");
	crm_xml_add(login, "user", private->user);
	crm_xml_add(login, "password", private->passwd);
	crm_xml_add(login, "hidden", "password");

	crm_send_remote_msg(connection->session, login, connection->encrypted);
	free_xml(login);

	answer = crm_recv_remote_msg(connection->session, connection->encrypted);
	crm_log_xml_trace(answer, "Reply");
	if (answer == NULL) {
	rc = -EPROTO;

	} else {
	/* grab the token */
	const char *msg_type = crm_element_value(answer, F_CIB_OPERATION);
	const char *tmp_ticket = crm_element_value(answer, F_CIB_CLIENTID);

	if (safe_str_neq(msg_type, CRM_OP_REGISTER)) {
	crm_err("Invalid registration message: %s", msg_type);
	rc = -EPROTO;

	} else if (tmp_ticket == NULL) {
	rc = -EPROTO;

	} else {
	connection->token = strdup(tmp_ticket);
	}
	}

	if (rc != 0) {
	cib_tls_close(cib);
	}

	connection->socket = sock;
	connection->source = mainloop_add_fd("cib-remote", connection->socket, cib, &cib_fd_callbacks);
	return rc;
	}

	void
	cib_remote_connection_destroy(gpointer user_data)
	{
	crm_err("Connection destroyed");
	#ifdef HAVE_GNUTLS_GNUTLS_H
	cib_tls_close(user_data);
	#endif
	return;
	}

	int
	cib_remote_dispatch(gpointer user_data)
	{
	cib_t *cib = user_data;
	cib_remote_opaque_t *private = cib->variant_opaque;

	xmlNode *msg = NULL;
	const char *type = NULL;

	crm_info("Message on callback channel");
	msg = crm_recv_remote_msg(private->callback.session, private->callback.encrypted);

	type = crm_element_value(msg, F_TYPE);
	crm_trace("Activating %s callbacks...", type);

	if (safe_str_eq(type, T_CIB)) {
	cib_native_callback(cib, msg, 0, 0);

	} else if (safe_str_eq(type, T_CIB_NOTIFY)) {
	g_list_foreach(cib->notify_list, cib_native_notify, msg);

	} else {
	crm_err("Unknown message type: %s", type);
	}

	if (msg != NULL) {
	free_xml(msg);
	return 0;
	}
	return -1;
	}

	int
	cib_remote_signon(cib_t * cib, const char *name, enum cib_conn_type type)
	{
	int rc = pcmk_ok;
	cib_remote_opaque_t *private = cib->variant_opaque;

	if (private->passwd == NULL) {
	struct termios settings;
	int rc;

	rc = tcgetattr(0, &settings);
	settings.c_lflag &= ~ECHO;
	rc = tcsetattr(0, TCSANOW, &settings);

	fprintf(stderr, "Password: ");
	private->passwd = calloc(1, 1024);
	rc = scanf("%s", private->passwd);
	fprintf(stdout, "\n");
	/* fprintf(stderr, "entered: '%s'\n", buffer); */
	if (rc < 1) {
	private->passwd = NULL;
	}

	settings.c_lflag \|= ECHO;
	rc = tcsetattr(0, TCSANOW, &settings);
	}

	if (private->server == NULL \|\| private->user == NULL) {
	rc = -EINVAL;
	}

	if (rc == pcmk_ok) {
	rc = cib_tls_signon(cib, &(private->command));
	}

	if (rc == pcmk_ok) {
	rc = cib_tls_signon(cib, &(private->callback));
	}

	if (rc == pcmk_ok) {
	xmlNode *hello =
	cib_create_op(0, private->callback.token, CRM_OP_REGISTER, NULL, NULL, NULL, 0, NULL);
	crm_xml_add(hello, F_CIB_CLIENTNAME, name);
	crm_send_remote_msg(private->command.session, hello, private->command.encrypted);
	free_xml(hello);
	}

	if (rc == pcmk_ok) {
	fprintf(stderr, "%s: Opened connection to %s:%d\n", name, private->server, private->port);
	cib->state = cib_connected_command;
	cib->type = cib_command;

	} else {
	fprintf(stderr, "%s: Connection to %s:%d failed: %s\n",
	name, private->server, private->port, pcmk_strerror(rc));
	}

	return rc;
	}

	int
	cib_remote_signoff(cib_t * cib)
	{
	int rc = pcmk_ok;

	/* cib_remote_opaque_t private = cib->variant_opaque; /

	crm_debug("Signing out of the CIB Service");
	#ifdef HAVE_GNUTLS_GNUTLS_H
	cib_tls_close(cib);
	#endif

	cib->state = cib_disconnected;
	cib->type = cib_none;

	return rc;
	}

	int
	cib_remote_free(cib_t * cib)
	{
	int rc = pcmk_ok;

	crm_warn("Freeing CIB");
	if (cib->state != cib_disconnected) {
	rc = cib_remote_signoff(cib);
	if (rc == pcmk_ok) {
	cib_remote_opaque_t *private = cib->variant_opaque;

	free(private->server);
	free(private->user);
	free(private->passwd);
	free(cib->cmds);
	free(private);
	free(cib);
	}
	}

	return rc;
	}

	static gboolean timer_expired = FALSE;
	static struct timer_rec_s *sync_timer = NULL;
	static gboolean
	cib_timeout_handler(gpointer data)
	{
	struct timer_rec_s *timer = data;

	timer_expired = TRUE;
	crm_err("Call %d timed out after %ds", timer->call_id, timer->timeout);

	/* Always return TRUE, never remove the handler
	* We do that after the while-loop in cib_native_perform_op()
	*/
	return TRUE;
	}

	int
	cib_remote_perform_op(cib_t * cib, const char op, const char host, const char *section,
	- xmlNode * data, xmlNode ** output_data, int call_options)
	+ xmlNode * data, xmlNode ** output_data, int call_options, const char *name)
	{
	int rc = pcmk_ok;

	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	cib_remote_opaque_t *private = cib->variant_opaque;

	if (sync_timer == NULL) {
	sync_timer = calloc(1, sizeof(struct timer_rec_s));
	}

	if (cib->state == cib_disconnected) {
	return -ENOTCONN;
	}

	if (output_data != NULL) {
	*output_data = NULL;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	cib->call_id++;
	/* prevent call_id from being negative (or zero) and conflicting
	* with the cib_errors enum
	* use 2 because we use it as (cib->call_id - 1) below
	*/
	if (cib->call_id < 1) {
	cib->call_id = 1;
	}

	op_msg =
	cib_create_op(cib->call_id, private->callback.token, op, host, section, data, call_options,
	NULL);
	if (op_msg == NULL) {
	return -EPROTO;
	}

	crm_trace("Sending %s message to CIB service", op);
	crm_send_remote_msg(private->command.session, op_msg, private->command.encrypted);
	free_xml(op_msg);

	if ((call_options & cib_discard_reply)) {
	crm_trace("Discarding reply");
	return pcmk_ok;

	} else if (!(call_options & cib_sync_call)) {
	return cib->call_id;
	}

	crm_trace("Waiting for a syncronous reply");

	if (cib->call_timeout > 0) {
	/* We need this, even with msgfromIPC_timeout(), because we might
	* get other/older replies that don't match the active request
	*/
	timer_expired = FALSE;
	sync_timer->call_id = cib->call_id;
	sync_timer->timeout = cib->call_timeout * 1000;
	sync_timer->ref = g_timeout_add(sync_timer->timeout, cib_timeout_handler, sync_timer);
	}

	while (timer_expired == FALSE) {
	int reply_id = -1;
	int msg_id = cib->call_id;

	op_reply = crm_recv_remote_msg(private->command.session, private->command.encrypted);
	if (op_reply == NULL) {
	break;
	}

	crm_element_value_int(op_reply, F_CIB_CALLID, &reply_id);
	CRM_CHECK(reply_id > 0, free_xml(op_reply);
	if (sync_timer->ref > 0) {
	g_source_remove(sync_timer->ref); sync_timer->ref = 0;}
	return -ENOMSG) ;

	if (reply_id == msg_id) {
	break;

	} else if (reply_id < msg_id) {
	crm_debug("Received old reply: %d (wanted %d)", reply_id, msg_id);
	crm_log_xml_trace(op_reply, "Old reply");

	} else if ((reply_id - 10000) > msg_id) {
	/* wrap-around case */
	crm_debug("Received old reply: %d (wanted %d)", reply_id, msg_id);
	crm_log_xml_trace(op_reply, "Old reply");
	} else {
	crm_err("Received a __future__ reply:" " %d (wanted %d)", reply_id, msg_id);
	}

	free_xml(op_reply);
	op_reply = NULL;
	}

	if (sync_timer->ref > 0) {
	g_source_remove(sync_timer->ref);
	sync_timer->ref = 0;
	}

	if (timer_expired) {
	return -ETIME;
	}

	/* if(IPC_ISRCONN(native->command_channel) == FALSE) { */
	/* crm_err("CIB disconnected: %d", */
	/* native->command_channel->ch_status); */
	/* cib->state = cib_disconnected; */
	/* } */

	if (op_reply == NULL) {
	crm_err("No reply message - empty");
	return -ENOMSG;
	}

	crm_trace("Syncronous reply received");

	/* Start processing the reply... */
	if (crm_element_value_int(op_reply, F_CIB_RC, &rc) != 0) {
	rc = -EPROTO;
	}

	if (rc == -pcmk_err_diff_resync) {
	/* This is an internal value that clients do not and should not care about */
	rc = pcmk_ok;
	}

	if (rc == pcmk_ok \|\| rc == -EPERM) {
	crm_log_xml_debug(op_reply, "passed");

	} else {
	/* } else if(rc == -ETIME) { */
	crm_err("Call failed: %s", pcmk_strerror(rc));
	crm_log_xml_warn(op_reply, "failed");
	}

	if (output_data == NULL) {
	/* do nothing more */

	} else if (!(call_options & cib_discard_reply)) {
	xmlNode *tmp = get_message_xml(op_reply, F_CIB_CALLDATA);

	if (tmp == NULL) {
	crm_trace("No output in reply to \"%s\" command %d", op, cib->call_id - 1);
	} else {
	*output_data = copy_xml(tmp);
	}
	}

	free_xml(op_reply);

	return rc;
	}
	diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c
	index 4773733165..06fbfbb78a 100644
	--- a/lib/cib/cib_utils.c
	+++ b/lib/cib/cib_utils.c
	@@ -1,772 +1,783 @@
	/*
	* Copyright (c) 2004 International Business Machines
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/
	#include <crm_internal.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <sys/utsname.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/pengine/rules.h>

	struct config_root_s {
	const char *name;
	const char *parent;
	const char *path;
	};

	/*
	* "//crm_config" will also work in place of "/cib/configuration/crm_config"
	* The / prefix means find starting from the root, whereas the // prefix means
	* find anywhere and risks multiple matches
	*/
	/* INDENT-OFF */
	struct config_root_s known_paths[] = {
	{ NULL, NULL, "//cib" },
	{ XML_TAG_CIB, NULL, "//cib" },
	{ XML_CIB_TAG_STATUS, "/cib", "//cib/status" },
	{ XML_CIB_TAG_CONFIGURATION,"/cib", "//cib/configuration" },
	{ XML_CIB_TAG_CRMCONFIG, "/cib/configuration", "//cib/configuration/crm_config" },
	{ XML_CIB_TAG_NODES, "/cib/configuration", "//cib/configuration/nodes" },
	{ XML_CIB_TAG_DOMAINS, "/cib/configuration", "//cib/configuration/domains" },
	{ XML_CIB_TAG_RESOURCES, "/cib/configuration", "//cib/configuration/resources" },
	{ XML_CIB_TAG_CONSTRAINTS, "/cib/configuration", "//cib/configuration/constraints" },
	{ XML_CIB_TAG_OPCONFIG, "/cib/configuration", "//cib/configuration/op_defaults" },
	{ XML_CIB_TAG_RSCCONFIG, "/cib/configuration", "//cib/configuration/rsc_defaults" },
	{ XML_CIB_TAG_ACLS, "/cib/configuration", "//cib/configuration/acls" },
	{ XML_TAG_FENCING_TOPOLOGY, "/cib/configuration", "//cib/configuration/fencing-topology" },
	{ XML_CIB_TAG_SECTION_ALL, NULL, "//cib" },
	};
	/* INDENT-ON */

	int
	cib_compare_generation(xmlNode * left, xmlNode * right)
	{
	int lpc = 0;

	const char *attributes[] = {
	XML_ATTR_GENERATION_ADMIN,
	XML_ATTR_GENERATION,
	XML_ATTR_NUMUPDATES,
	};

	crm_log_xml_trace(left, "left");
	crm_log_xml_trace(right, "right");

	for (lpc = 0; lpc < DIMOF(attributes); lpc++) {
	int int_elem_l = -1;
	int int_elem_r = -1;
	const char *elem_r = NULL;
	const char *elem_l = crm_element_value(left, attributes[lpc]);

	if (right != NULL) {
	elem_r = crm_element_value(right, attributes[lpc]);
	}

	if (elem_l != NULL) {
	int_elem_l = crm_parse_int(elem_l, NULL);
	}
	if (elem_r != NULL) {
	int_elem_r = crm_parse_int(elem_r, NULL);
	}

	if (int_elem_l < int_elem_r) {
	crm_trace("%s (%s < %s)", attributes[lpc], crm_str(elem_l), crm_str(elem_r));
	return -1;

	} else if (int_elem_l > int_elem_r) {
	crm_trace("%s (%s > %s)", attributes[lpc], crm_str(elem_l), crm_str(elem_r));
	return 1;
	}
	}

	return 0;
	}

	xmlNode *
	get_cib_copy(cib_t * cib)
	{
	xmlNode *xml_cib;
	int options = cib_scope_local \| cib_sync_call;

	if (cib->cmds->query(cib, NULL, &xml_cib, options) != pcmk_ok) {
	crm_err("Couldnt retrieve the CIB");
	return NULL;
	} else if (xml_cib == NULL) {
	crm_err("The CIB result was empty");
	return NULL;
	}

	if (safe_str_eq(crm_element_name(xml_cib), XML_TAG_CIB)) {
	return xml_cib;
	}
	free_xml(xml_cib);
	return NULL;
	}

	xmlNode *
	cib_get_generation(cib_t * cib)
	{
	xmlNode *the_cib = get_cib_copy(cib);
	xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);

	if (the_cib != NULL) {
	copy_in_properties(generation, the_cib);
	free_xml(the_cib);
	}

	return generation;
	}

	void
	log_cib_diff(int log_level, xmlNode * diff, const char *function)
	{
	int add_updates = 0;
	int add_epoch = 0;
	int add_admin_epoch = 0;

	int del_updates = 0;
	int del_epoch = 0;
	int del_admin_epoch = 0;

	if (diff == NULL) {
	return;
	}

	cib_diff_version_details(diff, &add_admin_epoch, &add_epoch, &add_updates,
	&del_admin_epoch, &del_epoch, &del_updates);

	if (add_updates != del_updates) {
	do_crm_log(log_level, "%s: Diff: --- %d.%d.%d", function,
	del_admin_epoch, del_epoch, del_updates);
	do_crm_log(log_level, "%s: Diff: +++ %d.%d.%d", function,
	add_admin_epoch, add_epoch, add_updates);
	} else if (diff != NULL) {
	do_crm_log(log_level,
	"%s: Local-only Change: %d.%d.%d", function,
	add_admin_epoch, add_epoch, add_updates);
	}

	log_xml_diff(log_level, diff, function);
	}

	gboolean
	cib_version_details(xmlNode * cib, int admin_epoch, int epoch, int *updates)
	{
	*epoch = -1;
	*updates = -1;
	*admin_epoch = -1;

	if (cib == NULL) {
	return FALSE;

	} else {
	crm_element_value_int(cib, XML_ATTR_GENERATION, epoch);
	crm_element_value_int(cib, XML_ATTR_NUMUPDATES, updates);
	crm_element_value_int(cib, XML_ATTR_GENERATION_ADMIN, admin_epoch);
	}
	return TRUE;
	}

	gboolean
	cib_diff_version_details(xmlNode * diff, int admin_epoch, int epoch, int *updates,
	int _admin_epoch, int _epoch, int *_updates)
	{
	xmlNode *tmp = NULL;

	tmp = find_xml_node(diff, "diff-added", FALSE);
	tmp = find_xml_node(tmp, XML_TAG_CIB, FALSE);
	cib_version_details(tmp, admin_epoch, epoch, updates);

	tmp = find_xml_node(diff, "diff-removed", FALSE);
	cib_version_details(tmp, _admin_epoch, _epoch, _updates);
	return TRUE;
	}

	/*
	* The caller should never free the return value
	*/

	const char *
	get_object_path(const char *object_type)
	{
	int lpc = 0;
	int max = DIMOF(known_paths);

	for (; lpc < max; lpc++) {
	if ((object_type == NULL && known_paths[lpc].name == NULL)
	\|\| safe_str_eq(object_type, known_paths[lpc].name)) {
	return known_paths[lpc].path;
	}
	}
	return NULL;
	}

	const char *
	get_object_parent(const char *object_type)
	{
	int lpc = 0;
	int max = DIMOF(known_paths);

	for (; lpc < max; lpc++) {
	if (safe_str_eq(object_type, known_paths[lpc].name)) {
	return known_paths[lpc].parent;
	}
	}
	return NULL;
	}

	xmlNode *
	get_object_root(const char object_type, xmlNode the_root)
	{
	const char *xpath = get_object_path(object_type);

	if (xpath == NULL) {
	return the_root; /* or return NULL? */
	}

	return get_xpath_object(xpath, the_root, LOG_DEBUG_4);
	}

	xmlNode *
	create_cib_fragment_adv(xmlNode * update, const char update_section, const char source)
	{
	xmlNode *cib = NULL;
	gboolean whole_cib = FALSE;
	xmlNode *object_root = NULL;
	char *local_section = NULL;

	/* crm_debug("Creating a blank fragment: %s", update_section); */

	if (update == NULL && update_section == NULL) {
	crm_trace("Creating a blank fragment");
	update = createEmptyCib();
	crm_xml_add(cib, XML_ATTR_ORIGIN, source);
	return update;

	} else if (update == NULL) {
	crm_err("No update to create a fragment for");
	return NULL;

	}

	CRM_CHECK(update_section != NULL, return NULL);
	if (safe_str_eq(crm_element_name(update), XML_TAG_CIB)) {
	whole_cib = TRUE;
	}

	if (whole_cib == FALSE) {
	cib = createEmptyCib();
	crm_xml_add(cib, XML_ATTR_ORIGIN, source);
	object_root = get_object_root(update_section, cib);
	add_node_copy(object_root, update);

	} else {
	cib = copy_xml(update);
	crm_xml_add(cib, XML_ATTR_ORIGIN, source);
	}

	free(local_section);
	crm_trace("Verifying created fragment");
	return cib;
	}

	/*
	* It is the callers responsibility to free both the new CIB (output)
	* and the new CIB (input)
	*/
	xmlNode *
	createEmptyCib(void)
	{
	xmlNode cib_root = NULL, config = NULL;

	cib_root = create_xml_node(NULL, XML_TAG_CIB);

	config = create_xml_node(cib_root, XML_CIB_TAG_CONFIGURATION);
	create_xml_node(cib_root, XML_CIB_TAG_STATUS);

	/* crm_xml_add(cib_root, "version", "1"); */
	create_xml_node(config, XML_CIB_TAG_CRMCONFIG);
	create_xml_node(config, XML_CIB_TAG_NODES);
	create_xml_node(config, XML_CIB_TAG_RESOURCES);
	create_xml_node(config, XML_CIB_TAG_CONSTRAINTS);

	return cib_root;
	}

	static unsigned int dtd_throttle = 0;

	void
	fix_cib_diff(xmlNode * last, xmlNode * next, xmlNode * local_diff, gboolean changed)
	{
	xmlNode *cib = NULL;
	xmlNode *diff_child = NULL;
	const char *tag = NULL;
	const char *value = NULL;

	tag = "diff-removed";
	diff_child = find_xml_node(local_diff, tag, FALSE);
	if (diff_child == NULL) {
	diff_child = create_xml_node(local_diff, tag);
	}

	tag = XML_TAG_CIB;
	cib = find_xml_node(diff_child, tag, FALSE);
	if (cib == NULL) {
	cib = create_xml_node(diff_child, tag);
	}

	tag = XML_ATTR_GENERATION_ADMIN;
	value = crm_element_value(last, tag);
	crm_xml_add(diff_child, tag, value);
	if (changed) {
	crm_xml_add(cib, tag, value);
	}

	tag = XML_ATTR_GENERATION;
	value = crm_element_value(last, tag);
	crm_xml_add(diff_child, tag, value);
	if (changed) {
	crm_xml_add(cib, tag, value);
	}

	tag = XML_ATTR_NUMUPDATES;
	value = crm_element_value(last, tag);
	crm_xml_add(cib, tag, value);
	crm_xml_add(diff_child, tag, value);

	tag = "diff-added";
	diff_child = find_xml_node(local_diff, tag, FALSE);
	if (diff_child == NULL) {
	diff_child = create_xml_node(local_diff, tag);
	}

	tag = XML_TAG_CIB;
	cib = find_xml_node(diff_child, tag, FALSE);
	if (cib == NULL) {
	cib = create_xml_node(diff_child, tag);
	}

	if (next) {
	xmlAttrPtr xIter = NULL;

	for (xIter = next->properties; xIter; xIter = xIter->next) {
	const char p_name = (const char )xIter->name;
	const char *p_value = crm_element_value(next, p_name);

	xmlSetProp(cib, (const xmlChar )p_name, (const xmlChar )p_value);
	}
	}

	crm_log_xml_trace(local_diff, "Repaired-diff");
	}

	int
	cib_perform_op(const char op, int call_options, cib_op_t fn, gboolean is_query,
	const char section, xmlNode req, xmlNode * input,
	gboolean manage_counters, gboolean * config_changed,
	xmlNode * current_cib, xmlNode result_cib, xmlNode diff, xmlNode ** output)
	{

	int rc = pcmk_ok;
	gboolean check_dtd = TRUE;
	xmlNode *scratch = NULL;
	xmlNode *local_diff = NULL;
	const char *current_dtd = "unknown";

	CRM_CHECK(output != NULL, return -ENOMSG);
	CRM_CHECK(result_cib != NULL, return -ENOMSG);
	CRM_CHECK(config_changed != NULL, return -ENOMSG);

	*output = NULL;
	*result_cib = NULL;
	*config_changed = FALSE;

	if (fn == NULL) {
	return -EINVAL;
	}

	if (is_query) {
	rc = (*fn) (op, call_options, section, req, input, current_cib, result_cib, output);
	return rc;
	}

	scratch = copy_xml(current_cib);
	rc = (*fn) (op, call_options, section, req, input, current_cib, &scratch, output);

	CRM_CHECK(current_cib != scratch, return -EINVAL);

	if (rc == pcmk_ok && scratch == NULL) {
	rc = -EINVAL;
	}

	if (rc == pcmk_ok && scratch) {
	const char *new_version = crm_element_value(scratch, XML_ATTR_CRM_VERSION);

	if (new_version && compare_version(new_version, CRM_FEATURE_SET) > 0) {
	crm_err("Discarding update with feature set '%s' greater than our own '%s'",
	new_version, CRM_FEATURE_SET);
	rc = -EPROTONOSUPPORT;
	}
	}

	if (rc == pcmk_ok && current_cib) {
	int old = 0;
	int new = 0;

	crm_element_value_int(scratch, XML_ATTR_GENERATION_ADMIN, &new);
	crm_element_value_int(current_cib, XML_ATTR_GENERATION_ADMIN, &old);

	if (old > new) {
	crm_err("%s went backwards: %d -> %d (Opts: 0x%x)",
	XML_ATTR_GENERATION_ADMIN, old, new, call_options);
	crm_log_xml_warn(req, "Bad Op");
	crm_log_xml_warn(input, "Bad Data");
	rc = -pcmk_err_old_data;

	} else if (old == new) {
	crm_element_value_int(scratch, XML_ATTR_GENERATION, &new);
	crm_element_value_int(current_cib, XML_ATTR_GENERATION, &old);
	if (old > new) {
	crm_err("%s went backwards: %d -> %d (Opts: 0x%x)",
	XML_ATTR_GENERATION, old, new, call_options);
	crm_log_xml_warn(req, "Bad Op");
	crm_log_xml_warn(input, "Bad Data");
	rc = -pcmk_err_old_data;
	}
	}
	}

	if (rc == pcmk_ok) {
	fix_plus_plus_recursive(scratch);
	current_dtd = crm_element_value(scratch, XML_ATTR_VALIDATION);

	if (manage_counters) {
	if (is_set(call_options, cib_inhibit_bcast) && safe_str_eq(section, XML_CIB_TAG_STATUS)) {
	/* Fast-track changes connections which wont be broadcasting anywhere */
	cib_update_counter(scratch, XML_ATTR_NUMUPDATES, FALSE);
	goto done;
	}

	/* The diff calculation in cib_config_changed() accounts for 25% of the
	* CIB's total CPU usage on the DC
	*
	* RNG validation on the otherhand, accounts for only 9%...
	*/
	*config_changed = cib_config_changed(current_cib, scratch, &local_diff);

	if (*config_changed) {
	cib_update_counter(scratch, XML_ATTR_NUMUPDATES, TRUE);
	cib_update_counter(scratch, XML_ATTR_GENERATION, FALSE);

	} else {
	/* Previously we only did this if the diff detected a change
	*
	* But we replies are still sent, even if nothing changes so we
	* don't save any network traffic and means we need to jump
	* through expensive hoops to detect ordering changes - see below
	*/
	cib_update_counter(scratch, XML_ATTR_NUMUPDATES, FALSE);

	if (local_diff == NULL) {
	/* Nothing to check */
	check_dtd = FALSE;

	/* Create a fake diff so that notifications, which include a _digest_,
	* will be sent to our peers
	*
	* This is the cheapest way to detect changes to group/set ordering
	*
	* Previously we compared the old and new digest in cib_config_changed(),
	* but that accounted for 15% of the CIB's total CPU usage on the DC
	*/
	local_diff = create_xml_node(NULL, "diff");
	crm_xml_add(local_diff, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	create_xml_node(local_diff, "diff-removed");
	create_xml_node(local_diff, "diff-added");

	/* Usually these are attrd re-updates */
	crm_log_xml_trace(req, "Non-change");

	} else if (dtd_throttle++ % 20) {
	/* Throttle the amount of costly validation we perform due to status updates
	* a) we don't really care whats in the status section
	* b) we don't validate any of it's contents at the moment anyway
	*/
	check_dtd = FALSE;
	}
	}
	}
	}

	if (diff != NULL && local_diff != NULL) {
	/* Only fix the diff if we'll return it... */
	fix_cib_diff(current_cib, scratch, local_diff, *config_changed);
	*diff = local_diff;
	local_diff = NULL;
	}

	done:
	if (rc == pcmk_ok && check_dtd && validate_xml(scratch, NULL, TRUE) == FALSE) {
	crm_warn("Updated CIB does not validate against %s schema/dtd", crm_str(current_dtd));
	rc = -pcmk_err_dtd_validation;
	}

	*result_cib = scratch;
	free_xml(local_diff);
	return rc;
	}

	xmlNode *
	cib_create_op(int call_id, const char token, const char op, const char host, const char section,
	xmlNode * data, int call_options, const char *user_name)
	{
	xmlNode *op_msg = create_xml_node(NULL, "cib_command");

	CRM_CHECK(op_msg != NULL, return NULL);
	CRM_CHECK(token != NULL, return NULL);

	crm_xml_add(op_msg, F_XML_TAGNAME, "cib_command");

	crm_xml_add(op_msg, F_TYPE, T_CIB);
	crm_xml_add(op_msg, F_CIB_CALLBACK_TOKEN, token);
	crm_xml_add(op_msg, F_CIB_OPERATION, op);
	crm_xml_add(op_msg, F_CIB_HOST, host);
	crm_xml_add(op_msg, F_CIB_SECTION, section);
	crm_xml_add_int(op_msg, F_CIB_CALLID, call_id);
	#if ENABLE_ACL
	if (user_name) {
	crm_xml_add(op_msg, F_CIB_USER, user_name);
	}
	#endif
	crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options);
	crm_xml_add_int(op_msg, F_CIB_CALLOPTS, call_options);

	if (data != NULL) {
	add_message_xml(op_msg, F_CIB_CALLDATA, data);
	}

	if (call_options & cib_inhibit_bcast) {
	CRM_CHECK((call_options & cib_scope_local), return NULL);
	}
	return op_msg;
	}

	void
	cib_native_callback(cib_t * cib, xmlNode * msg, int call_id, int rc)
	{
	xmlNode *output = NULL;
	cib_callback_client_t *blob = NULL;
	cib_callback_client_t local_blob;

	local_blob.id = NULL;
	local_blob.callback = NULL;
	local_blob.user_data = NULL;
	local_blob.only_success = FALSE;

	if (msg != NULL) {
	crm_element_value_int(msg, F_CIB_RC, &rc);
	crm_element_value_int(msg, F_CIB_CALLID, &call_id);
	output = get_message_xml(msg, F_CIB_CALLDATA);
	}

	blob = g_hash_table_lookup(cib_op_callback_table, GINT_TO_POINTER(call_id));

	if (blob != NULL) {
	local_blob = *blob;
	blob = NULL;

	remove_cib_op_callback(call_id, FALSE);

	} else {
	crm_trace("No callback found for call %d", call_id);
	local_blob.callback = NULL;
	}

	if (cib == NULL) {
	crm_debug("No cib object supplied");
	}

	if (rc == -pcmk_err_diff_resync) {
	/* This is an internal value that clients do not and should not care about */
	rc = pcmk_ok;
	}

	if (local_blob.callback != NULL && (rc == pcmk_ok \|\| local_blob.only_success == FALSE)) {
	crm_trace("Invoking callback %s for call %d", crm_str(local_blob.id), call_id);
	local_blob.callback(msg, call_id, rc, output, local_blob.user_data);

	} else if (cib && cib->op_callback == NULL && rc != pcmk_ok) {
	crm_warn("CIB command failed: %s", pcmk_strerror(rc));
	crm_log_xml_debug(msg, "Failed CIB Update");
	}

	if (cib && cib->op_callback != NULL) {
	crm_trace("Invoking global callback for call %d", call_id);
	cib->op_callback(msg, call_id, rc, output);
	}
	crm_trace("OP callback activated.");
	}

	void
	cib_native_notify(gpointer data, gpointer user_data)
	{
	xmlNode *msg = user_data;
	cib_notify_client_t *entry = data;
	const char *event = NULL;

	if (msg == NULL) {
	crm_warn("Skipping callback - NULL message");
	return;
	}

	event = crm_element_value(msg, F_SUBTYPE);

	if (entry == NULL) {
	crm_warn("Skipping callback - NULL callback client");
	return;

	} else if (entry->callback == NULL) {
	crm_warn("Skipping callback - NULL callback");
	return;

	} else if (safe_str_neq(entry->event, event)) {
	crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event);
	return;
	}

	crm_trace("Invoking callback for %p/%s event...", entry, event);
	entry->callback(event, msg);
	crm_trace("Callback invoked...");
	}

	gboolean
	determine_host(cib_t * cib_conn, char node_uname, char node_uuid)
	{
	CRM_CHECK(node_uname != NULL, return FALSE);

	if (*node_uname == NULL) {
	struct utsname name;

	if (uname(&name) < 0) {
	crm_perror(LOG_ERR, "uname(2) call failed");
	return FALSE;
	}
	*node_uname = strdup(name.nodename);
	crm_info("Detected uname: %s", *node_uname);
	}

	if (cib_conn && node_uname != NULL && node_uuid != NULL && node_uuid == NULL) {
	int rc = query_node_uuid(cib_conn, *node_uname, node_uuid);

	if (rc != pcmk_ok) {
	fprintf(stderr, "Could not map uname=%s to a UUID: %s\n",
	*node_uname, pcmk_strerror(rc));
	return FALSE;
	}
	crm_info("Mapped %s to %s", node_uname, crm_str(node_uuid));
	}
	return TRUE;
	}

	pe_cluster_option cib_opts[] = {
	/* name, old-name, validate, default, description */
	{"enable-acl", NULL, "boolean", NULL, "false", &check_boolean,
	"Enable CIB ACL", NULL}
	,
	};

	void
	cib_metadata(void)
	{
	config_metadata("Cluster Information Base", "1.0",
	"Cluster Information Base Options",
	"This is a fake resource that details the options that can be configured for the Cluster Information Base.",
	cib_opts, DIMOF(cib_opts));
	}

	void
	verify_cib_options(GHashTable * options)
	{
	verify_all_options(options, cib_opts, DIMOF(cib_opts));
	}

	const char *
	cib_pref(GHashTable * options, const char *name)
	{
	return get_cluster_pref(options, cib_opts, DIMOF(cib_opts), name);
	}

	gboolean
	cib_read_config(GHashTable * options, xmlNode * current_cib)
	{
	xmlNode *config = NULL;
	ha_time_t *now = NULL;

	if (options == NULL \|\| current_cib == NULL) {
	return FALSE;
	}

	now = new_ha_date(TRUE);

	g_hash_table_remove_all(options);

	config = get_object_root(XML_CIB_TAG_CRMCONFIG, current_cib);
	if (config) {
	unpack_instance_attributes(current_cib, config, XML_CIB_TAG_PROPSET, NULL, options,
	CIB_OPTIONS_FIRST, FALSE, now);
	}

	verify_cib_options(options);

	free_ha_date(now);

	return TRUE;
	}

	gboolean
	cib_internal_config_changed(xmlNode * diff)
	{
	gboolean changed = FALSE;
	const char *config_xpath =
	"//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_CRMCONFIG;
	xmlXPathObject *xpathObj = NULL;

	if (diff == NULL) {
	return FALSE;
	}

	xpathObj = xpath_search(diff, config_xpath);
	if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
	changed = TRUE;
	}

	if (xpathObj) {
	xmlXPathFreeObject(xpathObj);
	}

	return changed;
	}
	+
	+int cib_internal_op(cib_t * cib, const char op, const char host,
	+ const char section, xmlNode data,
	+ xmlNode ** output_data, int call_options, const char *user_name)
	+{
	+ int (delegate)(cib_t cib, const char op, const char host,
	+ const char section, xmlNode data,
	+ xmlNode ** output_data, int call_options, const char *user_name) = cib->delegate_fn;
	+
	+ return delegate(cib, op, host, section, data, output_data, call_options, user_name);
	+}
	diff --git a/tools/cibadmin.c b/tools/cibadmin.c
	index 39b2976e68..bcff299a99 100644
	--- a/tools/cibadmin.c
	+++ b/tools/cibadmin.c
	@@ -1,594 +1,593 @@

	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/ipc.h>
	#include <crm/cib/internal.h>

	int exit_code = pcmk_ok;
	int message_timer_id = -1;
	int message_timeout_ms = 30;

	GMainLoop *mainloop = NULL;

	const char *host = NULL;
	void usage(const char *cmd, int exit_status);
	int do_init(void);
	int do_work(xmlNode * input, int command_options, xmlNode ** output);

	gboolean admin_message_timeout(gpointer data);
	void cib_connection_destroy(gpointer user_data);
	void cibadmin_op_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);

	int command_options = 0;
	const char *cib_action = NULL;

	typedef struct str_list_s {
	int num_items;
	char *value;
	struct str_list_s *next;
	} str_list_t;

	char *obj_type = NULL;
	char *status = NULL;
	char *migrate_from = NULL;
	char *migrate_res = NULL;
	char *subtype = NULL;
	char *reset = NULL;

	int request_id = 0;
	int operation_status = 0;
	cib_t *the_cib = NULL;
	gboolean force_flag = FALSE;

	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	{"help", 0, 0, '?', "\tThis text"},
	{"version", 0, 0, '$', "\tVersion information" },
	{"verbose", 0, 0, 'V', "\tIncrease debug output\n"},

	{"-spacer-", 0, 0, '-', "Commands:"},
	{"upgrade", 0, 0, 'u', "\tUpgrade the configuration to the latest syntax"},
	{"query", 0, 0, 'Q', "\tQuery the contents of the CIB"},
	{"erase", 0, 0, 'E', "\tErase the contents of the whole CIB"},
	{"bump", 0, 0, 'B', "\tIncrease the CIB's epoch value by 1"},
	{"create", 0, 0, 'C', "\tCreate an object in the CIB. Will fail if the object already exists."},
	{"modify", 0, 0, 'M', "\tFind the object somewhere in the CIB's XML tree and update it. Fails if the object does not exist unless -c is specified"},
	{"patch", 0, 0, 'P', "\tSupply an update in the form of an xml diff (See also: crm_diff)"},
	{"replace", 0, 0, 'R', "\tRecursivly replace an object in the CIB"},
	{"delete", 0, 0, 'D', "\tDelete the first object matching the supplied criteria, Eg. <op id=\"rsc1_op1\" name=\"monitor\"/>"},
	{"-spacer-", 0, 0, '-', "\n\t\t\tThe tagname and all attributes must match in order for the element to be deleted"},
	{"delete-all", 0, 0, 'd', "\tWhen used with --xpath, remove all matching objects in the configuration instead of just the first one"},
	{"md5-sum", 0, 0, '5', "\tCalculate the on-disk CIB digest"},
	{"md5-sum-versioned", 0, 0, '6', "\tCalculate an on-the-wire versioned CIB digest"},
	{"sync", 0, 0, 'S', "\t(Advanced) Force a refresh of the CIB to all nodes\n"},
	{"make-slave", 0, 0, 'r', NULL, 1},
	{"make-master", 0, 0, 'w', NULL, 1},
	{"is-master", 0, 0, 'm', NULL, 1},
	{"empty", 0, 0, 'a', "\tOutput an empty CIB"},
	{"blank", 0, 0, 'a', NULL, 1},

	{"-spacer-",1, 0, '-', "\nAdditional options:"},
	{"force", 0, 0, 'f'},
	{"timeout", 1, 0, 't', "Time (in seconds) to wait before declaring the operation failed"},
	{"sync-call", 0, 0, 's', "Wait for call to complete before returning"},
	{"local", 0, 0, 'l', "\tCommand takes effect locally. Should only be used for queries"},
	{"allow-create",0, 0, 'c', "(Advanced) Allow the target of a -M operation to be created if they do not exist"},
	{"no-children", 0, 0, 'n', "(Advanced) When querying an object, do not return include its children in the result\n"},
	{"no-bcast", 0, 0, 'b', NULL, 1},

	{"-spacer-", 0, 0, '-', "Data:"},
	{"xml-text", 1, 0, 'X', "Retrieve XML from the supplied string"},
	{"xml-file", 1, 0, 'x', "Retrieve XML from the named file"},
	{"xml-pipe", 0, 0, 'p', "Retrieve XML from stdin\n"},

	{"xpath", 1, 0, 'A', "A valid XPath to use instead of -o"},
	{"scope", 1, 0, 'o', "Limit the scope of the operation to a specific section of the CIB."},
	{"-spacer-", 0, 0, '-', "\t\t\tValid values are: nodes, resources, constraints, crm_config, rsc_defaults, op_defaults, status"},
	{"node", 1, 0, 'N', "(Advanced) Send command to the specified host\n"},
	{"-space-", 0, 0, '!', NULL, 1},

	{"-spacer-", 0, 0, '-', "\nExamples:\n"},
	{"-spacer-", 0, 0, '-', "Query the configuration from the local node:", pcmk_option_paragraph},
	{"-spacer-", 0, 0, '-', " cibadmin --query --local", pcmk_option_example},

	{"-spacer-", 0, 0, '-', "Query the just the cluster options configuration:", pcmk_option_paragraph},
	{"-spacer-", 0, 0, '-', " cibadmin --query --scope crm_config", pcmk_option_example},

	{"-spacer-", 0, 0, '-', "Query all 'target-role' settings:", pcmk_option_paragraph},
	{"-spacer-", 0, 0, '-', " cibadmin --query --xpath \"//nvpair[@name='target-role']\"", pcmk_option_example},

	{"-spacer-", 0, 0, '-', "Remove all 'is-managed' settings:", pcmk_option_paragraph},
	{"-spacer-", 0, 0, '-', " cibadmin --delete-all --xpath \"//nvpair[@name='is-managed']\"", pcmk_option_example},

	{"-spacer-", 0, 0, '-', "Remove the resource named 'old':", pcmk_option_paragraph},
	{"-spacer-", 0, 0, '-', " cibadmin --delete --xml-text '<primitive id=\"old\"/>'", pcmk_option_example},

	{"-spacer-", 0, 0, '-', "Remove all resources from the configuration:", pcmk_option_paragraph},
	{"-spacer-", 0, 0, '-', " cibadmin --replace --scope resources --xml-text '<resources/>'", pcmk_option_example},

	{"-spacer-", 0, 0, '-', "Replace the complete configuration with the contents of $HOME/pacemaker.xml:", pcmk_option_paragraph},
	{"-spacer-", 0, 0, '-', " cibadmin --replace --xml-file $HOME/pacemaker.xml", pcmk_option_example},

	{"-spacer-", 0, 0, '-', "Replace the constraints section of the configuration with the contents of $HOME/constraints.xml:", pcmk_option_paragraph},
	{"-spacer-", 0, 0, '-', " cibadmin --replace --scope constraints --xml-file $HOME/constraints.xml", pcmk_option_example},

	{"-spacer-", 0, 0, '-', "Increase the configuration version to prevent old configurations from being loaded accidentally:", pcmk_option_paragraph},
	{"-spacer-", 0, 0, '-', " cibadmin --modify --xml-text '<cib admin_epoch=\"admin_epoch++\"/>'", pcmk_option_example},

	{"-spacer-", 0, 0, '-', "Edit the configuration with your favorite $EDITOR:", pcmk_option_paragraph},
	{"-spacer-", 0, 0, '-', " cibadmin --query > $HOME/local.xml", pcmk_option_example},
	{"-spacer-", 0, 0, '-', " $EDITOR $HOME/local.xml", pcmk_option_example},
	{"-spacer-", 0, 0, '-', " cibadmin --replace --xml-file $HOME/local.xml", pcmk_option_example},

	{"-spacer-", 0, 0, '-', "SEE ALSO:"},
	{"-spacer-", 0, 0, '-', " CRM shell, crm(8), crm_shadow(8)"},

	/* Legacy options */
	{"host", 0, 0, 'h', NULL, 1},
	{"force-quorum", 0, 0, 'f', NULL, 1},
	{"obj_type", 1, 0, 'o', NULL, 1},
	{F_CRM_DATA, 1, 0, 'X', NULL, 1},
	{CIB_OP_ERASE, 0, 0, 'E', NULL, 1},
	{CIB_OP_QUERY, 0, 0, 'Q', NULL, 1},
	{CIB_OP_CREATE, 0, 0, 'C', NULL, 1},
	{CIB_OP_REPLACE, 0, 0, 'R', NULL, 1},
	{CIB_OP_UPDATE, 0, 0, 'U', NULL, 1},
	{CIB_OP_MODIFY, 0, 0, 'M', NULL, 1},
	{CIB_OP_DELETE, 0, 0, 'D', NULL, 1},
	{CIB_OP_BUMP, 0, 0, 'B', NULL, 1},
	{CIB_OP_SYNC, 0, 0, 'S', NULL, 1},
	{CIB_OP_SLAVE, 0, 0, 'r', NULL, 1},
	{CIB_OP_MASTER, 0, 0, 'w', NULL, 1},
	{CIB_OP_ISMASTER,0, 0, 'm', NULL, 1},

	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	int
	main(int argc, char **argv)
	{
	int argerr = 0;
	int flag;
	const char *source = NULL;
	char *admin_input_xml = NULL;
	char *admin_input_file = NULL;
	gboolean dangerous_cmd = FALSE;
	gboolean admin_input_stdin = FALSE;
	xmlNode *output = NULL;
	xmlNode *input = NULL;

	int option_index = 0;

	crm_log_init(NULL, LOG_CRIT, FALSE, FALSE, argc, argv, FALSE);
	crm_set_options(NULL, "command [options] [data]", long_options,
	"Provides direct access to the cluster configuration."
	"\n\nAllows the configuration, or sections of it, to be queried, modified, replaced and deleted."
	"\n\nWhere necessary, XML data will be obtained using the -X, -x, or -p options\n");

	if (argc < 2) {
	crm_help('?', EX_USAGE);
	}

	while (1) {
	flag = crm_get_option(argc, argv, &option_index);
	if (flag == -1)
	break;

	switch (flag) {
	case 't':
	message_timeout_ms = atoi(optarg);
	if (message_timeout_ms < 1) {
	message_timeout_ms = 30;
	}
	break;
	case 'A':
	obj_type = strdup(optarg);
	command_options \|= cib_xpath;
	break;
	case 'u':
	cib_action = CIB_OP_UPGRADE;
	dangerous_cmd = TRUE;
	break;
	case 'E':
	cib_action = CIB_OP_ERASE;
	dangerous_cmd = TRUE;
	break;
	case 'Q':
	cib_action = CIB_OP_QUERY;
	break;
	case 'P':
	cib_action = CIB_OP_APPLY_DIFF;
	break;
	case 'S':
	cib_action = CIB_OP_SYNC;
	break;
	case 'U':
	case 'M':
	cib_action = CIB_OP_MODIFY;
	break;
	case 'R':
	cib_action = CIB_OP_REPLACE;
	break;
	case 'C':
	cib_action = CIB_OP_CREATE;
	break;
	case 'D':
	cib_action = CIB_OP_DELETE;
	break;
	case '5':
	cib_action = "md5-sum";
	break;
	case '6':
	cib_action = "md5-sum-versioned";
	break;
	case 'c':
	command_options \|= cib_can_create;
	break;
	case 'n':
	command_options \|= cib_no_children;
	break;
	case 'm':
	cib_action = CIB_OP_ISMASTER;
	command_options \|= cib_scope_local;
	break;
	case 'B':
	cib_action = CIB_OP_BUMP;
	break;
	case 'r':
	dangerous_cmd = TRUE;
	cib_action = CIB_OP_SLAVE;
	break;
	case 'w':
	dangerous_cmd = TRUE;
	cib_action = CIB_OP_MASTER;
	command_options \|= cib_scope_local;
	break;
	case 'V':
	command_options = command_options \| cib_verbose;
	crm_bump_log_level();
	break;
	case '?':
	case '$':
	case '!':
	crm_help(flag, EX_OK);
	break;
	case 'o':
	crm_trace("Option %c => %s", flag, optarg);
	obj_type = strdup(optarg);
	break;
	case 'X':
	crm_trace("Option %c => %s", flag, optarg);
	admin_input_xml = strdup(optarg);
	break;
	case 'x':
	crm_trace("Option %c => %s", flag, optarg);
	admin_input_file = strdup(optarg);
	break;
	case 'p':
	admin_input_stdin = TRUE;
	break;
	case 'h':
	host = strdup(optarg);
	break;
	case 'l':
	command_options \|= cib_scope_local;
	break;
	case 'd':
	cib_action = CIB_OP_DELETE;
	command_options \|= cib_multiple;
	dangerous_cmd = TRUE;
	break;
	case 'b':
	dangerous_cmd = TRUE;
	command_options \|= cib_inhibit_bcast;
	command_options \|= cib_scope_local;
	break;
	case 's':
	command_options \|= cib_sync_call;
	break;
	case 'f':
	force_flag = TRUE;
	command_options \|= cib_quorum_override;
	break;
	case 'a':
	output = createEmptyCib();
	crm_xml_add(output, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	if (optind >= argc) {
	crm_xml_add(output, XML_ATTR_VALIDATION, LATEST_SCHEMA_VERSION);
	} else {
	crm_xml_add(output, XML_ATTR_VALIDATION, argv[optind]);
	}
	crm_xml_add_int(output, XML_ATTR_GENERATION_ADMIN, 1);
	crm_xml_add_int(output, XML_ATTR_GENERATION, 0);
	crm_xml_add_int(output, XML_ATTR_NUMUPDATES, 0);

	admin_input_xml = dump_xml_formatted(output);
	fprintf(stdout, "%s\n", crm_str(admin_input_xml));
	exit(0);
	break;
	default:
	printf("Argument code 0%o (%c)" " is not (?yet?) supported\n", flag, flag);
	++argerr;
	break;
	}
	}

	if (optind < argc) {
	printf("non-option ARGV-elements: ");
	while (optind < argc)
	printf("%s ", argv[optind++]);
	printf("\n");
	crm_help('?', EX_USAGE);
	}

	if (optind > argc \|\| cib_action == NULL) {
	++argerr;
	}

	if (argerr) {
	crm_help('?', EX_USAGE);
	}

	if (dangerous_cmd && force_flag == FALSE) {
	fprintf(stderr, "The supplied command is considered dangerous."
	" To prevent accidental destruction of the cluster,"
	" the --force flag is required in order to proceed.\n");
	fflush(stderr);
	exit(1);
	}

	if (admin_input_file != NULL) {
	input = filename2xml(admin_input_file);
	source = admin_input_file;

	} else if (admin_input_xml != NULL) {
	source = "input string";
	input = string2xml(admin_input_xml);

	} else if (admin_input_stdin) {
	source = "STDIN";
	input = stdin2xml();
	}

	if (input != NULL) {
	crm_log_xml_debug(input, "[admin input]");

	} else if (source) {
	fprintf(stderr, "Couldn't parse input from %s.\n", source);
	return 1;
	}

	if (safe_str_eq(cib_action, "md5-sum")) {
	char *digest = NULL;

	if (input == NULL) {
	fprintf(stderr, "Please supply XML to process with -X, -x or -p\n");
	exit(1);
	}

	digest = calculate_on_disk_digest(input);
	fprintf(stderr, "Digest: ");
	fprintf(stdout, "%s\n", crm_str(digest));
	free(digest);
	exit(0);

	} else if (safe_str_eq(cib_action, "md5-sum-versioned")) {
	char *digest = NULL;
	const char *version = NULL;

	if (input == NULL) {
	fprintf(stderr, "Please supply XML to process with -X, -x or -p\n");
	exit(1);
	}

	version = crm_element_value(input, XML_ATTR_CRM_VERSION);
	digest = calculate_xml_versioned_digest(input, FALSE, TRUE, version);
	fprintf(stderr, "Versioned (%s) digest: ", version);
	fprintf(stdout, "%s\n", crm_str(digest));
	free(digest);
	exit(0);
	}

	exit_code = do_init();
	if (exit_code != pcmk_ok) {
	crm_err("Init failed, could not perform requested operations");
	fprintf(stderr, "Init failed, could not perform requested operations\n");
	return -exit_code;
	}

	exit_code = do_work(input, command_options, &output);
	if (exit_code > 0) {
	/* wait for the reply by creating a mainloop and running it until
	* the callbacks are invoked...
	*/
	request_id = exit_code;

	the_cib->cmds->register_callback(the_cib, request_id, message_timeout_ms, FALSE, NULL,
	"cibadmin_op_callback", cibadmin_op_callback);

	mainloop = g_main_new(FALSE);

	crm_trace("%s waiting for reply from the local CIB", crm_system_name);

	crm_info("Starting mainloop");
	g_main_run(mainloop);

	} else if (exit_code < 0) {
	crm_err("Call failed: %s", pcmk_strerror(exit_code));
	fprintf(stderr, "Call failed: %s\n", pcmk_strerror(exit_code));
	operation_status = exit_code;

	if (exit_code == -pcmk_err_dtd_validation) {
	if (crm_str_eq(cib_action, CIB_OP_UPGRADE, TRUE)) {
	xmlNode *obj = NULL;
	int version = 0, rc = 0;

	rc = the_cib->cmds->query(the_cib, NULL, &obj, command_options);
	if (rc == pcmk_ok) {
	update_validation(&obj, &version, TRUE, FALSE);
	}

	} else if (output) {
	validate_xml_verbose(output);
	}
	}
	}

	if (output != NULL) {
	char *buffer = dump_xml_formatted(output);

	fprintf(stdout, "%s\n", crm_str(buffer));
	free(buffer);
	free_xml(output);
	}

	crm_trace("%s exiting normally", crm_system_name);

	free_xml(input);
	free(admin_input_xml);
	free(admin_input_file);
	the_cib->cmds->signoff(the_cib);
	cib_delete(the_cib);
	crm_xml_cleanup();

	return -exit_code;
	}

	int
	do_work(xmlNode * input, int call_options, xmlNode ** output)
	{
	/* construct the request */
	the_cib->call_timeout = message_timeout_ms;
	if (strcasecmp(CIB_OP_REPLACE, cib_action) == 0
	&& safe_str_eq(crm_element_name(input), XML_TAG_CIB)) {
	xmlNode *status = get_object_root(XML_CIB_TAG_STATUS, input);

	if (status == NULL) {
	create_xml_node(input, XML_CIB_TAG_STATUS);
	}
	}

	if (strcasecmp(CIB_OP_SYNC, cib_action) == 0) {
	crm_trace("Performing %s op...", cib_action);
	return the_cib->cmds->sync_from(the_cib, host, obj_type, call_options);

	} else if (strcasecmp(CIB_OP_SLAVE, cib_action) == 0 && (call_options ^ cib_scope_local)) {
	crm_trace("Performing %s op on all nodes...", cib_action);
	return the_cib->cmds->set_slave_all(the_cib, call_options);

	} else if (strcasecmp(CIB_OP_MASTER, cib_action) == 0) {
	crm_trace("Performing %s op on all nodes...", cib_action);
	return the_cib->cmds->set_master(the_cib, call_options);

	} else if (cib_action != NULL) {
	crm_trace("Passing \"%s\" to variant_op...", cib_action);
	- return the_cib->cmds->variant_op(the_cib, cib_action, host, obj_type,
	- input, output, call_options);
	+ return cib_internal_op(the_cib, cib_action, host, obj_type, input, output, call_options, NULL);

	} else {
	crm_err("You must specify an operation");
	}
	return -EINVAL;
	}

	int
	do_init(void)
	{
	int rc = pcmk_ok;

	the_cib = cib_new();
	rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command);
	if (rc != pcmk_ok) {
	crm_err("Signon to CIB failed: %s", pcmk_strerror(rc));
	fprintf(stderr, "Signon to CIB failed: %s\n", pcmk_strerror(rc));
	}

	return rc;
	}

	void
	cib_connection_destroy(gpointer user_data)
	{
	crm_err("Connection to the CIB terminated... exiting");
	g_main_quit(mainloop);
	return;
	}

	void
	cibadmin_op_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	char *admin_input_xml = NULL;

	exit_code = rc;

	if (output != NULL) {
	admin_input_xml = dump_xml_formatted(output);
	}

	if (safe_str_eq(cib_action, CIB_OP_ISMASTER) && rc != pcmk_ok) {
	crm_info("CIB on %s is _not_ the master instance", host ? host : "localhost");
	fprintf(stderr, "CIB on %s is _not_ the master instance\n", host ? host : "localhost");

	} else if (safe_str_eq(cib_action, CIB_OP_ISMASTER)) {
	crm_info("CIB on %s _is_ the master instance", host ? host : "localhost");
	fprintf(stderr, "CIB on %s _is_ the master instance\n", host ? host : "localhost");

	} else if (rc != 0) {
	crm_warn("Call %s failed (%d): %s", cib_action, rc, pcmk_strerror(rc));
	fprintf(stderr, "Call %s failed (%d): %s\n", cib_action, rc, pcmk_strerror(rc));
	fprintf(stdout, "%s\n", crm_str(admin_input_xml));

	} else if (safe_str_eq(cib_action, CIB_OP_QUERY) && output == NULL) {
	crm_err("Output expected in query response");
	crm_log_xml_err(msg, "no output");

	} else if (output == NULL) {
	crm_info("Call passed");

	} else {
	crm_info("Call passed");
	fprintf(stdout, "%s\n", crm_str(admin_input_xml));
	}
	free(admin_input_xml);

	if (call_id == request_id) {
	g_main_quit(mainloop);

	} else {
	crm_info("Message was not the response we were looking for (%d vs. %d", call_id,
	request_id);
	}
	}

File Metadata

Mime Type: text/x-diff
Expires: Thu, Oct 16, 12:14 AM (1 d, 15 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2530753
Default Alt Text: (212 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions