No OneTemporary
Actions

Size

179 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h
	index c75d72410d..0b2b2edd0a 100644
	--- a/include/crm/msg_xml.h
	+++ b/include/crm/msg_xml.h
	@@ -1,278 +1,280 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#ifndef XML_TAGS__H
	#define XML_TAGS__H

	#define CIB_OPTIONS_FIRST "cib-bootstrap-options"

	#define F_CRM_DATA "crm_xml"
	#define F_CRM_TASK "crm_task"
	#define F_CRM_HOST_TO "crm_host_to"
	#define F_CRM_MSG_TYPE F_SUBTYPE
	#define F_CRM_SYS_TO "crm_sys_to"
	#define F_CRM_SYS_FROM "crm_sys_from"
	#define F_CRM_HOST_FROM F_ORIG
	#define F_CRM_REFERENCE XML_ATTR_REFERENCE
	#define F_CRM_VERSION XML_ATTR_VERSION
	#define F_CRM_ORIGIN "origin"
	#define F_CRM_JOIN_ID "join_id"
	#define F_CRM_ELECTION_ID "election-id"
	#define F_CRM_ELECTION_OWNER "election-owner"
	#define F_CRM_TGRAPH "crm-tgraph"
	#define F_CRM_TGRAPH_INPUT "crm-tgraph-in"

	/---- Common tags/attrs /
	#define XML_DIFF_MARKER "__crm_diff_marker__"
	#define XML_ATTR_TAGNAME F_XML_TAGNAME
	#define XML_TAG_CIB "cib"
	#define XML_TAG_FAILED "failed"

	#define XML_ATTR_CRM_VERSION "crm_feature_set"
	#define XML_ATTR_DIGEST "digest"
	#define XML_ATTR_VALIDATION "validate-with"

	#define XML_ATTR_QUORUM_PANIC "no-quorum-panic"
	#define XML_ATTR_HAVE_QUORUM "have-quorum"
	#define XML_ATTR_EXPECTED_VOTES "expected-quorum-votes"
	#define XML_ATTR_GENERATION "epoch"
	#define XML_ATTR_GENERATION_ADMIN "admin_epoch"
	#define XML_ATTR_NUMUPDATES "num_updates"
	#define XML_ATTR_TIMEOUT "timeout"
	#define XML_ATTR_ORIGIN "crm-debug-origin"
	#define XML_ATTR_TSTAMP "crm-timestamp"
	#define XML_CIB_ATTR_WRITTEN "cib-last-written"
	#define XML_ATTR_VERSION "version"
	#define XML_ATTR_DESC "description"
	#define XML_ATTR_ID "id"
	#define XML_ATTR_IDREF "id-ref"
	#define XML_ATTR_ID_LONG "long-id"
	#define XML_ATTR_TYPE "type"
	#define XML_ATTR_FILTER_TYPE "type-filter"
	#define XML_ATTR_FILTER_ID "id-filter"
	#define XML_ATTR_FILTER_PRIORITY "priority-filter"
	#define XML_ATTR_VERBOSE "verbose"
	#define XML_ATTR_OP "op"
	#define XML_ATTR_DC "is_dc"
	#define XML_ATTR_DC_UUID "dc-uuid"

	#define XML_BOOLEAN_TRUE "true"
	#define XML_BOOLEAN_FALSE "false"
	#define XML_BOOLEAN_YES XML_BOOLEAN_TRUE
	#define XML_BOOLEAN_NO XML_BOOLEAN_FALSE

	#define XML_TAG_OPTIONS "options"

	/---- top level tags/attrs /
	#define XML_MSG_TAG "crm_message"
	#define XML_MSG_TAG_DATA "msg_data"
	#define XML_ATTR_REQUEST "request"
	#define XML_ATTR_RESPONSE "response"

	#define XML_ATTR_UNAME "uname"
	#define XML_ATTR_UUID "id"
	#define XML_ATTR_REFERENCE "reference"

	#define XML_FAIL_TAG_RESOURCE "failed_resource"

	#define XML_FAILRES_ATTR_RESID "resource_id"
	#define XML_FAILRES_ATTR_REASON "reason"
	#define XML_FAILRES_ATTR_RESSTATUS "resource_status"

	#define XML_CRM_TAG_PING "ping_response"
	#define XML_PING_ATTR_STATUS "result"
	#define XML_PING_ATTR_SYSFROM "crm_subsystem"

	#define XML_TAG_FRAGMENT "cib_fragment"
	#define XML_ATTR_RESULT "result"
	#define XML_ATTR_SECTION "section"

	#define XML_FAIL_TAG_CIB "failed_update"

	#define XML_FAILCIB_ATTR_ID "id"
	#define XML_FAILCIB_ATTR_OBJTYPE "object_type"
	#define XML_FAILCIB_ATTR_OP "operation"
	#define XML_FAILCIB_ATTR_REASON "reason"

	/---- CIB specific tags/attrs /
	#define XML_CIB_TAG_SECTION_ALL "all"
	#define XML_CIB_TAG_CONFIGURATION "configuration"
	#define XML_CIB_TAG_STATUS "status"
	#define XML_CIB_TAG_RESOURCES "resources"
	#define XML_CIB_TAG_NODES "nodes"
	+#define XML_CIB_TAG_DOMAINS "domains"
	#define XML_CIB_TAG_CONSTRAINTS "constraints"
	#define XML_CIB_TAG_CRMCONFIG "crm_config"
	#define XML_CIB_TAG_OPCONFIG "op_defaults"
	#define XML_CIB_TAG_RSCCONFIG "rsc_defaults"

	#define XML_CIB_TAG_STATE "node_state"
	#define XML_CIB_TAG_NODE "node"
	+#define XML_CIB_TAG_DOMAIN "domain"
	#define XML_CIB_TAG_CONSTRAINT "constraint"
	#define XML_CIB_TAG_NVPAIR "nvpair"

	#define XML_CIB_TAG_PROPSET "cluster_property_set"
	#define XML_TAG_ATTR_SETS "instance_attributes"
	#define XML_TAG_META_SETS "meta_attributes"
	#define XML_TAG_ATTRS "attributes"
	#define XML_TAG_PARAMS "parameters"
	#define XML_TAG_PARAM "param"
	#define XML_TAG_UTILIZATION "utilization"

	#define XML_TAG_RESOURCE_REF "resource_ref"
	#define XML_CIB_TAG_RESOURCE "primitive"
	#define XML_CIB_TAG_GROUP "group"
	#define XML_CIB_TAG_INCARNATION "clone"
	#define XML_CIB_TAG_MASTER "master"

	#define XML_RSC_ATTR_RESTART "restart-type"
	#define XML_RSC_ATTR_ORDERED "ordered"
	#define XML_RSC_ATTR_INTERLEAVE "interleave"
	#define XML_RSC_ATTR_INCARNATION "clone"
	#define XML_RSC_ATTR_INCARNATION_MAX "clone-max"
	#define XML_RSC_ATTR_INCARNATION_NODEMAX "clone-node-max"
	#define XML_RSC_ATTR_MASTER_MAX "master-max"
	#define XML_RSC_ATTR_MASTER_NODEMAX "master-node-max"
	#define XML_RSC_ATTR_STATE "clone-state"
	#define XML_RSC_ATTR_MANAGED "is-managed"
	#define XML_RSC_ATTR_TARGET_ROLE "target-role"
	#define XML_RSC_ATTR_UNIQUE "globally-unique"
	#define XML_RSC_ATTR_NOTIFY "notify"
	#define XML_RSC_ATTR_STICKINESS "resource-stickiness"
	#define XML_RSC_ATTR_FAIL_STICKINESS "migration-threshold"
	#define XML_RSC_ATTR_FAIL_TIMEOUT "failure-timeout"
	#define XML_RSC_ATTR_MULTIPLE "multiple-active"
	#define XML_RSC_ATTR_PRIORITY "priority"
	#define XML_OP_ATTR_ON_FAIL "on-fail"
	#define XML_OP_ATTR_START_DELAY "start-delay"
	#define XML_OP_ATTR_ALLOW_MIGRATE "allow-migrate"
	#define XML_OP_ATTR_ORIGIN "interval-origin"
	#define XML_OP_ATTR_PENDING "record-pending"

	#define XML_CIB_TAG_LRM "lrm"
	#define XML_LRM_TAG_RESOURCES "lrm_resources"
	#define XML_LRM_TAG_RESOURCE "lrm_resource"
	#define XML_LRM_TAG_AGENTS "lrm_agents"
	#define XML_LRM_TAG_AGENT "lrm_agent"
	#define XML_LRM_TAG_RSC_OP "lrm_rsc_op"
	#define XML_AGENT_ATTR_CLASS "class"
	#define XML_AGENT_ATTR_PROVIDER "provider"
	#define XML_LRM_TAG_ATTRIBUTES "attributes"

	#define XML_CIB_ATTR_REPLACE "replace"
	#define XML_CIB_ATTR_SOURCE "source"

	#define XML_CIB_ATTR_HEALTH "health"
	#define XML_CIB_ATTR_WEIGHT "weight"
	#define XML_CIB_ATTR_PRIORITY "priority"
	#define XML_CIB_ATTR_CLEAR "clear_on"
	#define XML_CIB_ATTR_SOURCE "source"

	#define XML_CIB_ATTR_JOINSTATE "join"
	#define XML_CIB_ATTR_EXPSTATE "expected"
	#define XML_CIB_ATTR_INCCM "in_ccm"
	#define XML_CIB_ATTR_CRMDSTATE "crmd"
	#define XML_CIB_ATTR_HASTATE "ha"

	#define XML_CIB_ATTR_SHUTDOWN "shutdown"
	#define XML_CIB_ATTR_STONITH "stonith"

	#define XML_LRM_ATTR_INTERVAL "interval"
	#define XML_LRM_ATTR_TASK "operation"
	#define XML_LRM_ATTR_TASK_KEY "operation_key"
	#define XML_LRM_ATTR_TARGET "on_node"
	#define XML_LRM_ATTR_TARGET_UUID "on_node_uuid"
	#define XML_LRM_ATTR_RSCID "rsc-id"
	#define XML_LRM_ATTR_OPSTATUS "op-status"
	#define XML_LRM_ATTR_RC "rc-code"
	#define XML_LRM_ATTR_CALLID "call-id"
	#define XML_LRM_ATTR_OP_DIGEST "op-digest"
	#define XML_LRM_ATTR_OP_RESTART "op-force-restart"
	#define XML_LRM_ATTR_RESTART_DIGEST "op-restart-digest"

	#define XML_TAG_GRAPH "transition_graph"
	#define XML_GRAPH_TAG_RSC_OP "rsc_op"
	#define XML_GRAPH_TAG_PSEUDO_EVENT "pseudo_event"
	#define XML_GRAPH_TAG_CRM_EVENT "crm_event"

	#define XML_TAG_RULE "rule"
	#define XML_RULE_ATTR_SCORE "score"
	#define XML_RULE_ATTR_SCORE_ATTRIBUTE "score-attribute"
	#define XML_RULE_ATTR_SCORE_MANGLED "score-attribute-mangled"
	#define XML_RULE_ATTR_ROLE "role"
	#define XML_RULE_ATTR_RESULT "result"
	#define XML_RULE_ATTR_BOOLEAN_OP "boolean-op"

	#define XML_TAG_EXPRESSION "expression"
	#define XML_EXPR_ATTR_ATTRIBUTE "attribute"
	#define XML_EXPR_ATTR_OPERATION "operation"
	#define XML_EXPR_ATTR_VALUE "value"
	#define XML_EXPR_ATTR_TYPE "type"

	#define XML_CONS_TAG_RSC_DEPEND "rsc_colocation"
	#define XML_CONS_TAG_RSC_ORDER "rsc_order"
	#define XML_CONS_TAG_RSC_LOCATION "rsc_location"
	#define XML_CONS_TAG_RSC_SET "resource_set"
	#define XML_CONS_ATTR_SYMMETRICAL "symmetrical"

	#define XML_COLOC_ATTR_SOURCE "rsc"
	#define XML_COLOC_ATTR_SOURCE_ROLE "rsc-role"
	#define XML_COLOC_ATTR_TARGET "with-rsc"
	#define XML_COLOC_ATTR_TARGET_ROLE "with-rsc-role"
	#define XML_COLOC_ATTR_NODE_ATTR "node-attribute"
	#define XML_COLOC_ATTR_SOURCE_INSTANCE "rsc-instance"
	#define XML_COLOC_ATTR_TARGET_INSTANCE "with-rsc-instance"

	#define XML_ORDER_ATTR_FIRST "first"
	#define XML_ORDER_ATTR_THEN "then"
	#define XML_ORDER_ATTR_FIRST_ACTION "first-action"
	#define XML_ORDER_ATTR_THEN_ACTION "then-action"
	#define XML_ORDER_ATTR_FIRST_INSTANCE "first-instance"
	#define XML_ORDER_ATTR_THEN_INSTANCE "then-instance"
	#define XML_ORDER_ATTR_KIND "kind"

	#define XML_NVPAIR_ATTR_NAME "name"
	#define XML_NVPAIR_ATTR_VALUE "value"

	#define XML_NODE_ATTR_STATE "state"

	#define XML_CONFIG_ATTR_DC_DEADTIME "dc-deadtime"
	#define XML_CONFIG_ATTR_ELECTION_FAIL "election-timeout"
	#define XML_CONFIG_ATTR_FORCE_QUIT "shutdown-escalation"
	#define XML_CONFIG_ATTR_RECHECK "cluster-recheck-interval"

	#define XML_CIB_TAG_GENERATION_TUPPLE "generation_tuple"

	#define XML_ATTR_TRANSITION_MAGIC "transition-magic"
	#define XML_ATTR_TRANSITION_KEY "transition-key"

	#define XML_ATTR_TE_NOWAIT "op_no_wait"
	#define XML_ATTR_TE_TARGET_RC "op_target_rc"
	#define XML_ATTR_TE_ALLOWFAIL "op_allow_fail"
	#define XML_ATTR_LRM_PROBE "lrm-is-probe"
	#define XML_TAG_TRANSIENT_NODEATTRS "transient_attributes"

	#define XML_TAG_DIFF_ADDED "diff-added"
	#define XML_TAG_DIFF_REMOVED "diff-removed"

	#include <crm/common/xml.h>

	#define ID(x) crm_element_value(x, XML_ATTR_ID)
	#define INSTANCE(x) crm_element_value(x, XML_CIB_ATTR_INSTANCE)
	#define TSTAMP(x) crm_element_value(x, XML_ATTR_TSTAMP)
	#define TYPE(x) crm_element_name(x)

	#endif
	diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h
	index b16729af3f..8396807348 100644
	--- a/include/crm/pengine/status.h
	+++ b/include/crm/pengine/status.h
	@@ -1,271 +1,272 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#ifndef PENGINE_STATUS__H
	#define PENGINE_STATUS__H

	#include <glib.h>
	#include <crm/common/iso8601.h>
	#include <crm/pengine/common.h>

	typedef struct node_s node_t;
	typedef struct action_s action_t;
	typedef struct action_s pe_action_t;
	typedef struct resource_s resource_t;

	typedef enum no_quorum_policy_e {
	no_quorum_freeze,
	no_quorum_stop,
	no_quorum_ignore,
	no_quorum_suicide
	} no_quorum_policy_t;

	enum node_type {
	node_ping,
	node_member
	};

	enum pe_restart {
	pe_restart_restart,
	pe_restart_ignore
	};

	#define pe_flag_have_quorum 0x00000001ULL
	#define pe_flag_symmetric_cluster 0x00000002ULL
	#define pe_flag_is_managed_default 0x00000004ULL
	#define pe_flag_maintenance_mode 0x00000008ULL

	#define pe_flag_stonith_enabled 0x00000010ULL
	#define pe_flag_have_stonith_resource 0x00000020ULL

	#define pe_flag_stop_rsc_orphans 0x00000100ULL
	#define pe_flag_stop_action_orphans 0x00000200ULL
	#define pe_flag_stop_everything 0x00000400ULL

	#define pe_flag_start_failure_fatal 0x00001000ULL
	#define pe_flag_remove_after_stop 0x00002000ULL

	#define pe_flag_startup_probes 0x00010000ULL

	typedef struct pe_working_set_s
	{
	xmlNode *input;
	ha_time_t *now;

	/* options extracted from the input */
	char *dc_uuid;
	node_t *dc_node;
	const char *stonith_action;
	const char *placement_strategy;

	unsigned long long flags;

	int stonith_timeout;
	int default_resource_stickiness;
	no_quorum_policy_t no_quorum_policy;

	GHashTable *config_hash;
	+ GHashTable *domains;

	GListPtr nodes;
	GListPtr resources;
	GListPtr placement_constraints;
	GListPtr ordering_constraints;
	GListPtr colocation_constraints;

	GListPtr actions;
	xmlNode *failed;
	xmlNode *op_defaults;
	xmlNode *rsc_defaults;

	/* stats */
	int num_synapse;
	int max_valid_nodes;
	int order_id;
	int action_id;

	/* final output */
	xmlNode *graph;

	} pe_working_set_t;

	struct node_shared_s {
	const char *id;
	const char *uname;
	gboolean online;
	gboolean standby;
	gboolean standby_onfail;
	gboolean pending;
	gboolean unclean;
	gboolean shutdown;
	gboolean expected_up;
	gboolean is_dc;
	int num_resources;
	GListPtr running_rsc; /* resource_t* */
	GListPtr allocated_rsc; /* resource_t* */

	GHashTable attrs; / char* => char* */
	enum node_type type;

	GHashTable *utilization;
	};

	struct node_s {
	int weight;
	gboolean fixed;
	int count;
	struct node_shared_s *details;
	};

	#include <crm/pengine/complex.h>

	#define pe_rsc_orphan 0x00000001ULL
	#define pe_rsc_managed 0x00000002ULL

	#define pe_rsc_notify 0x00000010ULL
	#define pe_rsc_unique 0x00000020ULL
	#define pe_rsc_can_migrate 0x00000040ULL

	#define pe_rsc_provisional 0x00000100ULL
	#define pe_rsc_allocating 0x00000200ULL
	#define pe_rsc_merging 0x00000400ULL

	#define pe_rsc_failed 0x00010000ULL
	#define pe_rsc_shutdown 0x00020000ULL
	#define pe_rsc_runnable 0x00040000ULL
	#define pe_rsc_start_pending 0x00080000ULL

	#define pe_rsc_starting 0x00100000ULL
	#define pe_rsc_stopping 0x00200000ULL

	struct resource_s {
	char *id;
	char *clone_name;
	char *long_name;
	xmlNode *xml;
	xmlNode *ops_xml;

	resource_t *parent;
	void *variant_opaque;
	enum pe_obj_types variant;
	resource_object_functions_t *fns;
	resource_alloc_functions_t *cmds;

	enum rsc_recovery_type recovery_type;
	enum pe_restart restart_type;

	int priority;
	int stickiness;
	int sort_index;
	int failure_timeout;
	int effective_priority;
	int migration_threshold;

	unsigned long long flags;

	GListPtr rsc_cons_lhs; /* rsc_colocation_t* */
	GListPtr rsc_cons; /* rsc_colocation_t* */
	GListPtr rsc_location; /* rsc_to_node_t* */
	GListPtr actions; /* action_t* */

	node_t *allocated_to;
	GListPtr running_on; /* node_t* */
	GListPtr known_on; /* node_t* */
	GListPtr allowed_nodes; /* node_t* */

	enum rsc_role_e role;
	enum rsc_role_e next_role;

	GHashTable *meta;
	GHashTable *parameters;
	GHashTable *utilization;

	GListPtr children; /* resource_t* */
	};

	struct action_s
	{
	int id;
	int priority;
	resource_t *rsc;
	void *rsc_opaque;
	node_t *node;
	char *task;

	char *uuid;
	xmlNode *op_entry;

	gboolean pseudo;
	gboolean runnable;
	gboolean optional;
	gboolean print_always;
	gboolean have_node_attrs;
	gboolean failure_is_fatal;
	gboolean implied_by_stonith;
	gboolean allow_reload_conversion;

	enum rsc_start_requirement needs;
	enum action_fail_response on_fail;
	enum rsc_role_e fail_role;

	gboolean dumped;
	gboolean processed;

	action_t *pre_notify;
	action_t *pre_notified;
	action_t *post_notify;
	action_t *post_notified;

	int seen_count;

	GHashTable *meta;
	GHashTable *extra;

	GListPtr actions_before; /* action_warpper_t* */
	GListPtr actions_after; /* action_warpper_t* */
	};

	typedef struct notify_data_s {
	GHashTable *keys;

	const char *action;

	action_t *pre;
	action_t *post;
	action_t *pre_done;
	action_t *post_done;

	GListPtr active; /* notify_entry_t* */
	GListPtr inactive; /* notify_entry_t* */
	GListPtr start; /* notify_entry_t* */
	GListPtr stop; /* notify_entry_t* */
	GListPtr demote; /* notify_entry_t* */
	GListPtr promote; /* notify_entry_t* */
	GListPtr master; /* notify_entry_t* */
	GListPtr slave; /* notify_entry_t* */

	} notify_data_t;

	gboolean cluster_status(pe_working_set_t *data_set);
	extern void set_working_set_defaults(pe_working_set_t *data_set);
	extern void cleanup_calculations(pe_working_set_t *data_set);
	extern resource_t pe_find_resource(GListPtr rsc_list, const char id_rh);
	extern node_t pe_find_node(GListPtr node_list, const char uname);
	extern node_t pe_find_node_id(GListPtr node_list, const char id);
	extern GListPtr find_operations(
	const char rsc, const char node, gboolean active_filter, pe_working_set_t *data_set);

	#endif
	diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c
	index 71d6a2f5cd..20c9ed49d2 100644
	--- a/lib/cib/cib_utils.c
	+++ b/lib/cib/cib_utils.c
	@@ -1,894 +1,895 @@
	/*
	* Copyright (c) 2004 International Business Machines
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*
	*/
	#include <crm_internal.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <sys/utsname.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <lib/cib/cib_private.h>

	struct config_root_s
	{
	const char *name;
	const char *parent;
	const char *path;
	};

	/*
	* "//crm_config" will also work in place of "/cib/configuration/crm_config"
	* The / prefix means find starting from the root, whereas the // prefix means
	* find anywhere and risks multiple matches
	*/
	struct config_root_s known_paths[] = {
	{ NULL, NULL, "//cib" },
	{ XML_TAG_CIB, NULL, "//cib" },
	{ XML_CIB_TAG_STATUS, "/cib", "//cib/status" },
	{ XML_CIB_TAG_CONFIGURATION,"/cib", "//cib/configuration" },
	{ XML_CIB_TAG_CRMCONFIG, "/cib/configuration", "//cib/configuration/crm_config" },
	{ XML_CIB_TAG_NODES, "/cib/configuration", "//cib/configuration/nodes" },
	+ { XML_CIB_TAG_DOMAINS, "/cib/configuration", "//cib/configuration/domains" },
	{ XML_CIB_TAG_RESOURCES, "/cib/configuration", "//cib/configuration/resources" },
	{ XML_CIB_TAG_CONSTRAINTS, "/cib/configuration", "//cib/configuration/constraints" },
	{ XML_CIB_TAG_OPCONFIG, "/cib/configuration", "//cib/configuration/op_defaults" },
	{ XML_CIB_TAG_RSCCONFIG, "/cib/configuration", "//cib/configuration/rsc_defaults" },
	{ XML_CIB_TAG_SECTION_ALL, NULL, "//cib" },
	};

	const char *
	cib_error2string(enum cib_errors return_code)
	{
	const char *error_msg = NULL;
	switch(return_code) {
	case cib_bad_permissions:
	error_msg = "bad permissions for the on-disk configuration. shutdown heartbeat and repair.";
	break;
	case cib_bad_digest:
	error_msg = "the on-disk configuration was manually altered. shutdown heartbeat and repair.";
	break;
	case cib_bad_config:
	error_msg = "the on-disk configuration is not valid";
	break;
	case cib_msg_field_add:
	error_msg = "failed adding field to cib message";
	break;
	case cib_id_check:
	error_msg = "missing id or id-collision detected";
	break;
	case cib_operation:
	error_msg = "invalid operation";
	break;
	case cib_create_msg:
	error_msg = "couldnt create cib message";
	break;
	case cib_client_gone:
	error_msg = "client left before we could send reply";
	break;
	case cib_not_connected:
	error_msg = "not connected";
	break;
	case cib_not_authorized:
	error_msg = "not authorized";
	break;
	case cib_send_failed:
	error_msg = "send failed";
	break;
	case cib_reply_failed:
	error_msg = "reply failed";
	break;
	case cib_return_code:
	error_msg = "no return code";
	break;
	case cib_output_ptr:
	error_msg = "nowhere to store output";
	break;
	case cib_output_data:
	error_msg = "corrupt output data";
	break;
	case cib_connection:
	error_msg = "connection failed";
	break;
	case cib_callback_register:
	error_msg = "couldnt register callback channel";
	break;
	case cib_authentication:
	error_msg = "";
	break;
	case cib_registration_msg:
	error_msg = "invalid registration msg";
	break;
	case cib_callback_token:
	error_msg = "callback token not found";
	break;
	case cib_missing:
	error_msg = "cib object missing";
	break;
	case cib_variant:
	error_msg = "unknown/corrupt cib variant";
	break;
	case CIBRES_MISSING_ID:
	error_msg = "The id field is missing";
	break;
	case CIBRES_MISSING_TYPE:
	error_msg = "The type field is missing";
	break;
	case CIBRES_MISSING_FIELD:
	error_msg = "A required field is missing";
	break;
	case CIBRES_OBJTYPE_MISMATCH:
	error_msg = "CIBRES_OBJTYPE_MISMATCH";
	break;
	case cib_EXISTS:
	error_msg = "The object already exists";
	break;
	case cib_NOTEXISTS:
	error_msg = "The object/attribute does not exist";
	break;
	case CIBRES_CORRUPT:
	error_msg = "The CIB is corrupt";
	break;
	case cib_NOOBJECT:
	error_msg = "The update was empty";
	break;
	case cib_NOPARENT:
	error_msg = "The parent object does not exist";
	break;
	case cib_NODECOPY:
	error_msg = "Failed while copying update";
	break;
	case CIBRES_OTHER:
	error_msg = "CIBRES_OTHER";
	break;
	case cib_ok:
	error_msg = "ok";
	break;
	case cib_unknown:
	error_msg = "Unknown error";
	break;
	case cib_STALE:
	error_msg = "Discarded old update";
	break;
	case cib_ACTIVATION:
	error_msg = "Activation Failed";
	break;
	case cib_NOSECTION:
	error_msg = "Required section was missing";
	break;
	case cib_NOTSUPPORTED:
	error_msg = "The action/feature is not supported";
	break;
	case cib_not_master:
	error_msg = "Local service is not the master instance";
	break;
	case cib_client_corrupt:
	error_msg = "Service client not valid";
	break;
	case cib_remote_timeout:
	error_msg = "Remote node did not respond";
	break;
	case cib_master_timeout:
	error_msg = "No master service is currently active";
	break;
	case cib_revision_unsupported:
	error_msg = "The required CIB revision number is not supported";
	break;
	case cib_revision_unknown:
	error_msg = "The CIB revision number could not be determined";
	break;
	case cib_missing_data:
	error_msg = "Required data for this CIB API call not found";
	break;
	case cib_no_quorum:
	error_msg = "Write requires quorum";
	break;
	case cib_diff_failed:
	error_msg = "Application of an update diff failed";
	break;
	case cib_diff_resync:
	error_msg = "Application of an update diff failed, requesting a full refresh";
	break;
	case cib_bad_section:
	error_msg = "Invalid CIB section specified";
	break;
	case cib_old_data:
	error_msg = "Update was older than existing configuration";
	break;
	case cib_dtd_validation:
	error_msg = "Update does not conform to the configured schema/DTD";
	break;
	case cib_invalid_argument:
	error_msg = "Invalid argument";
	break;
	case cib_transform_failed:
	error_msg = "Schema transform failed";
	break;
	}

	if(error_msg == NULL) {
	crm_err("Unknown CIB Error Code: %d", return_code);
	error_msg = "<unknown error>";
	}

	return error_msg;
	}

	int
	cib_section2enum(const char *a_section)
	{
	if(a_section == NULL \|\| strcasecmp(a_section, "all") == 0) {
	return cib_section_all;

	} else if(strcasecmp(a_section, XML_CIB_TAG_NODES) == 0) {
	return cib_section_nodes;

	} else if(strcasecmp(a_section, XML_CIB_TAG_STATUS) == 0) {
	return cib_section_status;

	} else if(strcasecmp(a_section, XML_CIB_TAG_CONSTRAINTS) == 0) {
	return cib_section_constraints;

	} else if(strcasecmp(a_section, XML_CIB_TAG_RESOURCES) == 0) {
	return cib_section_resources;

	} else if(strcasecmp(a_section, XML_CIB_TAG_CRMCONFIG) == 0) {
	return cib_section_crmconfig;

	}
	crm_err("Unknown CIB section: %s", a_section);
	return cib_section_none;
	}


	int
	cib_compare_generation(xmlNode left, xmlNode right)
	{
	int lpc = 0;
	const char *attributes[] = {
	XML_ATTR_GENERATION_ADMIN,
	XML_ATTR_GENERATION,
	XML_ATTR_NUMUPDATES,
	};

	crm_log_xml_debug_3(left, "left");
	crm_log_xml_debug_3(right, "right");

	for(lpc = 0; lpc < DIMOF(attributes); lpc++) {
	int int_elem_l = -1;
	int int_elem_r = -1;
	const char *elem_r = NULL;
	const char *elem_l = crm_element_value(left, attributes[lpc]);

	if(right != NULL) {
	elem_r = crm_element_value(right, attributes[lpc]);
	}

	if(elem_l != NULL) { int_elem_l = crm_parse_int(elem_l, NULL); }
	if(elem_r != NULL) { int_elem_r = crm_parse_int(elem_r, NULL); }

	if(int_elem_l < int_elem_r) {
	crm_debug_2("%s (%s < %s)", attributes[lpc],
	crm_str(elem_l), crm_str(elem_r));
	return -1;

	} else if(int_elem_l > int_elem_r) {
	crm_debug_2("%s (%s > %s)", attributes[lpc],
	crm_str(elem_l), crm_str(elem_r));
	return 1;
	}
	}

	return 0;
	}

	xmlNode*
	get_cib_copy(cib_t *cib)
	{
	xmlNode *xml_cib;
	int options = cib_scope_local\|cib_sync_call;
	if(cib->cmds->query(cib, NULL, &xml_cib, options) != cib_ok) {
	crm_err("Couldnt retrieve the CIB");
	return NULL;
	} else if(xml_cib == NULL) {
	crm_err("The CIB result was empty");
	return NULL;
	}

	if(safe_str_eq(crm_element_name(xml_cib), XML_TAG_CIB)) {
	return xml_cib;
	}
	free_xml(xml_cib);
	return NULL;
	}

	xmlNode*
	cib_get_generation(cib_t *cib)
	{
	xmlNode *the_cib = get_cib_copy(cib);
	xmlNode *generation = create_xml_node(
	NULL, XML_CIB_TAG_GENERATION_TUPPLE);

	if(the_cib != NULL) {
	copy_in_properties(generation, the_cib);
	free_xml(the_cib);
	}

	return generation;
	}


	void
	log_cib_diff(int log_level, xmlNode diff, const char function)
	{
	int add_updates = 0;
	int add_epoch = 0;
	int add_admin_epoch = 0;

	int del_updates = 0;
	int del_epoch = 0;
	int del_admin_epoch = 0;

	if(diff == NULL) {
	return;
	}

	cib_diff_version_details(
	diff, &add_admin_epoch, &add_epoch, &add_updates,
	&del_admin_epoch, &del_epoch, &del_updates);

	if(add_updates != del_updates) {
	do_crm_log(log_level, "%s: Diff: --- %d.%d.%d", function,
	del_admin_epoch, del_epoch, del_updates);
	do_crm_log(log_level, "%s: Diff: +++ %d.%d.%d", function,
	add_admin_epoch, add_epoch, add_updates);
	} else if(diff != NULL) {
	do_crm_log(log_level,
	"%s: Local-only Change: %d.%d.%d", function,
	add_admin_epoch, add_epoch, add_updates);
	}

	log_xml_diff(log_level, diff, function);
	}

	gboolean
	cib_version_details(
	xmlNode cib, int admin_epoch, int epoch, int updates)
	{
	if(cib == NULL) {
	*admin_epoch = -1;
	*epoch = -1;
	*updates = -1;
	return FALSE;

	} else {
	crm_element_value_int(cib, XML_ATTR_GENERATION, epoch);
	crm_element_value_int(cib, XML_ATTR_NUMUPDATES, updates);
	crm_element_value_int(cib, XML_ATTR_GENERATION_ADMIN, admin_epoch);
	}
	return TRUE;
	}

	gboolean
	cib_diff_version_details(
	xmlNode diff, int admin_epoch, int epoch, int updates,
	int _admin_epoch, int _epoch, int *_updates)
	{
	xmlNode *tmp = NULL;

	tmp = find_xml_node(diff, "diff-added", FALSE);
	cib_version_details(tmp, admin_epoch, epoch, updates);

	tmp = find_xml_node(diff, "diff-removed", FALSE);
	cib_version_details(tmp, _admin_epoch, _epoch, _updates);
	return TRUE;
	}

	/*
	* The caller should never free the return value
	*/

	const char get_object_path(const char object_type)
	{
	int lpc = 0;
	int max = DIMOF(known_paths);
	for(; lpc < max; lpc++) {
	if((object_type == NULL && known_paths[lpc].name == NULL)
	\|\| safe_str_eq(object_type, known_paths[lpc].name)) {
	return known_paths[lpc].path;
	}
	}
	return NULL;
	}

	const char get_object_parent(const char object_type)
	{
	int lpc = 0;
	int max = DIMOF(known_paths);
	for(; lpc < max; lpc++) {
	if(safe_str_eq(object_type, known_paths[lpc].name)) {
	return known_paths[lpc].parent;
	}
	}
	return NULL;
	}

	xmlNode*
	get_object_root(const char object_type, xmlNode the_root)
	{
	const char *xpath = get_object_path(object_type);

	if(xpath == NULL) {
	return the_root; /* or return NULL? */
	}

	return get_xpath_object(xpath, the_root, LOG_DEBUG_2);
	}

	xmlNode*
	create_cib_fragment_adv(
	xmlNode update, const char update_section, const char *source)
	{
	xmlNode *cib = NULL;
	gboolean whole_cib = FALSE;
	xmlNode *object_root = NULL;
	char *local_section = NULL;

	/* crm_debug("Creating a blank fragment: %s", update_section); */

	if(update == NULL && update_section == NULL) {
	crm_debug_3("Creating a blank fragment");
	update = createEmptyCib();
	crm_xml_add(cib, XML_ATTR_ORIGIN, source);
	return update;

	} else if(update == NULL) {
	crm_err("No update to create a fragment for");
	return NULL;

	}

	CRM_CHECK(update_section != NULL, return NULL);
	if(safe_str_eq(crm_element_name(update), XML_TAG_CIB)) {
	whole_cib = TRUE;
	}

	if(whole_cib == FALSE) {
	cib = createEmptyCib();
	crm_xml_add(cib, XML_ATTR_ORIGIN, source);
	object_root = get_object_root(update_section, cib);
	add_node_copy(object_root, update);

	} else {
	cib = copy_xml(update);
	crm_xml_add(cib, XML_ATTR_ORIGIN, source);
	}

	crm_free(local_section);
	crm_debug_3("Verifying created fragment");
	return cib;
	}

	/*
	* It is the callers responsibility to free both the new CIB (output)
	* and the new CIB (input)
	*/
	xmlNode*
	createEmptyCib(void)
	{
	xmlNode cib_root = NULL, config = NULL, *status = NULL;

	cib_root = create_xml_node(NULL, XML_TAG_CIB);

	config = create_xml_node(cib_root, XML_CIB_TAG_CONFIGURATION);
	status = create_xml_node(cib_root, XML_CIB_TAG_STATUS);

	/* crm_xml_add(cib_root, "version", "1"); */
	create_xml_node(config, XML_CIB_TAG_CRMCONFIG);
	create_xml_node(config, XML_CIB_TAG_NODES);
	create_xml_node(config, XML_CIB_TAG_RESOURCES);
	create_xml_node(config, XML_CIB_TAG_CONSTRAINTS);

	return cib_root;
	}

	static unsigned int dtd_throttle = 0;

	enum cib_errors
	cib_perform_op(const char op, int call_options, cib_op_t fn, gboolean is_query,
	const char section, xmlNode req, xmlNode *input,
	gboolean manage_counters, gboolean *config_changed,
	xmlNode current_cib, xmlNode result_cib, xmlNode diff, xmlNode *output)
	{

	int rc = cib_ok;
	gboolean check_dtd = TRUE;
	xmlNode *scratch = NULL;
	xmlNode *local_diff = NULL;
	const char *current_dtd = "unknown";

	CRM_CHECK(output != NULL, return cib_output_data);
	CRM_CHECK(result_cib != NULL, return cib_output_data);
	CRM_CHECK(config_changed != NULL, return cib_output_data);

	*output = NULL;
	*result_cib = NULL;
	*config_changed = FALSE;

	if(fn == NULL) {
	return cib_operation;
	}

	if(is_query) {
	rc = (*fn)(op, call_options, section, req, input, current_cib, result_cib, output);
	return rc;
	}

	scratch = copy_xml(current_cib);
	rc = (*fn)(op, call_options, section, req, input, current_cib, &scratch, output);

	CRM_CHECK(current_cib != scratch, return cib_unknown);

	if(rc == cib_ok && scratch == NULL) {
	rc = cib_unknown;
	}

	if(rc == cib_ok && current_cib) {
	int old = 0;
	int new = 0;
	crm_element_value_int(scratch, XML_ATTR_GENERATION_ADMIN, &new);
	crm_element_value_int(current_cib, XML_ATTR_GENERATION_ADMIN, &old);

	if(old > new) {
	crm_err("%s went backwards: %d -> %d (Opts: 0x%x)",
	XML_ATTR_GENERATION_ADMIN, old, new, call_options);
	crm_log_xml_warn(req, "Bad Op");
	crm_log_xml_warn(input, "Bad Data");
	rc = cib_old_data;

	} else if(old == new) {
	crm_element_value_int(scratch, XML_ATTR_GENERATION, &new);
	crm_element_value_int(current_cib, XML_ATTR_GENERATION, &old);
	if(old > new) {
	crm_err("%s went backwards: %d -> %d (Opts: 0x%x)",
	XML_ATTR_GENERATION, old, new, call_options);
	crm_log_xml_warn(req, "Bad Op");
	crm_log_xml_warn(input, "Bad Data");
	rc = cib_old_data;
	}
	}
	}

	if(rc == cib_ok) {
	fix_plus_plus_recursive(scratch);
	current_dtd = crm_element_value(scratch, XML_ATTR_VALIDATION);

	if(manage_counters) {
	local_diff = diff_xml_object(current_cib, scratch, FALSE);
	*config_changed = cib_config_changed(local_diff);

	if(*config_changed) {
	cib_update_counter(scratch, XML_ATTR_NUMUPDATES, TRUE);
	cib_update_counter(scratch, XML_ATTR_GENERATION, FALSE);

	} else if(local_diff != NULL){
	cib_update_counter(scratch, XML_ATTR_NUMUPDATES, FALSE);
	if(dtd_throttle++ % 20) {
	check_dtd = FALSE; /* Throttle the amount of costly validation we perform due to status updates
	* a) we don't really care whats in the status section
	* b) we don't validate any of it's contents at the moment anyway
	*/
	}
	}
	}
	}

	if(diff != NULL && local_diff != NULL) {
	/* Only fix the diff if we'll return it... */

	xmlNode *cib = NULL;
	xmlNode *diff_child = NULL;
	const char *tag = NULL;
	const char *value = NULL;

	tag = "diff-removed";
	diff_child = find_xml_node(local_diff, tag, FALSE);
	if(diff_child == NULL) {
	diff_child = create_xml_node(local_diff, tag);
	}

	tag = XML_TAG_CIB;
	cib = find_xml_node(diff_child, tag, FALSE);
	if(cib == NULL) {
	cib = create_xml_node(diff_child, tag);
	}

	tag = XML_ATTR_GENERATION_ADMIN;
	value = crm_element_value(current_cib, tag);
	crm_xml_add(diff_child, tag, value);
	if(*config_changed) {
	crm_xml_add(cib, tag, value);
	}

	tag = XML_ATTR_GENERATION;
	value = crm_element_value(current_cib, tag);
	crm_xml_add(diff_child, tag, value);
	if(*config_changed) {
	crm_xml_add(cib, tag, value);
	}

	tag = XML_ATTR_NUMUPDATES;
	value = crm_element_value(current_cib, tag);
	crm_xml_add(cib, tag, value);
	crm_xml_add(diff_child, tag, value);

	tag = "diff-added";
	diff_child = find_xml_node(local_diff, tag, FALSE);
	if(diff_child == NULL) {
	diff_child = create_xml_node(local_diff, tag);
	}

	tag = XML_TAG_CIB;
	cib = find_xml_node(diff_child, tag, FALSE);
	if(cib == NULL) {
	cib = create_xml_node(diff_child, tag);
	}

	tag = XML_ATTR_GENERATION_ADMIN;
	value = crm_element_value(scratch, tag);
	crm_xml_add(diff_child, tag, value);
	if(*config_changed) {
	crm_xml_add(cib, tag, value);
	}

	tag = XML_ATTR_GENERATION;
	value = crm_element_value(scratch, tag);
	crm_xml_add(diff_child, tag, value);
	if(*config_changed) {
	crm_xml_add(cib, tag, value);
	}

	tag = XML_ATTR_NUMUPDATES;
	value = crm_element_value(scratch, tag);
	crm_xml_add(cib, tag, value);
	crm_xml_add(diff_child, tag, value);

	*diff = local_diff;
	local_diff = NULL;
	}

	if(rc == cib_ok
	&& check_dtd
	&& validate_xml(scratch, NULL, TRUE) == FALSE) {
	crm_warn("Updated CIB does not validate against %s schema/dtd", crm_str(current_dtd));
	rc = cib_dtd_validation;
	}

	*result_cib = scratch;
	free_xml(local_diff);
	return rc;
	}

	int get_channel_token(IPC_Channel ch, char *token)
	{
	int rc = cib_ok;
	xmlNode *reg_msg = NULL;
	const char *msg_type = NULL;
	const char *tmp_ticket = NULL;

	CRM_CHECK(ch != NULL, return cib_missing);
	CRM_CHECK(token != NULL, return cib_output_ptr);

	crm_debug_4("Waiting for msg on command channel");

	reg_msg = xmlfromIPC(ch, MAX_IPC_DELAY);

	if(ch->ops->get_chan_status(ch) != IPC_CONNECT) {
	crm_err("No reply message - disconnected");
	free_xml(reg_msg);
	return cib_not_connected;

	} else if(reg_msg == NULL) {
	crm_err("No reply message - empty");
	return cib_reply_failed;
	}

	msg_type = crm_element_value(reg_msg, F_CIB_OPERATION);
	tmp_ticket = crm_element_value(reg_msg, F_CIB_CLIENTID);

	if(safe_str_neq(msg_type, CRM_OP_REGISTER) ) {
	crm_err("Invalid registration message: %s", msg_type);
	rc = cib_registration_msg;

	} else if(tmp_ticket == NULL) {
	rc = cib_callback_token;

	} else {
	*token = crm_strdup(tmp_ticket);
	}

	free_xml(reg_msg);
	return rc;
	}


	xmlNode *
	cib_create_op(
	int call_id, const char token, const char op, const char host, const char section,
	xmlNode *data, int call_options)
	{
	int rc = HA_OK;
	xmlNode *op_msg = create_xml_node(NULL, "cib_command");
	CRM_CHECK(op_msg != NULL, return NULL);
	CRM_CHECK(token != NULL, return NULL);

	crm_xml_add(op_msg, F_XML_TAGNAME, "cib_command");

	crm_xml_add(op_msg, F_TYPE, T_CIB);
	crm_xml_add(op_msg, F_CIB_CALLBACK_TOKEN, token);
	crm_xml_add(op_msg, F_CIB_OPERATION, op);
	crm_xml_add(op_msg, F_CIB_HOST, host);
	crm_xml_add(op_msg, F_CIB_SECTION, section);
	crm_xml_add_int(op_msg, F_CIB_CALLID, call_id);
	crm_debug_4("Sending call options: %.8lx, %d",
	(long)call_options, call_options);
	crm_xml_add_int(op_msg, F_CIB_CALLOPTS, call_options);

	if(data != NULL) {
	add_message_xml(op_msg, F_CIB_CALLDATA, data);
	}

	if (rc != HA_OK) {
	crm_err("Failed to create CIB operation message");
	crm_log_xml(LOG_ERR, "op", op_msg);
	free_xml(op_msg);
	return NULL;
	}

	if(call_options & cib_inhibit_bcast) {
	CRM_CHECK((call_options & cib_scope_local), return NULL);
	}
	return op_msg;
	}

	void
	cib_native_callback(cib_t cib, xmlNode msg, int call_id, int rc)
	{
	xmlNode *output = NULL;
	cib_callback_client_t *blob = NULL;
	cib_callback_client_t local_blob;

	local_blob.id = NULL;
	local_blob.callback = NULL;
	local_blob.user_data = NULL;
	local_blob.only_success = FALSE;

	if(msg != NULL) {
	crm_element_value_int(msg, F_CIB_RC, &rc);
	crm_element_value_int(msg, F_CIB_CALLID, &call_id);
	output = get_message_xml(msg, F_CIB_CALLDATA);
	}

	blob = g_hash_table_lookup(
	cib_op_callback_table, GINT_TO_POINTER(call_id));

	if(blob != NULL) {
	local_blob = *blob;
	blob = NULL;

	remove_cib_op_callback(call_id, FALSE);

	} else {
	crm_debug_2("No callback found for call %d", call_id);
	local_blob.callback = NULL;
	}

	if(cib == NULL) {
	crm_debug("No cib object supplied");
	}

	if(rc == cib_diff_resync) {
	/* This is an internal value that clients do not and should not care about */
	rc = cib_ok;
	}

	if(local_blob.callback != NULL
	&& (rc == cib_ok \|\| local_blob.only_success == FALSE)) {
	crm_debug_2("Invoking callback %s for call %d", crm_str(local_blob.id), call_id);
	local_blob.callback(msg, call_id, rc, output, local_blob.user_data);

	} else if(cib && cib->op_callback == NULL && rc != cib_ok) {
	crm_warn("CIB command failed: %s", cib_error2string(rc));
	crm_log_xml(LOG_DEBUG, "Failed CIB Update", msg);
	}

	if(cib && cib->op_callback != NULL) {
	crm_debug_2("Invoking global callback for call %d", call_id);
	cib->op_callback(msg, call_id, rc, output);
	}
	crm_debug_4("OP callback activated.");
	}


	void
	cib_native_notify(gpointer data, gpointer user_data)
	{
	xmlNode *msg = user_data;
	cib_notify_client_t *entry = data;
	const char *event = NULL;

	if(msg == NULL) {
	crm_warn("Skipping callback - NULL message");
	return;
	}

	event = crm_element_value(msg, F_SUBTYPE);

	if(entry == NULL) {
	crm_warn("Skipping callback - NULL callback client");
	return;

	} else if(entry->callback == NULL) {
	crm_warn("Skipping callback - NULL callback");
	return;

	} else if(safe_str_neq(entry->event, event)) {
	crm_debug_4("Skipping callback - event mismatch %p/%s vs. %s",
	entry, entry->event, event);
	return;
	}

	crm_debug_4("Invoking callback for %p/%s event...", entry, event);
	entry->callback(event, msg);
	crm_debug_4("Callback invoked...");
	}

	gboolean determine_host(cib_t cib_conn, char node_uname, char *node_uuid)
	{
	CRM_CHECK(node_uname != NULL, return FALSE);

	if(*node_uname == NULL) {
	struct utsname name;
	if(uname(&name) < 0) {
	crm_perror(LOG_ERR,"uname(2) call failed");
	return FALSE;
	}
	*node_uname = crm_strdup(name.nodename);
	crm_info("Detected uname: %s", *node_uname);
	}

	if(cib_conn && *node_uname != NULL
	&& node_uuid != NULL && *node_uuid == NULL) {
	int rc = query_node_uuid(cib_conn, *node_uname, node_uuid);
	if(rc != cib_ok) {
	fprintf(stderr,"Could not map uname=%s to a UUID: %s\n",
	*node_uname, cib_error2string(rc));
	return FALSE;
	}
	crm_info("Mapped %s to %s", node_uname, crm_str(node_uuid));
	}
	return TRUE;
	}

	diff --git a/lib/pengine/status.c b/lib/pengine/status.c
	index e3c152bbd5..75f073f5a7 100644
	--- a/lib/pengine/status.c
	+++ b/lib/pengine/status.c
	@@ -1,291 +1,296 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/msg.h>


	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <utils.h>
	#include <unpack.h>

	xmlNode * do_calculations(
	pe_working_set_t data_set, xmlNode xml_input, ha_time_t *now);

	extern xmlNode*get_object_root(
	const char object_type, xmlNode the_root);

	#define MEMCHECK_STAGE_0 0

	#define check_and_exit(stage) cleanup_calculations(data_set); \
	crm_mem_stats(NULL); \
	crm_err("Exiting: stage %d", stage); \
	exit(1);


	/*
	* Unpack everything
	* At the end you'll have:
	* - A list of nodes
	* - A list of resources (each with any dependencies on other resources)
	* - A list of constraints between resources and nodes
	* - A list of constraints between start/stop actions
	* - A list of nodes that need to be stonith'd
	* - A list of nodes that need to be shutdown
	* - A list of the possible stop/start actions (without dependencies)
	*/
	gboolean
	cluster_status(pe_working_set_t *data_set)
	{
	xmlNode * config = get_object_root(
	XML_CIB_TAG_CRMCONFIG, data_set->input);
	xmlNode * cib_nodes = get_object_root(
	XML_CIB_TAG_NODES, data_set->input);
	xmlNode * cib_resources = get_object_root(
	XML_CIB_TAG_RESOURCES, data_set->input);
	xmlNode * cib_status = get_object_root(
	XML_CIB_TAG_STATUS, data_set->input);
	+ xmlNode * cib_domains = get_object_root(
	+ XML_CIB_TAG_DOMAINS, data_set->input);
	const char *value = crm_element_value(
	data_set->input, XML_ATTR_HAVE_QUORUM);

	crm_debug_3("Beginning unpack");

	/* reset remaining global variables */

	if(data_set->input == NULL) {
	return FALSE;
	}

	if(data_set->now == NULL) {
	data_set->now = new_ha_date(TRUE);
	}

	if(data_set->input != NULL
	&& crm_element_value(data_set->input, XML_ATTR_DC_UUID) != NULL) {
	/* this should always be present */
	data_set->dc_uuid = crm_element_value_copy(
	data_set->input, XML_ATTR_DC_UUID);
	}

	clear_bit_inplace(data_set->flags, pe_flag_have_quorum);
	if(crm_is_true(value)) {
	set_bit_inplace(data_set->flags, pe_flag_have_quorum);
	}

	data_set->op_defaults = get_object_root(XML_CIB_TAG_OPCONFIG, data_set->input);
	data_set->rsc_defaults = get_object_root(XML_CIB_TAG_RSCCONFIG, data_set->input);

	unpack_config(config, data_set);

	if(is_set(data_set->flags, pe_flag_have_quorum) == FALSE
	&& data_set->no_quorum_policy != no_quorum_ignore) {
	crm_warn("We do not have quorum"
	" - fencing and resource management disabled");
	}

	unpack_nodes(cib_nodes, data_set);
	+ unpack_domains(cib_domains, data_set);
	unpack_resources(cib_resources, data_set);
	unpack_status(cib_status, data_set);

	return TRUE;
	}

	static void
	pe_free_resources(GListPtr resources)
	{
	resource_t *rsc = NULL;
	GListPtr iterator = resources;
	while(iterator != NULL) {
	iterator = iterator;
	rsc = (resource_t *)iterator->data;
	iterator = iterator->next;
	rsc->fns->free(rsc);
	}
	if(resources != NULL) {
	g_list_free(resources);
	}
	}

	static void
	pe_free_actions(GListPtr actions)
	{
	GListPtr iterator = actions;
	while(iterator != NULL) {
	pe_free_action(iterator->data);
	iterator = iterator->next;
	}
	if(actions != NULL) {
	g_list_free(actions);
	}
	}

	static void
	pe_free_nodes(GListPtr nodes)
	{
	GListPtr iterator = nodes;
	while(iterator != NULL) {
	node_t node = (node_t)iterator->data;
	struct node_shared_s *details = node->details;
	iterator = iterator->next;

	crm_debug_5("deleting node");
	crm_debug_5("%s is being deleted", details->uname);
	print_node("delete", node, FALSE);

	if(details != NULL) {
	if(details->attrs != NULL) {
	g_hash_table_destroy(details->attrs);
	}
	if(details->utilization != NULL) {
	g_hash_table_destroy(details->utilization);
	}
	pe_free_shallow_adv(details->running_rsc, FALSE);
	pe_free_shallow_adv(details->allocated_rsc, FALSE);
	crm_free(details);
	}
	crm_free(node);
	}
	if(nodes != NULL) {
	g_list_free(nodes);
	}
	}

	void
	cleanup_calculations(pe_working_set_t *data_set)
	{
	if(data_set == NULL) {
	return;
	}

	if(data_set->config_hash != NULL) {
	g_hash_table_destroy(data_set->config_hash);
	}

	crm_free(data_set->dc_uuid);

	crm_debug_3("deleting resources");
	pe_free_resources(data_set->resources);

	crm_debug_3("deleting actions");
	pe_free_actions(data_set->actions);

	+ g_hash_table_destroy(data_set->domains);
	+ data_set->domains = NULL;
	+
	crm_debug_3("deleting nodes");
	pe_free_nodes(data_set->nodes);

	free_xml(data_set->graph);
	free_ha_date(data_set->now);
	free_xml(data_set->input);
	free_xml(data_set->failed);
	data_set->stonith_action = NULL;

	CRM_CHECK(data_set->ordering_constraints == NULL, ;);
	CRM_CHECK(data_set->placement_constraints == NULL, ;);
	xmlCleanupParser();
	}

	-
	void
	set_working_set_defaults(pe_working_set_t *data_set)
	{
	data_set->failed = create_xml_node(NULL, "failed-ops");

	data_set->now = NULL;
	data_set->input = NULL;
	data_set->graph = NULL;
	data_set->dc_uuid = NULL;
	data_set->dc_node = NULL;

	data_set->nodes = NULL;
	data_set->actions = NULL;
	data_set->resources = NULL;
	data_set->config_hash = NULL;
	data_set->stonith_action = NULL;
	data_set->ordering_constraints = NULL;
	data_set->placement_constraints = NULL;
	data_set->colocation_constraints = NULL;

	data_set->order_id = 1;
	data_set->action_id = 1;
	data_set->num_synapse = 0;
	data_set->max_valid_nodes = 0;
	data_set->no_quorum_policy = no_quorum_freeze;

	data_set->default_resource_stickiness = 0;

	data_set->flags = 0x0ULL;
	set_bit_inplace(data_set->flags, pe_flag_stop_rsc_orphans);
	set_bit_inplace(data_set->flags, pe_flag_symmetric_cluster);
	set_bit_inplace(data_set->flags, pe_flag_is_managed_default);
	set_bit_inplace(data_set->flags, pe_flag_stop_action_orphans);
	}


	resource_t *
	pe_find_resource(GListPtr rsc_list, const char *id)
	{
	unsigned lpc = 0;
	resource_t *rsc = NULL;
	resource_t *match = NULL;

	if(id == NULL) {
	return NULL;
	}

	for(lpc = 0; lpc < g_list_length(rsc_list); lpc++) {
	rsc = g_list_nth_data(rsc_list, lpc);

	match = rsc->fns->find_rsc(rsc, id, TRUE, FALSE, NULL, TRUE);
	if(match != NULL) {
	return match;
	}
	}
	crm_debug_2("No match for %s", id);
	return NULL;
	}

	node_t *
	pe_find_node_id(GListPtr nodes, const char *id)
	{
	slist_iter(node, node_t, nodes, lpc,
	if(node && safe_str_eq(node->details->id, id)) {
	return node;
	}
	);
	/* error */
	return NULL;
	}

	node_t *
	pe_find_node(GListPtr nodes, const char *uname)
	{
	slist_iter(node, node_t, nodes, lpc,
	if(node && safe_str_eq(node->details->uname, uname)) {
	return node;
	}
	);
	/* error */
	return NULL;
	}
	diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
	index d4751352a2..fefbbf9f9b 100644
	--- a/lib/pengine/unpack.c
	+++ b/lib/pengine/unpack.c
	@@ -1,1754 +1,1815 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <crm_internal.h>

	#include <lrm/lrm_api.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/msg.h>
	#include <crm/common/util.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/rules.h>
	#include <utils.h>
	#include <unpack.h>

	#define set_config_flag(data_set, option, flag) do { \
	const char *tmp = pe_pref(data_set->config_hash, option); \
	if(tmp) { \
	if(crm_is_true(tmp)) { \
	set_bit_inplace(data_set->flags, flag); \
	} else { \
	clear_bit_inplace(data_set->flags, flag); \
	} \
	} \
	} while(0)

	gboolean unpack_rsc_op(
	resource_t rsc, node_t node, xmlNode *xml_op,
	enum action_fail_response failed, pe_working_set_t data_set);

	static void pe_fence_node(pe_working_set_t data_set, node_t node, const char *reason)
	{
	CRM_CHECK(node, return);
	if(node->details->unclean == FALSE) {
	if(is_set(data_set->flags, pe_flag_stonith_enabled)) {
	crm_warn("Node %s will be fenced %s", node->details->uname, reason);
	} else {
	crm_warn("Node %s is unclean %s", node->details->uname, reason);
	}
	}
	node->details->unclean = TRUE;
	}


	gboolean
	unpack_config(xmlNode config, pe_working_set_t data_set)
	{
	const char *value = NULL;
	GHashTable *config_hash = g_hash_table_new_full(
	g_str_hash,g_str_equal, g_hash_destroy_str,g_hash_destroy_str);

	data_set->config_hash = config_hash;

	unpack_instance_attributes(
	data_set->input, config, XML_CIB_TAG_PROPSET, NULL, config_hash,
	CIB_OPTIONS_FIRST, FALSE, data_set->now);

	verify_pe_options(data_set->config_hash);

	set_config_flag(data_set, "enable-startup-probes", pe_flag_startup_probes);
	crm_info("Startup probes: %s",
	is_set(data_set->flags, pe_flag_startup_probes)?"enabled":"disabled (dangerous)");

	value = pe_pref(data_set->config_hash, "stonith-timeout");
	data_set->stonith_timeout = crm_get_msec(value);
	crm_debug("STONITH timeout: %d", data_set->stonith_timeout);

	set_config_flag(data_set, "stonith-enabled", pe_flag_stonith_enabled);
	crm_debug("STONITH of failed nodes is %s",
	is_set(data_set->flags, pe_flag_stonith_enabled)?"enabled":"disabled");

	data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action");
	crm_debug_2("STONITH will %s nodes", data_set->stonith_action);

	set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything);
	crm_debug("Stop all active resources: %s",
	is_set(data_set->flags, pe_flag_stop_everything)?"true":"false");

	set_config_flag(data_set, "symmetric-cluster", pe_flag_symmetric_cluster);
	if(is_set(data_set->flags, pe_flag_symmetric_cluster)) {
	crm_debug("Cluster is symmetric"
	" - resources can run anywhere by default");
	}

	value = pe_pref(data_set->config_hash, "default-resource-stickiness");
	data_set->default_resource_stickiness = char2score(value);
	crm_debug("Default stickiness: %d",
	data_set->default_resource_stickiness);

	value = pe_pref(data_set->config_hash, "no-quorum-policy");

	if(safe_str_eq(value, "ignore")) {
	data_set->no_quorum_policy = no_quorum_ignore;

	} else if(safe_str_eq(value, "freeze")) {
	data_set->no_quorum_policy = no_quorum_freeze;

	} else if(safe_str_eq(value, "suicide")) {
	gboolean do_panic = FALSE;
	crm_element_value_int(data_set->input, XML_ATTR_QUORUM_PANIC, &do_panic);

	if(is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE){
	crm_config_err("Setting no-quorum-policy=suicide makes no sense if stonith-enabled=false");
	}

	if(do_panic && is_set(data_set->flags, pe_flag_stonith_enabled)) {
	data_set->no_quorum_policy = no_quorum_suicide;

	} else if(is_set(data_set->flags, pe_flag_have_quorum) == FALSE && do_panic == FALSE) {
	crm_notice("Resetting no-quorum-policy to 'stop': The cluster has never had quorum");
	data_set->no_quorum_policy = no_quorum_stop;
	}

	} else {
	data_set->no_quorum_policy = no_quorum_stop;
	}

	switch (data_set->no_quorum_policy) {
	case no_quorum_freeze:
	crm_debug("On loss of CCM Quorum: Freeze resources");
	break;
	case no_quorum_stop:
	crm_debug("On loss of CCM Quorum: Stop ALL resources");
	break;
	case no_quorum_suicide:
	crm_notice("On loss of CCM Quorum: Fence all remaining nodes");
	break;
	case no_quorum_ignore:
	crm_notice("On loss of CCM Quorum: Ignore");
	break;
	}

	set_config_flag(data_set, "stop-orphan-resources", pe_flag_stop_rsc_orphans);
	crm_debug_2("Orphan resources are %s",
	is_set(data_set->flags, pe_flag_stop_rsc_orphans)?"stopped":"ignored");

	set_config_flag(data_set, "stop-orphan-actions", pe_flag_stop_action_orphans);
	crm_debug_2("Orphan resource actions are %s",
	is_set(data_set->flags, pe_flag_stop_action_orphans)?"stopped":"ignored");

	set_config_flag(data_set, "remove-after-stop", pe_flag_remove_after_stop);
	crm_debug_2("Stopped resources are removed from the status section: %s",
	is_set(data_set->flags, pe_flag_remove_after_stop)?"true":"false");

	set_config_flag(data_set, "maintenance-mode", pe_flag_maintenance_mode);
	crm_debug_2("Maintenance mode: %s",
	is_set(data_set->flags, pe_flag_maintenance_mode)?"true":"false");

	if(is_set(data_set->flags, pe_flag_maintenance_mode)) {
	clear_bit(data_set->flags, pe_flag_is_managed_default);
	} else {
	set_config_flag(data_set, "is-managed-default", pe_flag_is_managed_default);
	}
	crm_debug_2("By default resources are %smanaged",
	is_set(data_set->flags, pe_flag_is_managed_default)?"":"not ");

	set_config_flag(data_set, "start-failure-is-fatal", pe_flag_start_failure_fatal);
	crm_debug_2("Start failures are %s",
	is_set(data_set->flags, pe_flag_start_failure_fatal)?"always fatal":"handled by failcount");

	node_score_red = char2score(pe_pref(data_set->config_hash, "node-health-red"));
	node_score_green = char2score(pe_pref(data_set->config_hash, "node-health-green"));
	node_score_yellow = char2score(pe_pref(data_set->config_hash, "node-health-yellow"));

	crm_info("Node scores: 'red' = %s, 'yellow' = %s, 'green' = %s",
	pe_pref(data_set->config_hash, "node-health-red"),
	pe_pref(data_set->config_hash, "node-health-yellow"),
	pe_pref(data_set->config_hash, "node-health-green"));

	data_set->placement_strategy = pe_pref(data_set->config_hash, "placement-strategy");
	crm_debug_2("Placement strategy: %s", data_set->placement_strategy);

	return TRUE;
	}

	gboolean
	unpack_nodes(xmlNode * xml_nodes, pe_working_set_t *data_set)
	{
	node_t *new_node = NULL;
	const char *id = NULL;
	const char *uname = NULL;
	const char *type = NULL;
	gboolean unseen_are_unclean = TRUE;
	const char *blind_faith = pe_pref(
	data_set->config_hash, "startup-fencing");

	if(crm_is_true(blind_faith) == FALSE) {
	unseen_are_unclean = FALSE;
	crm_warn("Blind faith: not fencing unseen nodes");
	}

	xml_child_iter_filter(
	xml_nodes, xml_obj, XML_CIB_TAG_NODE,

	new_node = NULL;

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
	type = crm_element_value(xml_obj, XML_ATTR_TYPE);
	crm_debug_3("Processing node %s/%s", uname, id);

	if(id == NULL) {
	crm_config_err("Must specify id tag in <node>");
	continue;
	}
	if(type == NULL) {
	crm_config_err("Must specify type tag in <node>");
	continue;
	}
	if(pe_find_node(data_set->nodes, uname) != NULL) {
	crm_config_warn("Detected multiple node entries with uname=%s"
	" - this is rarely intended", uname);
	}

	crm_malloc0(new_node, sizeof(node_t));
	if(new_node == NULL) {
	return FALSE;
	}

	new_node->weight = 0;
	new_node->fixed = FALSE;
	crm_malloc0(new_node->details,
	sizeof(struct node_shared_s));

	if(new_node->details == NULL) {
	crm_free(new_node);
	return FALSE;
	}

	crm_debug_3("Creaing node for entry %s/%s", uname, id);
	new_node->details->id = id;
	new_node->details->uname = uname;
	new_node->details->type = node_ping;
	new_node->details->online = FALSE;
	new_node->details->shutdown = FALSE;
	new_node->details->running_rsc = NULL;
	new_node->details->attrs = g_hash_table_new_full(
	g_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);
	new_node->details->utilization = g_hash_table_new_full(
	g_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	/* if(data_set->have_quorum == FALSE */
	/* && data_set->no_quorum_policy == no_quorum_stop) { */
	/* /\* start shutting resources down \/ /
	/* new_node->weight = -INFINITY; */
	/* } */

	if(is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE \|\| unseen_are_unclean == FALSE) {
	/* blind faith... */
	new_node->details->unclean = FALSE;

	} else {
	/* all nodes are unclean until we've seen their
	* status entry
	*/
	new_node->details->unclean = TRUE;
	}

	if(type == NULL
	\|\| safe_str_eq(type, "member")
	\|\| safe_str_eq(type, NORMALNODE)) {
	new_node->details->type = node_member;
	}

	add_node_attrs(xml_obj, new_node, FALSE, data_set);
	unpack_instance_attributes(
	data_set->input, xml_obj, XML_TAG_UTILIZATION, NULL,
	new_node->details->utilization, NULL, FALSE, data_set->now);

	data_set->nodes = g_list_append(data_set->nodes, new_node);
	crm_debug_3("Done with node %s",
	crm_element_value(xml_obj, XML_ATTR_UNAME));
	);

	return TRUE;
	}

	+static void g_hash_destroy_node_list(gpointer data)
	+{
	+ GListPtr domain = data;
	+ slist_destroy(node_t, node, domain, crm_free(node));
	+}
	+
	+gboolean
	+unpack_domains(xmlNode xml_domains, pe_working_set_t data_set)
	+{
	+ GListPtr domain = NULL;
	+ const char *id = NULL;
	+
	+ crm_info("Unpacking domains");
	+ data_set->domains = g_hash_table_new_full(
	+ g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_node_list);
	+
	+ xml_child_iter_filter(
	+ xml_domains, xml_domain, XML_CIB_TAG_DOMAIN,
	+
	+ domain = NULL;
	+ id = crm_element_value(xml_domain, XML_ATTR_ID);
	+
	+ xml_child_iter_filter(
	+ xml_domain, xml_node, XML_CIB_TAG_NODE,
	+
	+ node_t *copy = NULL;
	+ node_t *node = NULL;
	+ const char *uname = crm_element_value(xml_node, "name");
	+ const char *score = crm_element_value(xml_node, XML_RULE_ATTR_SCORE);
	+
	+ if(uname == NULL) {
	+ crm_config_err("Invalid domain %s: Must specify id tag in <node>", id);
	+ continue;
	+ }
	+
	+ node = pe_find_node(data_set->nodes, uname);
	+ if(node == NULL) {
	+ node = pe_find_node_id(data_set->nodes, uname);
	+ continue;
	+ }
	+ if(node == NULL) {
	+ crm_config_warn("Invalid domain %s: Node %s does not exist", id, uname);
	+ continue;
	+ }
	+
	+ copy = node_copy(node);
	+ copy->weight = char2score(score);
	+ crm_debug("Adding %s to domain %s with score %s", node->details->uname, id, score);
	+
	+ domain = g_list_append(domain, copy);
	+ );
	+
	+ if(domain) {
	+ crm_debug("Created domain %s with %d members", id, g_list_length(domain));
	+ g_hash_table_replace(data_set->domains, crm_strdup(id), domain);
	+ }
	+ );
	+
	+ return TRUE;
	+}
	+
	gboolean
	unpack_resources(xmlNode * xml_resources, pe_working_set_t *data_set)
	{
	xml_child_iter(
	xml_resources, xml_obj,

	resource_t *new_rsc = NULL;
	crm_debug_3("Beginning unpack... <%s id=%s... >",
	crm_element_name(xml_obj), ID(xml_obj));
	if(common_unpack(xml_obj, &new_rsc, NULL, data_set)) {
	data_set->resources = g_list_append(
	data_set->resources, new_rsc);

	print_resource(LOG_DEBUG_3, "Added", new_rsc, FALSE);

	} else {
	crm_config_err("Failed unpacking %s %s",
	crm_element_name(xml_obj),
	crm_element_value(xml_obj, XML_ATTR_ID));
	if(new_rsc != NULL && new_rsc->fns != NULL) {
	new_rsc->fns->free(new_rsc);
	}
	}
	);

	data_set->resources = g_list_sort(
	data_set->resources, sort_rsc_priority);

	if(is_set(data_set->flags, pe_flag_stonith_enabled) && is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
	crm_config_err("Resource start-up disabled since no STONITH resources have been defined");
	crm_config_err("Either configure some or disable STONITH with the stonith-enabled option");
	crm_config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
	}

	return TRUE;
	}


	/* remove nodes that are down, stopping */
	/* create +ve rsc_to_node constraints between resources and the nodes they are running on */
	/* anything else? */
	gboolean
	unpack_status(xmlNode * status, pe_working_set_t *data_set)
	{
	const char *id = NULL;
	const char *uname = NULL;

	xmlNode * lrm_rsc = NULL;
	xmlNode * attrs = NULL;
	node_t *this_node = NULL;

	crm_debug_3("Beginning unpack");
	xml_child_iter_filter(
	status, node_state, XML_CIB_TAG_STATE,

	id = crm_element_value(node_state, XML_ATTR_ID);
	uname = crm_element_value(node_state, XML_ATTR_UNAME);
	attrs = find_xml_node(
	node_state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);

	lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);

	crm_debug_3("Processing node %s", uname);
	this_node = pe_find_node_id(data_set->nodes, id);

	if(uname == NULL) {
	/* error */
	continue;

	} else if(this_node == NULL) {
	crm_config_warn("Node %s in status section no longer exists",
	uname);
	continue;
	}

	/* Mark the node as provisionally clean
	* - at least we have seen it in the current cluster's lifetime
	*/
	this_node->details->unclean = FALSE;
	add_node_attrs(attrs, this_node, TRUE, data_set);

	if(crm_is_true(g_hash_table_lookup(this_node->details->attrs, "standby"))) {
	crm_info("Node %s is in standby-mode",
	this_node->details->uname);
	this_node->details->standby = TRUE;
	}

	crm_debug_3("determining node state");
	determine_online_status(node_state, this_node, data_set);

	if(this_node->details->online
	&& data_set->no_quorum_policy == no_quorum_suicide) {
	/* Everything else should flow from this automatically
	* At least until the PE becomes able to migrate off healthy resources
	*/
	pe_fence_node(data_set, this_node, "because the cluster does not have quorum");
	}

	if(this_node->details->online \|\| is_set(data_set->flags, pe_flag_stonith_enabled)) {
	/* offline nodes run no resources...
	* unless stonith is enabled in which case we need to
	* make sure rsc start events happen after the stonith
	*/
	crm_debug_3("Processing lrm resource entries");
	unpack_lrm_resources(this_node, lrm_rsc, data_set);
	}
	);

	return TRUE;

	}

	static gboolean
	determine_online_status_no_fencing(pe_working_set_t data_set, xmlNode node_state, node_t *this_node)
	{
	gboolean online = FALSE;
	const char *join_state = crm_element_value(node_state, XML_CIB_ATTR_JOINSTATE);
	const char *crm_state = crm_element_value(node_state, XML_CIB_ATTR_CRMDSTATE);
	const char *ccm_state = crm_element_value(node_state, XML_CIB_ATTR_INCCM);
	const char *ha_state = crm_element_value(node_state, XML_CIB_ATTR_HASTATE);
	const char *exp_state = crm_element_value(node_state, XML_CIB_ATTR_EXPSTATE);

	if(ha_state == NULL) {
	ha_state = DEADSTATUS;
	}

	if(!crm_is_true(ccm_state) \|\| safe_str_eq(ha_state, DEADSTATUS)){
	crm_debug_2("Node is down: ha_state=%s, ccm_state=%s",
	crm_str(ha_state), crm_str(ccm_state));

	} else if(!crm_is_true(ccm_state)
	\|\| safe_str_eq(ha_state, DEADSTATUS)) {

	} else if(safe_str_eq(crm_state, ONLINESTATUS)) {
	if(safe_str_eq(join_state, CRMD_JOINSTATE_MEMBER)) {
	online = TRUE;
	} else {
	crm_debug("Node is not ready to run resources: %s", join_state);
	}

	} else if(this_node->details->expected_up == FALSE) {
	crm_debug_2("CRMd is down: ha_state=%s, ccm_state=%s",
	crm_str(ha_state), crm_str(ccm_state));
	crm_debug_2("\tcrm_state=%s, join_state=%s, expected=%s",
	crm_str(crm_state), crm_str(join_state),
	crm_str(exp_state));

	} else {
	/* mark it unclean */
	pe_fence_node(data_set, this_node, "because it is partially and/or un-expectedly down");
	crm_info("\tha_state=%s, ccm_state=%s,"
	" crm_state=%s, join_state=%s, expected=%s",
	crm_str(ha_state), crm_str(ccm_state),
	crm_str(crm_state), crm_str(join_state),
	crm_str(exp_state));
	}
	return online;
	}

	static gboolean
	determine_online_status_fencing(pe_working_set_t data_set, xmlNode node_state, node_t *this_node)
	{
	gboolean online = FALSE;
	gboolean do_terminate = FALSE;
	const char *join_state = crm_element_value(node_state, XML_CIB_ATTR_JOINSTATE);
	const char *crm_state = crm_element_value(node_state, XML_CIB_ATTR_CRMDSTATE);
	const char *ccm_state = crm_element_value(node_state, XML_CIB_ATTR_INCCM);
	const char *ha_state = crm_element_value(node_state, XML_CIB_ATTR_HASTATE);
	const char *exp_state = crm_element_value(node_state, XML_CIB_ATTR_EXPSTATE);
	const char *terminate = g_hash_table_lookup(this_node->details->attrs, "terminate");

	if(ha_state == NULL) {
	ha_state = DEADSTATUS;
	}

	if(crm_is_true(terminate)) {
	do_terminate = TRUE;

	} else if(terminate != NULL && strlen(terminate) > 0) {
	/* could be a time() value */
	char t = terminate[0];
	if(t != '0' && isdigit(t)) {
	do_terminate = TRUE;
	}
	}

	if(crm_is_true(ccm_state)
	&& safe_str_eq(ha_state, ACTIVESTATUS)
	&& safe_str_eq(crm_state, ONLINESTATUS)) {

	if(safe_str_eq(join_state, CRMD_JOINSTATE_MEMBER)) {
	online = TRUE;
	if(do_terminate) {
	pe_fence_node(data_set, this_node, "because termination was requested");
	this_node->details->shutdown = TRUE;
	}

	} else if(join_state == exp_state /* == NULL */) {
	crm_info("Node %s is coming up", this_node->details->uname);
	crm_debug("\tha_state=%s, ccm_state=%s,"
	" crm_state=%s, join_state=%s, expected=%s",
	crm_str(ha_state), crm_str(ccm_state),
	crm_str(crm_state), crm_str(join_state),
	crm_str(exp_state));

	} else if(safe_str_eq(join_state, CRMD_JOINSTATE_PENDING)) {
	crm_info("Node %s is not ready to run resources",
	this_node->details->uname);
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;
	online = TRUE;

	} else if(safe_str_eq(join_state, CRMD_JOINSTATE_NACK)) {
	crm_warn("Node %s is not part of the cluster",
	this_node->details->uname);
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;
	online = TRUE;

	} else {
	crm_warn("Node %s (%s) is un-expectedly down",
	this_node->details->uname, this_node->details->id);
	crm_info("\tha_state=%s, ccm_state=%s,"
	" crm_state=%s, join_state=%s, expected=%s",
	crm_str(ha_state), crm_str(ccm_state),
	crm_str(crm_state), crm_str(join_state),
	crm_str(exp_state));
	pe_fence_node(data_set, this_node, "because it is un-expectedly down");
	}

	} else if(crm_is_true(ccm_state) == FALSE
	&& safe_str_eq(ha_state, DEADSTATUS)
	&& safe_str_eq(crm_state, OFFLINESTATUS)
	&& this_node->details->expected_up == FALSE) {
	crm_debug("Node %s is down: join_state=%s, expected=%s",
	this_node->details->uname,
	crm_str(join_state), crm_str(exp_state));

	#if 0
	/* While a nice optimization, it causes the cluster to block until the node
	* comes back online. Which is a serious problem if the cluster software
	* is not configured to start at boot or stonith is configured to merely
	* stop the node instead of restart it.
	* Easily triggered by setting terminate=true for the DC
	*/
	} else if(do_terminate) {
	crm_info("Node %s is %s after forced termination",
	this_node->details->uname, crm_is_true(ccm_state)?"coming up":"going down");
	crm_debug("\tha_state=%s, ccm_state=%s,"
	" crm_state=%s, join_state=%s, expected=%s",
	crm_str(ha_state), crm_str(ccm_state),
	crm_str(crm_state), crm_str(join_state),
	crm_str(exp_state));

	if(crm_is_true(ccm_state) == FALSE) {
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;
	online = TRUE;
	}
	#endif

	} else if(this_node->details->expected_up) {
	/* mark it unclean */
	pe_fence_node(data_set, this_node, "because it is un-expectedly down");
	crm_info("\tha_state=%s, ccm_state=%s,"
	" crm_state=%s, join_state=%s, expected=%s",
	crm_str(ha_state), crm_str(ccm_state),
	crm_str(crm_state), crm_str(join_state),
	crm_str(exp_state));

	} else {
	crm_info("Node %s is down", this_node->details->uname);
	crm_debug("\tha_state=%s, ccm_state=%s,"
	" crm_state=%s, join_state=%s, expected=%s",
	crm_str(ha_state), crm_str(ccm_state),
	crm_str(crm_state), crm_str(join_state),
	crm_str(exp_state));
	}
	return online;
	}

	gboolean
	determine_online_status(
	xmlNode * node_state, node_t this_node, pe_working_set_t data_set)
	{
	gboolean online = FALSE;
	const char *shutdown = NULL;
	const char *exp_state = crm_element_value(node_state, XML_CIB_ATTR_EXPSTATE);

	if(this_node == NULL) {
	crm_config_err("No node to check");
	return online;
	}

	this_node->details->shutdown = FALSE;
	this_node->details->expected_up = FALSE;
	shutdown = g_hash_table_lookup(this_node->details->attrs, XML_CIB_ATTR_SHUTDOWN);

	if(shutdown != NULL && safe_str_neq("0", shutdown)) {
	this_node->details->shutdown = TRUE;

	} else if(safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) {
	this_node->details->expected_up = TRUE;
	}

	if(is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
	online = determine_online_status_no_fencing(
	data_set, node_state, this_node);

	} else {
	online = determine_online_status_fencing(
	data_set, node_state, this_node);
	}

	if(online) {
	this_node->details->online = TRUE;

	} else {
	/* remove node from contention */
	this_node->fixed = TRUE;
	this_node->weight = -INFINITY;
	}

	if(online && this_node->details->shutdown) {
	/* dont run resources here */
	this_node->fixed = TRUE;
	this_node->weight = -INFINITY;
	}

	if(this_node->details->unclean) {
	pe_proc_warn("Node %s is unclean", this_node->details->uname);

	} else if(this_node->details->online) {
	crm_info("Node %s is %s", this_node->details->uname,
	this_node->details->shutdown?"shutting down":
	this_node->details->pending?"pending":
	this_node->details->standby?"standby":"online");

	} else {
	crm_debug_2("Node %s is offline", this_node->details->uname);
	}

	return online;
	}

	#define set_char(x) last_rsc_id[lpc] = x; complete = TRUE;

	char *
	clone_zero(const char *last_rsc_id)
	{
	int lpc = 0;
	char *zero = NULL;

	CRM_CHECK(last_rsc_id != NULL, return NULL);
	if(last_rsc_id != NULL) {
	lpc = strlen(last_rsc_id);
	}

	while(--lpc > 0) {
	switch(last_rsc_id[lpc]) {
	case 0:
	return NULL;
	break;
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;
	case ':':
	crm_malloc0(zero, lpc + 3);
	memcpy(zero, last_rsc_id, lpc);
	zero[lpc] = ':';
	zero[lpc+1] = '0';
	zero[lpc+2] = 0;
	return zero;
	}
	}
	return NULL;
	}

	char *
	increment_clone(char *last_rsc_id)
	{
	int lpc = 0;
	int len = 0;
	char *tmp = NULL;
	gboolean complete = FALSE;

	CRM_CHECK(last_rsc_id != NULL, return NULL);
	if(last_rsc_id != NULL) {
	len = strlen(last_rsc_id);
	}

	lpc = len-1;
	while(complete == FALSE && lpc > 0) {
	switch (last_rsc_id[lpc]) {
	case 0:
	lpc--;
	break;
	case '0':
	set_char('1');
	break;
	case '1':
	set_char('2');
	break;
	case '2':
	set_char('3');
	break;
	case '3':
	set_char('4');
	break;
	case '4':
	set_char('5');
	break;
	case '5':
	set_char('6');
	break;
	case '6':
	set_char('7');
	break;
	case '7':
	set_char('8');
	break;
	case '8':
	set_char('9');
	break;
	case '9':
	last_rsc_id[lpc] = '0';
	lpc--;
	break;
	case ':':
	tmp = last_rsc_id;
	crm_malloc0(last_rsc_id, len + 2);
	memcpy(last_rsc_id, tmp, len);
	last_rsc_id[++lpc] = '1';
	last_rsc_id[len] = '0';
	last_rsc_id[len+1] = 0;
	complete = TRUE;
	crm_free(tmp);
	break;
	default:
	crm_err("Unexpected char: %c (%d)",
	last_rsc_id[lpc], lpc);
	break;
	}
	}
	return last_rsc_id;
	}

	static resource_t *
	create_fake_resource(const char rsc_id, xmlNode rsc_entry, pe_working_set_t *data_set)
	{
	resource_t *rsc = NULL;
	xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
	copy_in_properties(xml_rsc, rsc_entry);
	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
	crm_log_xml_debug(xml_rsc, "Orphan resource");

	common_unpack(xml_rsc, &rsc, NULL, data_set);
	set_bit(rsc->flags, pe_rsc_orphan);

	data_set->resources = g_list_append(data_set->resources, rsc);
	return rsc;
	}

	extern resource_t create_child_clone(resource_t rsc, int sub_id, pe_working_set_t *data_set);

	static resource_t find_clone(pe_working_set_t data_set, node_t node, resource_t parent, const char *rsc_id)
	{
	int len = 0;
	resource_t *rsc = NULL;
	char *base = clone_zero(rsc_id);
	char *alt_rsc_id = crm_strdup(rsc_id);

	CRM_ASSERT(parent != NULL);
	CRM_ASSERT(parent->variant == pe_clone \|\| parent->variant == pe_master);

	if(base) {
	len = strlen(base);
	}
	if(len > 0) {
	base[len-1] = 0;
	}

	crm_debug_3("Looking for %s on %s in %s %d",
	rsc_id, node->details->uname, parent->id, is_set(parent->flags, pe_rsc_unique));

	if(is_set(parent->flags, pe_rsc_unique)) {
	crm_debug_3("Looking for %s", rsc_id);
	rsc = parent->fns->find_rsc(parent, rsc_id, FALSE, FALSE, NULL, TRUE);

	} else {
	rsc = parent->fns->find_rsc(parent, base, FALSE, TRUE, node, TRUE);
	if(rsc != NULL && rsc->running_on) {
	rsc = NULL;
	crm_debug_3("Looking for an existing orphan for %s: %s on %s", parent->id, rsc_id, node->details->uname);

	/* There is already an instance of this _anonymous_ clone active on "node".
	*
	* If there is a partially active orphan (only applies to clone groups) on
	* the same node, use that.
	* Otherwise create a new (orphaned) instance at "orphan_check:".
	*/
	slist_iter(child, resource_t, parent->children, lpc,
	node_t *loc = child->fns->location(child, NULL, TRUE);
	if(loc && loc->details == node->details) {
	resource_t *tmp = child->fns->find_rsc(child, base, FALSE, TRUE, NULL, TRUE);
	if(tmp && tmp->running_on == NULL) {
	rsc = tmp;
	break;
	}
	}
	);

	goto orphan_check;
	}

	while(rsc == NULL) {
	crm_debug_3("Trying %s", alt_rsc_id);
	rsc = parent->fns->find_rsc(parent, alt_rsc_id, FALSE, FALSE, NULL, TRUE);
	if(rsc == NULL) {
	break;

	} else if(rsc->running_on == NULL) {
	break;
	}
	alt_rsc_id = increment_clone(alt_rsc_id);
	rsc = NULL;
	}
	}

	orphan_check:
	if(rsc == NULL) {
	/* Create an extra orphan */
	resource_t *top = create_child_clone(parent, -1, data_set);
	crm_debug("Created orphan for %s: %s on %s", parent->id, rsc_id, node->details->uname);
	rsc = top->fns->find_rsc(top, base, FALSE, TRUE, NULL, TRUE);
	CRM_ASSERT(rsc != NULL);
	}

	crm_free(rsc->clone_name); rsc->clone_name = NULL;
	if(safe_str_neq(rsc_id, rsc->id)) {
	crm_info("Internally renamed %s on %s to %s%s",
	rsc_id, node->details->uname, rsc->id,
	is_set(rsc->flags, pe_rsc_orphan)?" (ORPHAN)":"");
	rsc->clone_name = crm_strdup(rsc_id);
	}

	crm_free(alt_rsc_id);
	crm_free(base);
	return rsc;
	}

	static resource_t *
	unpack_find_resource(
	pe_working_set_t data_set, node_t node, const char rsc_id, xmlNode rsc_entry)
	{
	resource_t *rsc = NULL;
	resource_t *clone_parent = NULL;
	char *alt_rsc_id = crm_strdup(rsc_id);

	crm_debug_2("looking for %s", rsc_id);

	rsc = pe_find_resource(data_set->resources, alt_rsc_id);
	/* no match */
	if(rsc == NULL) {
	/* Even when clone-max=0, we still create a single :0 orphan to match against */
	char *tmp = clone_zero(alt_rsc_id);
	resource_t *clone0 = pe_find_resource(data_set->resources, tmp);
	clone_parent = uber_parent(clone0);
	crm_free(tmp);

	crm_debug_2("%s not found: %s", alt_rsc_id, clone_parent?clone_parent->id:"orphan");

	} else {
	clone_parent = uber_parent(rsc);
	}

	if(clone_parent && clone_parent->variant > pe_group) {
	rsc = find_clone(data_set, node, clone_parent, rsc_id);
	CRM_ASSERT(rsc != NULL);
	}

	crm_free(alt_rsc_id);
	return rsc;
	}

	static resource_t *
	process_orphan_resource(xmlNode rsc_entry, node_t node, pe_working_set_t *data_set)
	{
	resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);

	crm_debug("Detected orphan resource %s on %s", rsc_id, node->details->uname);
	rsc = create_fake_resource(rsc_id, rsc_entry, data_set);

	if(is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
	clear_bit(rsc->flags, pe_rsc_managed);

	} else {
	print_resource(LOG_DEBUG_3, "Added orphan", rsc, FALSE);

	CRM_CHECK(rsc != NULL, return NULL);
	resource_location(rsc, NULL, -INFINITY, "__orphan_dont_run__", data_set);
	}
	return rsc;
	}

	static void
	process_rsc_state(resource_t rsc, node_t node,
	enum action_fail_response on_fail,
	xmlNode *migrate_op,
	pe_working_set_t *data_set)
	{
	if(on_fail == action_migrate_failure) {
	node_t *from = NULL;
	const char *uuid = crm_element_value(migrate_op, CRMD_ACTION_MIGRATED);

	on_fail = action_fail_recover;

	from = pe_find_node_id(data_set->nodes, uuid);
	if(from != NULL) {
	process_rsc_state(rsc, from, on_fail, NULL, data_set);
	} else {
	crm_log_xml_err(migrate_op, "Bad Op");
	}
	}

	crm_debug_2("Resource %s is %s on %s: on_fail=%s",
	rsc->id, role2text(rsc->role),
	node->details->uname, fail2text(on_fail));

	/* process current state */
	if(rsc->role != RSC_ROLE_UNKNOWN) {
	rsc->known_on = g_list_append(rsc->known_on, node);
	}

	if(node->details->unclean) {
	/* No extra processing needed
	* Also allows resources to be started again after a node is shot
	*/
	on_fail = action_fail_ignore;
	}

	switch(on_fail) {
	case action_fail_ignore:
	/* nothing to do */
	break;

	case action_fail_fence:
	/* treat it as if it is still running
	* but also mark the node as unclean
	*/
	pe_fence_node(data_set, node, "to recover from resource failure(s)");
	break;

	case action_fail_standby:
	node->details->standby = TRUE;
	node->details->standby_onfail = TRUE;
	break;

	case action_fail_block:
	/* is_managed == FALSE will prevent any
	* actions being sent for the resource
	*/
	clear_bit(rsc->flags, pe_rsc_managed);
	break;

	case action_fail_migrate:
	/* make sure it comes up somewhere else
	* or not at all
	*/
	resource_location(rsc, node, -INFINITY,
	"__action_migration_auto__",data_set);
	break;

	case action_fail_stop:
	rsc->next_role = RSC_ROLE_STOPPED;
	break;

	case action_fail_recover:
	if(rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
	set_bit(rsc->flags, pe_rsc_failed);
	stop_action(rsc, node, FALSE);
	}
	break;

	case action_migrate_failure:
	/* anything extra? */
	break;
	}

	if(rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
	if(is_set(rsc->flags, pe_rsc_orphan)) {
	if(is_set(rsc->flags, pe_rsc_managed)) {
	crm_config_warn("Detected active orphan %s running on %s",
	rsc->id, node->details->uname);
	} else {
	crm_config_warn("Cluster configured not to stop active orphans."
	" %s must be stopped manually on %s",
	rsc->id, node->details->uname);
	}
	}

	native_add_running(rsc, node, data_set);
	if(on_fail != action_fail_ignore) {
	set_bit(rsc->flags, pe_rsc_failed);
	}

	} else if(rsc->clone_name) {
	crm_debug_2("Resetting clone_name %s for %s (stopped)",
	rsc->clone_name, rsc->id);
	crm_free(rsc->clone_name);
	rsc->clone_name = NULL;

	} else {
	char *key = stop_key(rsc);
	GListPtr possible_matches = find_actions(rsc->actions, key, node);
	slist_iter(stop, action_t, possible_matches, lpc,
	stop->optional = TRUE;
	);
	crm_free(key);
	}
	}

	/* create active recurring operations as optional */
	static void
	process_recurring(node_t node, resource_t rsc,
	int start_index, int stop_index,
	GListPtr sorted_op_list, pe_working_set_t *data_set)
	{
	const char *task = NULL;
	const char *status = NULL;

	crm_debug_3("%s: Start index %d, stop index = %d",
	rsc->id, start_index, stop_index);
	slist_iter(rsc_op, xmlNode, sorted_op_list, lpc,
	int interval = 0;
	char *key = NULL;
	const char *id = ID(rsc_op);
	const char *interval_s = NULL;
	if(node->details->online == FALSE) {
	crm_debug_4("Skipping %s/%s: node is offline",
	rsc->id, node->details->uname);
	break;

	} else if(start_index < stop_index) {
	crm_debug_4("Skipping %s/%s: not active",
	rsc->id, node->details->uname);
	break;

	} else if(lpc <= start_index) {
	crm_debug_4("Skipping %s/%s: old",
	id, node->details->uname);
	continue;
	}

	interval_s = crm_element_value(rsc_op,XML_LRM_ATTR_INTERVAL);
	interval = crm_parse_int(interval_s, "0");
	if(interval == 0) {
	crm_debug_4("Skipping %s/%s: non-recurring",
	id, node->details->uname);
	continue;
	}

	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
	if(safe_str_eq(status, "-1")) {
	crm_debug_4("Skipping %s/%s: status",
	id, node->details->uname);
	continue;
	}
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	/* create the action */
	key = generate_op_key(rsc->id, task, interval);
	crm_debug_3("Creating %s/%s", key, node->details->uname);
	custom_action(rsc, key, task, node, TRUE, TRUE, data_set);
	);
	}

	void
	calculate_active_ops(GListPtr sorted_op_list, int start_index, int stop_index)
	{
	const char *task = NULL;
	const char *status = NULL;

	*stop_index = -1;
	*start_index = -1;

	slist_iter(
	rsc_op, xmlNode, sorted_op_list, lpc,

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);

	if(safe_str_eq(task, CRMD_ACTION_STOP)
	&& safe_str_eq(status, "0")) {
	*stop_index = lpc;

	} else if(safe_str_eq(task, CRMD_ACTION_START)) {
	*start_index = lpc;

	} else if(start_index <= stop_index
	&& safe_str_eq(task, CRMD_ACTION_STATUS)) {
	const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
	if(safe_str_eq(rc, "0") \|\| safe_str_eq(rc, "8")) {
	*start_index = lpc;
	}
	}
	);
	}


	static void
	unpack_lrm_rsc_state(
	node_t node, xmlNode rsc_entry, pe_working_set_t *data_set)
	{
	int stop_index = -1;
	int start_index = -1;
	enum rsc_role_e req_role = RSC_ROLE_UNKNOWN;

	const char *task = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);

	resource_t *rsc = NULL;
	GListPtr op_list = NULL;
	GListPtr sorted_op_list = NULL;

	xmlNode *migrate_op = NULL;

	enum action_fail_response on_fail = FALSE;
	enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN;

	crm_debug_3("[%s] Processing %s on %s",
	crm_element_name(rsc_entry), rsc_id, node->details->uname);

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	xml_child_iter_filter(
	rsc_entry, rsc_op, XML_LRM_TAG_RSC_OP,
	op_list = g_list_append(op_list, rsc_op);
	);

	if(op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return;
	}

	/* find the resource */
	rsc = unpack_find_resource(data_set, node, rsc_id, rsc_entry);
	if(rsc == NULL) {
	rsc = process_orphan_resource(rsc_entry, node, data_set);
	}
	CRM_ASSERT(rsc != NULL);

	/* process operations */
	saved_role = rsc->role;
	on_fail = action_fail_ignore;
	rsc->role = RSC_ROLE_UNKNOWN;
	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	slist_iter(
	rsc_op, xmlNode, sorted_op_list, lpc,

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	if(safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
	migrate_op = rsc_op;
	}

	unpack_rsc_op(rsc, node, rsc_op, &on_fail, data_set);
	);

	/* create active recurring operations as optional */
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
	process_recurring(node, rsc, start_index, stop_index,
	sorted_op_list, data_set);

	/* no need to free the contents */
	g_list_free(sorted_op_list);

	process_rsc_state(rsc, node, on_fail, migrate_op, data_set);

	if(get_target_role(rsc, &req_role)) {
	if(rsc->next_role == RSC_ROLE_UNKNOWN \|\| req_role < rsc->next_role) {
	crm_debug("%s: Overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, role2text(rsc->next_role),
	role2text(req_role));
	rsc->next_role = req_role;

	} else if(req_role > rsc->next_role) {
	crm_info("%s: Not overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, role2text(rsc->next_role),
	role2text(req_role));
	}
	}

	if(saved_role > rsc->role) {
	rsc->role = saved_role;
	}
	}

	gboolean
	unpack_lrm_resources(node_t node, xmlNode lrm_rsc_list, pe_working_set_t *data_set)
	{
	CRM_CHECK(node != NULL, return FALSE);

	crm_debug_3("Unpacking resources on %s", node->details->uname);

	xml_child_iter_filter(
	lrm_rsc_list, rsc_entry, XML_LRM_TAG_RESOURCE,
	unpack_lrm_rsc_state(node, rsc_entry, data_set);
	);

	return TRUE;
	}

	static void set_active(resource_t *rsc)
	{
	resource_t *top = uber_parent(rsc);
	if(top && top->variant == pe_master) {
	rsc->role = RSC_ROLE_SLAVE;
	} else {
	rsc->role = RSC_ROLE_STARTED;
	}
	}

	gboolean
	unpack_rsc_op(resource_t rsc, node_t node, xmlNode *xml_op,
	enum action_fail_response on_fail, pe_working_set_t data_set)
	{
	const char *id = NULL;
	const char *key = NULL;
	const char *task = NULL;
	const char *magic = NULL;
	const char *task_id = NULL;
	const char *actual_rc = NULL;
	/* const char target_rc = NULL; /
	const char *task_status = NULL;
	const char *interval_s = NULL;
	const char *op_digest = NULL;
	const char *op_version = NULL;

	int interval = 0;
	int task_status_i = -2;
	int actual_rc_i = 0;
	int target_rc = -1;

	action_t *action = NULL;
	node_t *effective_node = NULL;
	resource_t *failed = NULL;

	gboolean expired = FALSE;
	gboolean is_probe = FALSE;

	CRM_CHECK(rsc != NULL, return FALSE);
	CRM_CHECK(node != NULL, return FALSE);
	CRM_CHECK(xml_op != NULL, return FALSE);

	id = ID(xml_op);
	task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	task_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
	task_status = crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS);
	op_digest = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
	op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
	magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);
	key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);

	CRM_CHECK(id != NULL, return FALSE);
	CRM_CHECK(task != NULL, return FALSE);
	CRM_CHECK(task_status != NULL, return FALSE);

	task_status_i = crm_parse_int(task_status, NULL);

	CRM_CHECK(task_status_i <= LRM_OP_ERROR, return FALSE);
	CRM_CHECK(task_status_i >= LRM_OP_PENDING, return FALSE);

	if(safe_str_eq(task, CRMD_ACTION_NOTIFY)) {
	/* safe to ignore these */
	return TRUE;
	}

	if(rsc->failure_timeout > 0) {
	int last_run = 0;

	if(crm_element_value_int(xml_op, "last-run", &last_run) == 0) {
	/* int last_change = crm_element_value_int(xml_op, "last_rc_change"); */
	time_t now = get_timet_now(data_set);
	if(now > (last_run + rsc->failure_timeout)) {
	expired = TRUE;
	}
	}
	}

	crm_debug_2("Unpacking task %s/%s (call_id=%s, status=%s) on %s (role=%s)",
	id, task, task_id, task_status, node->details->uname,
	role2text(rsc->role));

	interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
	interval = crm_parse_int(interval_s, "0");

	if(interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
	is_probe = TRUE;
	}

	if(node->details->unclean) {
	crm_debug_2("Node %s (where %s is running) is unclean."
	" Further action depends on the value of the stop's on-fail attribue",
	node->details->uname, rsc->id);
	}

	actual_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC);
	CRM_CHECK(actual_rc != NULL, return FALSE);
	actual_rc_i = crm_parse_int(actual_rc, NULL);

	if(key) {
	int dummy = 0;
	char *dummy_string = NULL;
	decode_transition_key(key, &dummy_string, &dummy, &dummy, &target_rc);
	crm_free(dummy_string);
	}

	if(task_status_i == LRM_OP_DONE && target_rc >= 0) {
	if(target_rc == actual_rc_i) {
	task_status_i = LRM_OP_DONE;

	} else {
	task_status_i = LRM_OP_ERROR;
	crm_debug("%s on %s returned %d (%s) instead of the expected value: %d (%s)",
	id, node->details->uname,
	actual_rc_i, execra_code2string(actual_rc_i),
	target_rc, execra_code2string(target_rc));
	}

	} else if(task_status_i == LRM_OP_ERROR) {
	/* let us decide that */
	task_status_i = LRM_OP_DONE;
	}

	if(task_status_i == LRM_OP_NOTSUPPORTED) {
	actual_rc_i = EXECRA_UNIMPLEMENT_FEATURE;
	}

	if(task_status_i != actual_rc_i
	&& rsc->failure_timeout > 0
	&& get_failcount(node, rsc, NULL, data_set) == 0) {
	action_t *clear_op = NULL;
	clear_op = custom_action(
	rsc, crm_concat(rsc->id, CRM_OP_CLEAR_FAILCOUNT, '_'),
	CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE, data_set);

	add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	crm_notice("Clearing expired failcount for %s on %s", rsc->id, node->details->uname);
	}

	if(expired
	&& actual_rc_i != EXECRA_NOT_RUNNING
	&& actual_rc_i != EXECRA_RUNNING_MASTER
	&& actual_rc_i != EXECRA_OK) {
	crm_notice("Ignoring expired failure %s (rc=%d, magic=%s) on %s",
	id, actual_rc_i, magic, node->details->uname);
	goto done;
	}

	/* we could clean this up significantly except for old LRMs and CRMs that
	* didnt include target_rc and liked to remap status
	*/
	switch(actual_rc_i) {
	case EXECRA_NOT_RUNNING:
	if(is_probe \|\| target_rc == actual_rc_i) {
	task_status_i = LRM_OP_DONE;
	rsc->role = RSC_ROLE_STOPPED;

	/* clear any previous failure actions */
	*on_fail = action_fail_ignore;
	rsc->next_role = RSC_ROLE_UNKNOWN;

	} else if(safe_str_neq(task, CRMD_ACTION_STOP)) {
	task_status_i = LRM_OP_ERROR;
	}
	break;

	case EXECRA_RUNNING_MASTER:
	if(is_probe) {
	task_status_i = LRM_OP_DONE;
	crm_notice("Operation %s found resource %s active in master mode on %s",
	id, rsc->id, node->details->uname);

	} else if(target_rc == actual_rc_i) {
	/* nothing to do */

	} else if(target_rc >= 0) {
	task_status_i = LRM_OP_ERROR;

	/* legacy code for pre-0.6.5 operations */
	} else if(safe_str_neq(task, CRMD_ACTION_STATUS)
	\|\| rsc->role != RSC_ROLE_MASTER) {
	task_status_i = LRM_OP_ERROR;
	if(rsc->role != RSC_ROLE_MASTER) {
	crm_err("%s reported %s in master mode on %s",
	id, rsc->id,
	node->details->uname);
	}
	}
	rsc->role = RSC_ROLE_MASTER;
	break;

	case EXECRA_FAILED_MASTER:
	rsc->role = RSC_ROLE_MASTER;
	task_status_i = LRM_OP_ERROR;
	break;

	case EXECRA_UNIMPLEMENT_FEATURE:
	if(interval > 0) {
	task_status_i = LRM_OP_NOTSUPPORTED;
	break;
	}
	/* else: fall through */
	case EXECRA_INSUFFICIENT_PRIV:
	case EXECRA_NOT_INSTALLED:
	case EXECRA_INVALID_PARAM:
	effective_node = node;
	/* fall through */
	case EXECRA_NOT_CONFIGURED:
	failed = rsc;
	if(is_not_set(rsc->flags, pe_rsc_unique)) {
	failed = uber_parent(failed);
	}

	do_crm_log(actual_rc_i==EXECRA_NOT_INSTALLED?LOG_NOTICE:LOG_ERR,
	"Hard error - %s failed with rc=%d: Preventing %s from re-starting %s %s",
	id, actual_rc_i, failed->id,
	effective_node?"on":"anywhere",
	effective_node?effective_node->details->uname:"in the cluster");

	resource_location(failed, effective_node, -INFINITY, "hard-error", data_set);
	if(is_probe) {
	/* treat these like stops */
	task = CRMD_ACTION_STOP;
	task_status_i = LRM_OP_DONE;
	crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
	add_node_copy(data_set->failed, xml_op);
	}
	break;

	case EXECRA_OK:
	if(is_probe && target_rc == 7) {
	task_status_i = LRM_OP_DONE;
	crm_notice("Operation %s found resource %s active on %s",
	id, rsc->id, node->details->uname);

	/* legacy code for pre-0.6.5 operations */
	} else if(target_rc < 0
	&& interval > 0
	&& rsc->role == RSC_ROLE_MASTER) {
	/* catch status ops that return 0 instead of 8 while they
	* are supposed to be in master mode
	*/
	task_status_i = LRM_OP_ERROR;
	}

	break;

	default:
	if(task_status_i == LRM_OP_DONE) {
	crm_info("Remapping %s (rc=%d) on %s to an ERROR",
	id, actual_rc_i, node->details->uname);
	task_status_i = LRM_OP_ERROR;
	}
	}

	if(task_status_i == LRM_OP_ERROR
	\|\| task_status_i == LRM_OP_TIMEOUT
	\|\| task_status_i == LRM_OP_NOTSUPPORTED) {
	action = custom_action(rsc, crm_strdup(id), task, NULL, TRUE, FALSE, data_set);
	if(expired) {
	crm_notice("Ignoring expired failure (calculated) %s (rc=%d, magic=%s) on %s",
	id, actual_rc_i, magic, node->details->uname);
	goto done;

	} else if(action->on_fail == action_fail_ignore) {
	crm_warn("Remapping %s (rc=%d) on %s to DONE: ignore",
	id, actual_rc_i, node->details->uname);
	task_status_i = LRM_OP_DONE;
	}
	}

	switch(task_status_i) {
	case LRM_OP_PENDING:
	if(safe_str_eq(task, CRMD_ACTION_START)) {
	set_bit(rsc->flags, pe_rsc_start_pending);
	set_active(rsc);

	} else if(safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
	rsc->role = RSC_ROLE_MASTER;
	}
	break;

	case LRM_OP_DONE:
	crm_debug_3("%s/%s completed on %s",
	rsc->id, task, node->details->uname);

	if(actual_rc_i == EXECRA_NOT_RUNNING) {
	/* nothing to do */

	} else if(safe_str_eq(task, CRMD_ACTION_STOP)) {
	rsc->role = RSC_ROLE_STOPPED;

	/* clear any previous failure actions */
	switch(*on_fail) {
	case action_fail_block:
	case action_fail_stop:
	case action_fail_fence:
	case action_fail_migrate:
	case action_fail_standby:
	crm_debug_2("%s.%s is not cleared by a completed stop",
	rsc->id, fail2text(*on_fail));
	break;

	case action_fail_ignore:
	case action_fail_recover:
	case action_migrate_failure:
	*on_fail = action_fail_ignore;
	rsc->next_role = RSC_ROLE_UNKNOWN;
	}

	} else if(safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
	rsc->role = RSC_ROLE_MASTER;

	} else if(safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
	rsc->role = RSC_ROLE_SLAVE;

	} else if(rsc->role < RSC_ROLE_STARTED) {
	crm_debug_3("%s active on %s",
	rsc->id, node->details->uname);
	set_active(rsc);
	}
	break;

	case LRM_OP_ERROR:
	case LRM_OP_TIMEOUT:
	case LRM_OP_NOTSUPPORTED:
	crm_warn("Processing failed op %s on %s: %s (%d)",
	id, node->details->uname,
	execra_code2string(actual_rc_i), actual_rc_i);
	crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
	add_node_copy(data_set->failed, xml_op);

	if(*on_fail < action->on_fail) {
	*on_fail = action->on_fail;
	}

	if(safe_str_eq(task, CRMD_ACTION_STOP)) {
	resource_location(
	rsc, node, -INFINITY, "__stop_fail__", data_set);

	} else if(safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
	rsc->role = RSC_ROLE_MASTER;

	} else if(safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
	/*
	* staying in role=master ends up putting the PE/TE into a loop
	* setting role=slave is not dangerous because no master will be
	* promoted until the failed resource has been fully stopped
	*/
	crm_warn("Forcing %s to stop after a failed demote action", rsc->id);
	rsc->next_role = RSC_ROLE_STOPPED;
	rsc->role = RSC_ROLE_SLAVE;

	} else if(compare_version("2.0", op_version) > 0
	&& safe_str_eq(task, CRMD_ACTION_START)) {
	crm_warn("Compatibility handling for failed op %s on %s",
	id, node->details->uname);
	resource_location(
	rsc, node, -INFINITY, "__legacy_start__", data_set);
	}

	if(rsc->role < RSC_ROLE_STARTED) {
	set_active(rsc);
	}

	crm_debug_2("Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s",
	rsc->id, role2text(rsc->role),
	node->details->unclean?"true":"false",
	fail2text(action->on_fail),
	role2text(action->fail_role));

	if(action->fail_role != RSC_ROLE_STARTED
	&& rsc->next_role < action->fail_role) {
	rsc->next_role = action->fail_role;
	}

	if(action->fail_role == RSC_ROLE_STOPPED) {
	crm_err("Making sure %s doesn't come up again", rsc->id);
	/* make sure it doesnt come up again */
	pe_free_shallow_adv(rsc->allowed_nodes, TRUE);
	rsc->allowed_nodes = node_list_dup(
	data_set->nodes, FALSE, FALSE);
	slist_iter(
	node, node_t, rsc->allowed_nodes, lpc,
	node->weight = -INFINITY;
	);
	}

	pe_free_action(action);
	action = NULL;
	break;
	case LRM_OP_CANCELLED:
	/* do nothing?? */
	pe_err("Dont know what to do for cancelled ops yet");
	break;
	}

	done:
	crm_debug_3("Resource %s after %s: role=%s",
	rsc->id, task, role2text(rsc->role));

	pe_free_action(action);

	return TRUE;
	}

	gboolean
	add_node_attrs(xmlNode xml_obj, node_t node, gboolean overwrite, pe_working_set_t *data_set)
	{
	g_hash_table_insert(node->details->attrs,
	crm_strdup("#"XML_ATTR_UNAME),
	crm_strdup(node->details->uname));
	g_hash_table_insert(node->details->attrs,
	crm_strdup("#"XML_ATTR_ID),
	crm_strdup(node->details->id));
	if(safe_str_eq(node->details->id, data_set->dc_uuid)) {
	data_set->dc_node = node;
	node->details->is_dc = TRUE;
	g_hash_table_insert(node->details->attrs,
	crm_strdup("#"XML_ATTR_DC),
	crm_strdup(XML_BOOLEAN_TRUE));
	} else {
	g_hash_table_insert(node->details->attrs,
	crm_strdup("#"XML_ATTR_DC),
	crm_strdup(XML_BOOLEAN_FALSE));
	}

	unpack_instance_attributes(
	data_set->input, xml_obj, XML_TAG_ATTR_SETS, NULL,
	node->details->attrs, NULL, overwrite, data_set->now);

	return TRUE;
	}

	static GListPtr
	extract_operations(const char node, const char rsc, xmlNode *rsc_entry, gboolean active_filter)
	{
	int stop_index = -1;
	int start_index = -1;

	GListPtr op_list = NULL;
	GListPtr sorted_op_list = NULL;

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	xml_child_iter_filter(
	rsc_entry, rsc_op, XML_LRM_TAG_RSC_OP,
	crm_xml_add(rsc_op, "resource", rsc);
	crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
	op_list = g_list_append(op_list, rsc_op);
	);

	if(op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return NULL;
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	/* create active recurring operations as optional */
	if(active_filter == FALSE) {
	return sorted_op_list;
	}

	op_list = NULL;

	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
	slist_iter(rsc_op, xmlNode, sorted_op_list, lpc,
	if(start_index < stop_index) {
	crm_debug_4("Skipping %s: not active", ID(rsc_entry));
	break;

	} else if(lpc < start_index) {
	crm_debug_4("Skipping %s: old", ID(rsc_op));
	continue;
	}
	op_list = g_list_append(op_list, rsc_op);
	);

	g_list_free(sorted_op_list);
	return op_list;
	}

	GListPtr find_operations(
	const char rsc, const char node, gboolean active_filter, pe_working_set_t *data_set)
	{
	GListPtr output = NULL;
	GListPtr intermediate = NULL;

	xmlNode *tmp = NULL;
	xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE);

	const char *uname = NULL;
	node_t *this_node = NULL;

	xml_child_iter_filter(
	status, node_state, XML_CIB_TAG_STATE,

	uname = crm_element_value(node_state, XML_ATTR_UNAME);
	if(node != NULL && safe_str_neq(uname, node)) {
	continue;
	}

	this_node = pe_find_node(data_set->nodes, uname);
	CRM_CHECK(this_node != NULL, continue);

	determine_online_status(node_state, this_node, data_set);

	if(this_node->details->online \|\| is_set(data_set->flags, pe_flag_stonith_enabled)) {
	/* offline nodes run no resources...
	* unless stonith is enabled in which case we need to
	* make sure rsc start events happen after the stonith
	*/
	tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);

	xml_child_iter_filter(
	tmp, lrm_rsc, XML_LRM_TAG_RESOURCE,
	const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);
	if(rsc != NULL && safe_str_neq(rsc_id, rsc)) {
	continue;
	}

	intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
	output = g_list_concat(output, intermediate);
	);
	}
	);

	return output;
	}
	diff --git a/lib/pengine/unpack.h b/lib/pengine/unpack.h
	index 14beebd7ce..fa57928050 100644
	--- a/lib/pengine/unpack.h
	+++ b/lib/pengine/unpack.h
	@@ -1,94 +1,96 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#ifndef PENGINE_UNPACK__H
	#define PENGINE_UNPACK__H

	extern gboolean unpack_resources(
	xmlNode xml_resources, pe_working_set_t data_set);

	extern gboolean unpack_config(xmlNode config, pe_working_set_t data_set);

	extern gboolean unpack_nodes(xmlNode xml_nodes, pe_working_set_t data_set);

	+extern gboolean unpack_domains(xmlNode xml_domains, pe_working_set_t data_set);
	+
	extern gboolean unpack_status(xmlNode status, pe_working_set_t data_set);

	extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);

	extern gboolean unpack_lrm_resources(
	node_t node, xmlNode lrm_state, pe_working_set_t *data_set);

	extern gboolean add_node_attrs(
	xmlNode * attrs, node_t node, gboolean overwrite, pe_working_set_t data_set);

	extern gboolean determine_online_status(
	xmlNode * node_state, node_t this_node, pe_working_set_t data_set);

	extern const char *param_value(
	GHashTable hash, xmlNode parent, const char *name);

	extern char clone_zero(const char last_rsc_id);
	extern char increment_clone(char last_rsc_id);

	/*
	* The man pages for both curses and ncurses suggest inclusion of "curses.h".
	* We believe the following to be acceptable and portable.
	*/

	#if defined(HAVE_LIBNCURSES) \|\| defined(HAVE_LIBCURSES)
	# if defined(HAVE_NCURSES_H) && !defined(HAVE_INCOMPATIBLE_PRINTW)
	# include <ncurses.h>
	# define CURSES_ENABLED 1
	# elif defined(HAVE_NCURSES_NCURSES_H) && !defined(HAVE_INCOMPATIBLE_PRINTW)
	# include <ncurses/ncurses.h>
	# define CURSES_ENABLED 1
	# elif defined(HAVE_CURSES_H) && !defined(HAVE_INCOMPATIBLE_PRINTW)
	# include <curses.h>
	# define CURSES_ENABLED 1
	# elif defined(HAVE_CURSES_CURSES_H) && !defined(HAVE_INCOMPATIBLE_PRINTW)
	# include <curses/curses.h>
	# define CURSES_ENABLED 1
	# else
	# define CURSES_ENABLED 0
	# endif
	#else
	# define CURSES_ENABLED 0
	#endif

	#if CURSES_ENABLED
	# define status_printw(fmt, args...) printw(fmt, ##args)
	#else
	# define status_printw(fmt, args...) \
	crm_err("printw support requires ncurses to be available during configure"); \
	do_crm_log(LOG_WARNING, fmt, ##args);
	#endif

	#define status_print(fmt, args...) \
	if(options & pe_print_html) { \
	FILE *stream = print_data; \
	fprintf(stream, fmt, ##args); \
	} else if(options & pe_print_ncurses) { \
	status_printw(fmt, ##args); \
	} else if(options & pe_print_printf) { \
	FILE *stream = print_data; \
	fprintf(stream, fmt, ##args); \
	} else if(options & pe_print_log) { \
	int log_level = (int)print_data; \
	do_crm_log(log_level, fmt, ##args); \
	}

	#endif
	diff --git a/pengine/constraints.c b/pengine/constraints.c
	index fc6d41609d..5c45897bba 100644
	--- a/pengine/constraints.c
	+++ b/pengine/constraints.c
	@@ -1,1247 +1,1278 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/msg.h>


	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <pengine.h>
	#include <allocate.h>
	#include <utils.h>
	#include <crm/pengine/rules.h>
	#include <lib/pengine/utils.h>

	enum pe_order_kind
	{
	pe_order_kind_optional,
	pe_order_kind_mandatory,
	pe_order_kind_serialize,
	};

	enum pe_ordering get_flags(
	const char *id, enum pe_order_kind kind,
	const char action_first, const char action_then, gboolean invert);

	gboolean
	unpack_constraints(xmlNode * xml_constraints, pe_working_set_t *data_set)
	{
	xmlNode *lifetime = NULL;
	xml_child_iter(
	xml_constraints, xml_obj,

	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	if(id == NULL) {
	crm_config_err("Constraint <%s...> must have an id",
	crm_element_name(xml_obj));
	continue;
	}

	crm_debug_3("Processing constraint %s %s",
	crm_element_name(xml_obj),id);

	lifetime = first_named_child(xml_obj, "lifetime");
	if(lifetime) {
	crm_config_warn("Support for the lifetime tag, used by %s, is deprecated."
	" The rules it contains should instead be direct decendants of the constraint object", id);
	}

	if(test_ruleset(lifetime, NULL, data_set->now) == FALSE) {
	crm_info("Constraint %s %s is not active",
	crm_element_name(xml_obj), id);

	} else if(safe_str_eq(XML_CONS_TAG_RSC_ORDER,
	crm_element_name(xml_obj))) {
	unpack_rsc_order(xml_obj, data_set);

	} else if(safe_str_eq(XML_CONS_TAG_RSC_DEPEND,
	crm_element_name(xml_obj))) {
	unpack_rsc_colocation(xml_obj, data_set);

	} else if(safe_str_eq(XML_CONS_TAG_RSC_LOCATION,
	crm_element_name(xml_obj))) {
	unpack_rsc_location(xml_obj, data_set);

	} else {
	pe_err("Unsupported constraint type: %s",
	crm_element_name(xml_obj));
	}
	);

	return TRUE;
	}

	static const char *
	invert_action(const char *action)
	{
	if(safe_str_eq(action, RSC_START)) {
	return RSC_STOP;

	} else if(safe_str_eq(action, RSC_STOP)) {
	return RSC_START;

	} else if(safe_str_eq(action, RSC_PROMOTE)) {
	return RSC_DEMOTE;

	} else if(safe_str_eq(action, RSC_DEMOTE)) {
	return RSC_PROMOTE;

	} else if(safe_str_eq(action, RSC_PROMOTED)) {
	return RSC_DEMOTED;

	} else if(safe_str_eq(action, RSC_DEMOTED)) {
	return RSC_PROMOTED;

	} else if(safe_str_eq(action, RSC_STARTED)) {
	return RSC_STOPPED;

	} else if(safe_str_eq(action, RSC_STOPPED)) {
	return RSC_STARTED;
	}
	crm_config_warn("Unknown action: %s", action);
	return NULL;
	}

	static enum pe_order_kind get_ordering_type(xmlNode *xml_obj)
	{
	enum pe_order_kind kind_e = pe_order_kind_mandatory;
	const char *kind = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND);

	if(kind == NULL) {
	const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	kind_e = pe_order_kind_mandatory;

	if(score) {
	int score_i = char2score(score);
	if(score_i == 0) {
	kind_e = pe_order_kind_optional;
	}

	/* } else if(rsc_then->variant == pe_native && rsc_first->variant > pe_group) { */
	/* kind_e = pe_order_kind_optional; */
	}

	} else if(safe_str_eq(kind, "Mandatory")) {
	kind_e = pe_order_kind_mandatory;

	} else if(safe_str_eq(kind, "Optional")) {
	kind_e = pe_order_kind_optional;

	} else if(safe_str_eq(kind, "Serialize")) {
	kind_e = pe_order_kind_serialize;

	} else {
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	crm_config_err("Constraint %s: Unknown type '%s'", id, kind);
	}
	return kind_e;
	}

	static gboolean
	unpack_simple_rsc_order(xmlNode * xml_obj, pe_working_set_t *data_set)
	{
	int order_id = 0;
	resource_t *rsc_then = NULL;
	resource_t *rsc_first = NULL;
	gboolean invert_bool = TRUE;
	enum pe_order_kind kind = pe_order_kind_mandatory;
	enum pe_ordering cons_weight = pe_order_optional;

	const char *id_first = NULL;
	const char *id_then = NULL;
	const char *action_then = NULL;
	const char *action_first = NULL;
	const char *instance_then = NULL;
	const char *instance_first = NULL;

	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *invert = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);

	crm_str_to_boolean(invert, &invert_bool);

	if(xml_obj == NULL) {
	crm_config_err("No constraint object to process.");
	return FALSE;

	} else if(id == NULL) {
	crm_config_err("%s constraint must have an id",
	crm_element_name(xml_obj));
	return FALSE;
	}

	id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN);
	id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST);

	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);

	instance_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_INSTANCE);
	instance_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_INSTANCE);

	if(action_first == NULL) {
	action_first = RSC_START;
	}
	if(action_then == NULL) {
	action_then = action_first;
	}

	if(id_then == NULL \|\| id_first == NULL) {
	crm_config_err("Constraint %s needs two sides lh: %s rh: %s",
	id, crm_str(id_then), crm_str(id_first));
	return FALSE;
	}

	rsc_then = pe_find_resource(data_set->resources, id_then);
	rsc_first = pe_find_resource(data_set->resources, id_first);

	if(rsc_then == NULL) {
	crm_config_err("Constraint %s: no resource found for name '%s'", id, id_then);
	return FALSE;

	} else if(rsc_first == NULL) {
	crm_config_err("Constraint %s: no resource found for name '%s'", id, id_first);
	return FALSE;

	} else if(instance_then && rsc_then->variant < pe_clone) {
	crm_config_err("Invalid constraint '%s':"
	" Resource '%s' is not a clone but instance %s was requested",
	id, id_then, instance_then);
	return FALSE;

	} else if(instance_first && rsc_first->variant < pe_clone) {
	crm_config_err("Invalid constraint '%s':"
	" Resource '%s' is not a clone but instance %s was requested",
	id, id_first, instance_first);
	return FALSE;
	}

	if(instance_then) {
	rsc_then = find_clone_instance(rsc_then, instance_then, data_set);
	if(rsc_then == NULL) {
	crm_config_warn("Invalid constraint '%s': No instance '%s' of '%s'", id, instance_then, id_then);
	return FALSE;
	}
	}

	if(instance_first) {
	rsc_first = find_clone_instance(rsc_first, instance_first, data_set);
	if(rsc_first == NULL) {
	crm_config_warn("Invalid constraint '%s': No instance '%s' of '%s'", id, instance_first, id_first);
	return FALSE;
	}
	}

	cons_weight = pe_order_optional;
	kind = get_ordering_type(xml_obj);

	if(kind == pe_order_kind_optional && rsc_then->restart_type == pe_restart_restart) {
	crm_debug_2("Upgrade : recovery - implies right");
	cons_weight \|= pe_order_implies_right;
	}

	cons_weight \|= get_flags(id, kind, action_first, action_then, FALSE);
	order_id = new_rsc_order(
	rsc_first, action_first, rsc_then, action_then, cons_weight, data_set);

	crm_debug_2("order-%d (%s): %s_%s before %s_%s flags=0x%.6x",
	order_id, id, rsc_first->id, action_first, rsc_then->id, action_then,
	cons_weight);


	if(invert_bool == FALSE) {
	return TRUE;

	} else if(invert && kind == pe_order_kind_serialize) {
	crm_config_warn("Cannot invert serialized constraint set %s", id);
	return TRUE;

	} else if(kind == pe_order_kind_serialize) {
	return TRUE;
	}

	action_then = invert_action(action_then);
	action_first = invert_action(action_first);
	if(action_then == NULL \|\| action_first == NULL) {
	crm_config_err("Cannot invert rsc_order constraint %s."
	" Please specify the inverse manually.", id);
	return TRUE;
	}

	cons_weight = pe_order_optional;
	if(kind == pe_order_kind_optional && rsc_then->restart_type == pe_restart_restart) {
	crm_debug_2("Upgrade : recovery - implies left");
	cons_weight \|= pe_order_implies_left;
	}

	cons_weight \|= get_flags(id, kind, action_first, action_then, TRUE);
	order_id = new_rsc_order(
	rsc_then, action_then, rsc_first, action_first, cons_weight, data_set);

	crm_debug_2("order-%d (%s): %s_%s before %s_%s flags=0x%.6x",
	order_id, id, rsc_then->id, action_then, rsc_first->id, action_first,
	cons_weight);

	return TRUE;
	}

	gboolean
	unpack_rsc_location(xmlNode * xml_obj, pe_working_set_t *data_set)
	{
	gboolean empty = TRUE;
	+ rsc_to_node_t *location = NULL;
	const char *id_lh = crm_element_value(xml_obj, "rsc");
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	resource_t *rsc_lh = pe_find_resource(data_set->resources, id_lh);
	const char *node = crm_element_value(xml_obj, "node");
	const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	+ const char *domain = crm_element_value(xml_obj, XML_CIB_TAG_DOMAIN);
	+ const char *role = crm_element_value(xml_obj, XML_RULE_ATTR_ROLE);

	if(rsc_lh == NULL) {
	/* only a warn as BSC adds the constraint then the resource */
	crm_config_warn("No resource (con=%s, rsc=%s)", id, id_lh);
	return FALSE;
	}
	+
	+ if(domain) {
	+ GListPtr nodes = g_hash_table_lookup(data_set->domains, domain);
	+
	+ if(domain == NULL) {
	+ crm_config_err("Invalid constraint %s: Domain %s does not exist",
	+ id, domain);
	+ return FALSE;
	+ }

	- if(node != NULL && score != NULL) {
	+ location = rsc2node_new(id, rsc_lh, 0, NULL, data_set);
	+ location->node_list_rh = node_list_dup(nodes, FALSE, FALSE);
	+
	+ } else if(node != NULL && score != NULL) {
	int score_i = char2score(score);
	node_t *match = pe_find_node(data_set->nodes, node);

	- if(match) {
	- rsc2node_new(id, rsc_lh, score_i, match, data_set);
	- return TRUE;
	- } else {
	+ if(!match) {
	return FALSE;
	}
	- }
	-
	- xml_child_iter_filter(
	+ location = rsc2node_new(id, rsc_lh, score_i, match, data_set);
	+
	+ } else {
	+ xml_child_iter_filter(
	xml_obj, rule_xml, XML_TAG_RULE,
	empty = FALSE;
	crm_debug_2("Unpacking %s/%s", id, ID(rule_xml));
	generate_location_rule(rsc_lh, rule_xml, data_set);
	);

	- if(empty) {
	+ if(empty) {
	crm_config_err("Invalid location constraint %s:"
	" rsc_location must contain at least one rule",
	ID(xml_obj));
	+ }
	}
	- return TRUE;
	+
	+ if(location && role) {
	+ if(text2role(role) == RSC_ROLE_UNKNOWN) {
	+ pe_err("Invalid constraint %s: Bad role %s", id, role);
	+ return FALSE;
	+
	+ } else {
	+ location->role_filter = text2role(role);
	+ if(location->role_filter == RSC_ROLE_SLAVE) {
	+ /* Fold slave back into Started for simplicity
	+ * At the point Slave location constraints are evaluated,
	+ * all resources are still either stopped or started
	+ */
	+ location->role_filter = RSC_ROLE_STARTED;
	+ }
	+ }
	+ }
	+ return TRUE;
	}

	static int
	get_node_score(const char rule, const char score, gboolean raw, node_t *node)
	{
	int score_f = 0;
	if(score == NULL) {
	pe_err("Rule %s: no score specified. Assuming 0.", rule);

	} else if(raw) {
	score_f = char2score(score);

	} else {
	const char *attr_score = g_hash_table_lookup(
	node->details->attrs, score);
	if(attr_score == NULL) {
	crm_debug("Rule %s: node %s did not have a value for %s",
	rule, node->details->uname, score);
	score_f = -INFINITY;

	} else {
	crm_debug("Rule %s: node %s had value %s for %s",
	rule, node->details->uname, attr_score, score);
	score_f = char2score(attr_score);
	}
	}
	return score_f;
	}


	rsc_to_node_t *
	generate_location_rule(
	resource_t rsc, xmlNode rule_xml, pe_working_set_t *data_set)
	{
	const char *rule_id = NULL;
	const char *score = NULL;
	const char *boolean = NULL;
	const char *role = NULL;

	GListPtr match_L = NULL;

	int score_f = 0;
	gboolean do_and = TRUE;
	gboolean accept = TRUE;
	gboolean raw_score = TRUE;

	rsc_to_node_t *location_rule = NULL;

	rule_xml = expand_idref(rule_xml, data_set->input);
	rule_id = crm_element_value(rule_xml, XML_ATTR_ID);
	boolean = crm_element_value(rule_xml, XML_RULE_ATTR_BOOLEAN_OP);
	role = crm_element_value(rule_xml, XML_RULE_ATTR_ROLE);

	crm_debug_2("Processing rule: %s", rule_id);

	if(role != NULL && text2role(role) == RSC_ROLE_UNKNOWN) {
	pe_err("Bad role specified for %s: %s", rule_id, role);
	return NULL;
	}

	score = crm_element_value(rule_xml, XML_RULE_ATTR_SCORE);
	if(score != NULL) {
	score_f = char2score(score);

	} else {
	score = crm_element_value(
	rule_xml, XML_RULE_ATTR_SCORE_ATTRIBUTE);
	if(score == NULL) {
	score = crm_element_value(
	rule_xml, XML_RULE_ATTR_SCORE_MANGLED);
	}
	if(score != NULL) {
	raw_score = FALSE;
	}
	}
	if(safe_str_eq(boolean, "or")) {
	do_and = FALSE;
	}

	location_rule = rsc2node_new(rule_id, rsc, 0, NULL, data_set);

	if(location_rule == NULL) {
	return NULL;
	}
	if(role != NULL) {
	crm_debug_2("Setting role filter: %s", role);
	location_rule->role_filter = text2role(role);
	if(location_rule->role_filter == RSC_ROLE_SLAVE) {
	/* Fold slave back into Started for simplicity
	* At the point Slave location constraints are evaluated,
	* all resources are still either stopped or started
	*/
	location_rule->role_filter = RSC_ROLE_STARTED;
	}
	}
	if(do_and) {
	match_L = node_list_dup(data_set->nodes, TRUE, FALSE);
	slist_iter(
	node, node_t, match_L, lpc,
	node->weight = get_node_score(rule_id, score, raw_score, node);
	);
	}

	slist_iter(
	node, node_t, data_set->nodes, lpc,

	accept = test_rule(
	rule_xml, node->details->attrs, RSC_ROLE_UNKNOWN, data_set->now);

	crm_debug_2("Rule %s %s on %s", ID(rule_xml), accept?"passed":"failed", node->details->uname);

	score_f = get_node_score(rule_id, score, raw_score, node);
	/* if(accept && score_f == -INFINITY) { */
	/* accept = FALSE; */
	/* } */

	if(accept) {
	node_t *local = pe_find_node_id(
	match_L, node->details->id);
	if(local == NULL && do_and) {
	continue;

	} else if(local == NULL) {
	local = node_copy(node);
	match_L = g_list_append(match_L, local);
	}

	if(do_and == FALSE) {
	local->weight = merge_weights(
	local->weight, score_f);
	}
	crm_debug_2("node %s now has weight %d",
	node->details->uname, local->weight);

	} else if(do_and && !accept) {
	/* remove it */
	node_t *delete = pe_find_node_id(
	match_L, node->details->id);
	if(delete != NULL) {
	match_L = g_list_remove(match_L,delete);
	crm_debug_5("node %s did not match",
	node->details->uname);
	}
	crm_free(delete);
	}
	);

	location_rule->node_list_rh = match_L;
	if(location_rule->node_list_rh == NULL) {
	crm_debug_2("No matching nodes for rule %s", rule_id);
	return NULL;
	}

	crm_debug_3("%s: %d nodes matched",
	rule_id, g_list_length(location_rule->node_list_rh));
	return location_rule;
	}

	static gint sort_cons_priority_lh(gconstpointer a, gconstpointer b)
	{
	const rsc_colocation_t rsc_constraint1 = (const rsc_colocation_t)a;
	const rsc_colocation_t rsc_constraint2 = (const rsc_colocation_t)b;

	if(a == NULL) { return 1; }
	if(b == NULL) { return -1; }

	CRM_ASSERT(rsc_constraint1->rsc_lh != NULL);
	CRM_ASSERT(rsc_constraint1->rsc_rh != NULL);

	if(rsc_constraint1->rsc_lh->priority > rsc_constraint2->rsc_lh->priority) {
	return -1;
	}

	if(rsc_constraint1->rsc_lh->priority < rsc_constraint2->rsc_lh->priority) {
	return 1;
	}

	return strcmp(rsc_constraint1->rsc_lh->id, rsc_constraint2->rsc_lh->id);
	}

	static gint sort_cons_priority_rh(gconstpointer a, gconstpointer b)
	{
	const rsc_colocation_t rsc_constraint1 = (const rsc_colocation_t)a;
	const rsc_colocation_t rsc_constraint2 = (const rsc_colocation_t)b;

	if(a == NULL) { return 1; }
	if(b == NULL) { return -1; }

	CRM_ASSERT(rsc_constraint1->rsc_lh != NULL);
	CRM_ASSERT(rsc_constraint1->rsc_rh != NULL);

	if(rsc_constraint1->rsc_rh->priority > rsc_constraint2->rsc_rh->priority) {
	return -1;
	}

	if(rsc_constraint1->rsc_rh->priority < rsc_constraint2->rsc_rh->priority) {
	return 1;
	}
	return strcmp(rsc_constraint1->rsc_rh->id, rsc_constraint2->rsc_rh->id);
	}

	gboolean
	rsc_colocation_new(const char id, const char node_attr, int score,
	resource_t rsc_lh, resource_t rsc_rh,
	const char state_lh, const char state_rh,
	pe_working_set_t *data_set)
	{
	rsc_colocation_t *new_con = NULL;
	if(rsc_lh == NULL){
	crm_config_err("No resource found for LHS %s", id);
	return FALSE;

	} else if(rsc_rh == NULL){
	crm_config_err("No resource found for RHS of %s", id);
	return FALSE;
	}

	crm_malloc0(new_con, sizeof(rsc_colocation_t));
	if(new_con == NULL) {
	return FALSE;
	}

	if(state_lh == NULL
	\|\| safe_str_eq(state_lh, RSC_ROLE_STARTED_S)) {
	state_lh = RSC_ROLE_UNKNOWN_S;
	}

	if(state_rh == NULL
	\|\| safe_str_eq(state_rh, RSC_ROLE_STARTED_S)) {
	state_rh = RSC_ROLE_UNKNOWN_S;
	}

	new_con->id = id;
	new_con->rsc_lh = rsc_lh;
	new_con->rsc_rh = rsc_rh;
	new_con->score = score;
	new_con->role_lh = text2role(state_lh);
	new_con->role_rh = text2role(state_rh);
	new_con->node_attribute = node_attr;

	if(node_attr == NULL) {
	node_attr = "#"XML_ATTR_UNAME;
	}

	crm_debug_3("%s ==> %s (%s %d)", rsc_lh->id, rsc_rh->id, node_attr, score);

	rsc_lh->rsc_cons = g_list_insert_sorted(
	rsc_lh->rsc_cons, new_con, sort_cons_priority_rh);

	rsc_rh->rsc_cons_lhs = g_list_insert_sorted(
	rsc_rh->rsc_cons_lhs, new_con, sort_cons_priority_lh);

	data_set->colocation_constraints = g_list_append(
	data_set->colocation_constraints, new_con);

	return TRUE;
	}

	/* LHS before RHS */
	int
	new_rsc_order(resource_t lh_rsc, const char lh_task,
	resource_t rh_rsc, const char rh_task,
	enum pe_ordering type, pe_working_set_t *data_set)
	{
	char *lh_key = NULL;
	char *rh_key = NULL;

	CRM_CHECK(lh_rsc != NULL, return -1);
	CRM_CHECK(lh_task != NULL, return -1);
	CRM_CHECK(rh_rsc != NULL, return -1);
	CRM_CHECK(rh_task != NULL, return -1);

	lh_key = generate_op_key(lh_rsc->id, lh_task, 0);
	rh_key = generate_op_key(rh_rsc->id, rh_task, 0);

	return custom_action_order(lh_rsc, lh_key, NULL,
	rh_rsc, rh_key, NULL, type, data_set);
	}

	/* LHS before RHS */
	int
	custom_action_order(
	resource_t lh_rsc, char lh_action_task, action_t *lh_action,
	resource_t rh_rsc, char rh_action_task, action_t *rh_action,
	enum pe_ordering type, pe_working_set_t *data_set)
	{
	order_constraint_t *order = NULL;
	if(lh_rsc == NULL && lh_action) {
	lh_rsc = lh_action->rsc;
	}
	if(rh_rsc == NULL && rh_action) {
	rh_rsc = rh_action->rsc;
	}

	if((lh_action == NULL && lh_rsc == NULL)
	\|\| (rh_action == NULL && rh_rsc == NULL)){
	crm_config_err("Invalid inputs %p.%p %p.%p",
	lh_rsc, lh_action, rh_rsc, rh_action);
	crm_free(lh_action_task);
	crm_free(rh_action_task);
	return -1;
	}

	crm_malloc0(order, sizeof(order_constraint_t));

	crm_debug_3("Creating ordering constraint %d",
	data_set->order_id);

	order->id = data_set->order_id++;
	order->type = type;
	order->lh_rsc = lh_rsc;
	order->rh_rsc = rh_rsc;
	order->lh_action = lh_action;
	order->rh_action = rh_action;
	order->lh_action_task = lh_action_task;
	order->rh_action_task = rh_action_task;

	data_set->ordering_constraints = g_list_append(
	data_set->ordering_constraints, order);

	if(lh_rsc != NULL && rh_rsc != NULL) {
	crm_debug_4("Created ordering constraint %d (%s):"
	" %s/%s before %s/%s",
	order->id, ordering_type2text(order->type),
	lh_rsc->id, lh_action_task,
	rh_rsc->id, rh_action_task);

	} else if(lh_rsc != NULL) {
	crm_debug_4("Created ordering constraint %d (%s):"
	" %s/%s before action %d (%s)",
	order->id, ordering_type2text(order->type),
	lh_rsc->id, lh_action_task,
	rh_action->id, rh_action_task);

	} else if(rh_rsc != NULL) {
	crm_debug_4("Created ordering constraint %d (%s):"
	" action %d (%s) before %s/%s",
	order->id, ordering_type2text(order->type),
	lh_action->id, lh_action_task,
	rh_rsc->id, rh_action_task);

	} else {
	crm_debug_4("Created ordering constraint %d (%s):"
	" action %d (%s) before action %d (%s)",
	order->id, ordering_type2text(order->type),
	lh_action->id, lh_action_task,
	rh_action->id, rh_action_task);
	}

	return order->id;
	}

	enum pe_ordering get_flags(
	const char *id, enum pe_order_kind kind,
	const char action_first, const char action_then, gboolean invert) {
	enum pe_ordering flags = pe_order_optional;

	if(invert && kind == pe_order_kind_mandatory) {
	crm_debug_2("Upgrade %s: implies left", id);
	flags \|= pe_order_implies_left;
	if(safe_str_eq(action_then, RSC_DEMOTE)) {
	crm_debug_2("Upgrade %s: demote", id);
	flags \|= pe_order_demote;
	}

	} else if(kind == pe_order_kind_mandatory) {
	crm_debug_2("Upgrade %s: implies right", id);
	flags \|= pe_order_implies_right;
	if(safe_str_eq(action_first, RSC_START)
	\|\| safe_str_eq(action_first, RSC_PROMOTE)) {
	crm_debug_2("Upgrade %s: runnable", id);
	flags \|= pe_order_runnable_left;
	}

	} else if(kind == pe_order_kind_serialize) {
	flags \|= pe_order_serialize_only;
	}

	return flags;
	}


	static gboolean
	unpack_order_set(xmlNode set, enum pe_order_kind kind, resource_t *rsc,
	action_t begin, action_t end, action_t inv_begin, action_t inv_end,
	const char symmetrical, pe_working_set_t data_set)
	{
	GListPtr set_iter = NULL;
	GListPtr resources = NULL;

	resource_t *last = NULL;
	resource_t *resource = NULL;

	int local_kind = kind;
	gboolean sequential = FALSE;
	enum pe_ordering flags = pe_order_optional;

	char *key = NULL;
	const char *id = ID(set);
	const char *action = crm_element_value(set, "action");
	const char *sequential_s = crm_element_value(set, "sequential");
	const char *kind_s = crm_element_value(set, XML_ORDER_ATTR_KIND);

	char *pseudo_id = NULL;
	char *end_id = NULL;
	char *begin_id = NULL;

	if(action == NULL) {
	action = RSC_START;
	}

	if(kind_s) {
	local_kind = get_ordering_type(set);
	}
	if(sequential_s == NULL) {
	sequential_s = "1";
	}

	sequential = crm_is_true(sequential_s);
	flags = get_flags(id, local_kind, action, action, FALSE);

	xml_child_iter_filter(
	set, xml_rsc, XML_TAG_RESOURCE_REF,

	resource = pe_find_resource(data_set->resources, ID(xml_rsc));
	resources = g_list_append(resources, resource);
	);

	if(g_list_length(resources) == 1) {
	crm_err("Single set: %s", id);
	*rsc = resource;
	*end = NULL;
	*begin = NULL;
	*inv_end = NULL;
	*inv_begin = NULL;
	return TRUE;
	}

	pseudo_id = crm_concat(id, action, '-');
	end_id = crm_concat(pseudo_id, "end", '-');
	begin_id = crm_concat(pseudo_id, "begin", '-');

	*rsc = NULL;
	*end = get_pseudo_op(end_id, data_set);
	*begin = get_pseudo_op(begin_id, data_set);

	set_iter = resources;
	while(set_iter != NULL) {
	resource = (resource_t *) set_iter->data;
	set_iter = set_iter->next;

	key = generate_op_key(resource->id, action, 0);

	custom_action_order(NULL, NULL, *begin, resource, crm_strdup(key), NULL,
	flags\|pe_order_implies_left_printed, data_set);

	custom_action_order(resource, crm_strdup(key), NULL, NULL, NULL, *end,
	flags\|pe_order_implies_right_printed, data_set);

	if(local_kind == pe_order_kind_serialize) {
	/* Serialize before everything that comes after */
	slist_iter(
	then_rsc, resource_t, set_iter, lpc,

	char *then_key = generate_op_key(then_rsc->id, action, 0);
	custom_action_order(resource, crm_strdup(key), NULL, then_rsc, then_key, NULL,
	flags, data_set);
	);

	} else if(sequential) {
	if(last != NULL) {
	new_rsc_order(last, action, resource, action, flags, data_set);
	}
	last = resource;
	}
	}

	if(crm_is_true(symmetrical) == FALSE) {
	goto done;

	} else if(symmetrical && local_kind == pe_order_kind_serialize) {
	crm_config_warn("Cannot invert serialized constraint set %s", id);
	goto done;

	} else if(local_kind == pe_order_kind_serialize) {
	goto done;
	}


	last = NULL;
	action = invert_action(action);

	pseudo_id = crm_concat(id, action, '-');
	end_id = crm_concat(pseudo_id, "end", '-');
	begin_id = crm_concat(pseudo_id, "begin", '-');

	*inv_end = get_pseudo_op(end_id, data_set);
	*inv_begin = get_pseudo_op(begin_id, data_set);

	flags = get_flags(id, local_kind, action, action, TRUE);

	set_iter = resources;
	while(set_iter != NULL) {
	resource = (resource_t *) set_iter->data;
	set_iter = set_iter->next;

	key = generate_op_key(resource->id, action, 0);

	custom_action_order(NULL, NULL, *inv_begin, resource, crm_strdup(key), NULL,
	flags\|pe_order_implies_left_printed, data_set);

	custom_action_order(resource, crm_strdup(key), NULL, NULL, NULL, *inv_end,
	flags\|pe_order_implies_right_printed, data_set);

	if(sequential) {
	if(last != NULL) {
	new_rsc_order(resource, action, last, action, flags, data_set);
	}
	last = resource;
	}
	}

	done:
	g_list_free(resources);
	crm_free(pseudo_id);
	return TRUE;
	}

	static gboolean order_rsc_sets(
	const char id, xmlNode set1, xmlNode set2, enum pe_order_kind kind, pe_working_set_t data_set) {

	resource_t *rsc_1 = NULL;
	resource_t *rsc_2 = NULL;

	const char *action_1 = crm_element_value(set1, "action");
	const char *action_2 = crm_element_value(set2, "action");

	const char *sequential_1 = crm_element_value(set1, "sequential");
	const char *sequential_2 = crm_element_value(set2, "sequential");

	enum pe_ordering flags = get_flags(id, kind, action_1, action_2, FALSE);

	if(crm_is_true(sequential_1)) {
	/* get the first one */
	xml_child_iter_filter(
	set1, xml_rsc, XML_TAG_RESOURCE_REF,
	rsc_1 = pe_find_resource(data_set->resources, ID(xml_rsc));
	break;
	);
	}

	if(crm_is_true(sequential_2)) {
	/* get the last one */
	const char *rid = NULL;
	xml_child_iter_filter(
	set2, xml_rsc, XML_TAG_RESOURCE_REF,
	rid = ID(xml_rsc);
	);
	rsc_2 = pe_find_resource(data_set->resources, rid);
	}

	if(rsc_1 != NULL && rsc_2 != NULL) {
	new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);

	} else if(rsc_1 != NULL) {
	xml_child_iter_filter(
	set2, xml_rsc, XML_TAG_RESOURCE_REF,
	rsc_2 = pe_find_resource(data_set->resources, ID(xml_rsc));
	new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);
	);

	} else if(rsc_2 != NULL) {
	xml_child_iter_filter(
	set1, xml_rsc, XML_TAG_RESOURCE_REF,
	rsc_1 = pe_find_resource(data_set->resources, ID(xml_rsc));
	new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);
	);

	} else {
	xml_child_iter_filter(
	set1, xml_rsc, XML_TAG_RESOURCE_REF,
	rsc_1 = pe_find_resource(data_set->resources, ID(xml_rsc));

	xml_child_iter_filter(
	set2, xml_rsc_2, XML_TAG_RESOURCE_REF,
	rsc_2 = pe_find_resource(data_set->resources, ID(xml_rsc_2));
	new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);
	);
	);
	}
	return TRUE;
	}

	static char null_or_opkey(resource_t rsc, const char *action)
	{
	if(rsc == NULL) {
	return NULL;
	}
	return generate_op_key(rsc->id, action, 0);
	}

	gboolean
	unpack_rsc_order(xmlNode xml_obj, pe_working_set_t data_set)
	{
	gboolean any_sets = FALSE;

	resource_t *rsc = NULL;
	resource_t *last_rsc = NULL;

	action_t *set_end = NULL;
	action_t *set_begin = NULL;

	action_t *set_inv_end = NULL;
	action_t *set_inv_begin = NULL;

	xmlNode *last = NULL;
	action_t *last_end = NULL;
	action_t *last_begin = NULL;
	action_t *last_inv_end = NULL;
	action_t *last_inv_begin = NULL;

	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *invert = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
	enum pe_order_kind kind = get_ordering_type(xml_obj);

	if(invert == NULL) {
	invert = "true";
	}

	xml_child_iter_filter(
	xml_obj, set, XML_CONS_TAG_RSC_SET,

	any_sets = TRUE;
	set = expand_idref(set, data_set->input);
	if(unpack_order_set(set, kind, &rsc, &set_begin, &set_end,
	&set_inv_begin, &set_inv_end, invert, data_set) == FALSE) {
	return FALSE;

	} else if(last) {
	const char *set_action = crm_element_value(set, "action");
	const char *last_action = crm_element_value(last, "action");
	enum pe_ordering flags = get_flags(id, kind, last_action, set_action, FALSE);


	if(!set_action) { set_action = RSC_START; }
	if(!last_action) { last_action = RSC_START; }

	if(rsc == NULL && last_rsc == NULL) {
	order_actions(last_end, set_begin, flags);
	} else {
	custom_action_order(
	last_rsc, null_or_opkey(last_rsc, last_action), last_end,
	rsc, null_or_opkey(rsc, set_action), set_begin,
	flags, data_set);
	}

	if(crm_is_true(invert)) {
	set_action = invert_action(set_action);
	last_action = invert_action(last_action);

	flags = get_flags(id, kind, last_action, set_action, TRUE);
	if(rsc == NULL && last_rsc == NULL) {
	order_actions(last_inv_begin, set_inv_end, flags);

	} else {
	custom_action_order(
	last_rsc, null_or_opkey(last_rsc, last_action), last_inv_begin,
	rsc, null_or_opkey(rsc, set_action), set_inv_end,
	flags, data_set);
	}
	}

	} else if(/* never called */last && order_rsc_sets(id, last, set, kind, data_set) == FALSE) {
	return FALSE;

	}
	last = set;
	last_rsc = rsc;
	last_end = set_end;
	last_begin = set_begin;
	last_inv_end = set_inv_end;
	last_inv_begin = set_inv_begin;
	);

	if(any_sets == FALSE) {
	return unpack_simple_rsc_order(xml_obj, data_set);
	}

	return TRUE;
	}

	static gboolean
	unpack_colocation_set(xmlNode set, int score, pe_working_set_t data_set)
	{
	resource_t *with = NULL;
	resource_t *resource = NULL;
	const char *set_id = ID(set);
	const char *role = crm_element_value(set, "role");
	const char *sequential = crm_element_value(set, "sequential");
	int local_score = score;

	const char *score_s = crm_element_value(set, XML_RULE_ATTR_SCORE);
	if(score_s) {
	local_score = char2score(score_s);
	}

	if(sequential == NULL \|\| crm_is_true(sequential)) {
	xml_child_iter_filter(
	set, xml_rsc, XML_TAG_RESOURCE_REF,

	resource = pe_find_resource(data_set->resources, ID(xml_rsc));
	if(with != NULL) {
	crm_debug_2("Colocating %s with %s", resource->id, with->id);
	rsc_colocation_new(set_id, NULL, local_score, resource, with, role, role, data_set);
	}

	with = resource;
	);
	}

	return TRUE;
	}



	static gboolean colocate_rsc_sets(
	const char id, xmlNode set1, xmlNode set2, int score, pe_working_set_t data_set)
	{
	resource_t *rsc_1 = NULL;
	resource_t *rsc_2 = NULL;

	const char *role_1 = crm_element_value(set1, "role");
	const char *role_2 = crm_element_value(set2, "role");

	const char *sequential_1 = crm_element_value(set1, "sequential");
	const char *sequential_2 = crm_element_value(set2, "sequential");

	if(crm_is_true(sequential_1)) {
	/* get the first one */
	xml_child_iter_filter(
	set1, xml_rsc, XML_TAG_RESOURCE_REF,
	rsc_1 = pe_find_resource(data_set->resources, ID(xml_rsc));
	break;
	);
	}

	if(crm_is_true(sequential_2)) {
	/* get the last one */
	const char *rid = NULL;
	xml_child_iter_filter(
	set2, xml_rsc, XML_TAG_RESOURCE_REF,
	rid = ID(xml_rsc);
	);
	rsc_2 = pe_find_resource(data_set->resources, rid);
	}

	if(rsc_1 != NULL && rsc_2 != NULL) {
	rsc_colocation_new(id, NULL, score, rsc_1, rsc_2, role_1, role_2, data_set);

	} else if(rsc_1 != NULL) {
	xml_child_iter_filter(
	set2, xml_rsc, XML_TAG_RESOURCE_REF,
	rsc_2 = pe_find_resource(data_set->resources, ID(xml_rsc));
	rsc_colocation_new(id, NULL, score, rsc_1, rsc_2, role_1, role_2, data_set);
	);

	} else if(rsc_2 != NULL) {
	xml_child_iter_filter(
	set1, xml_rsc, XML_TAG_RESOURCE_REF,
	rsc_1 = pe_find_resource(data_set->resources, ID(xml_rsc));
	rsc_colocation_new(id, NULL, score, rsc_1, rsc_2, role_1, role_2, data_set);
	);

	} else {
	xml_child_iter_filter(
	set1, xml_rsc, XML_TAG_RESOURCE_REF,
	rsc_1 = pe_find_resource(data_set->resources, ID(xml_rsc));

	xml_child_iter_filter(
	set2, xml_rsc_2, XML_TAG_RESOURCE_REF,
	rsc_2 = pe_find_resource(data_set->resources, ID(xml_rsc_2));
	rsc_colocation_new(id, NULL, score, rsc_1, rsc_2, role_1, role_2, data_set);
	);
	);
	}

	return TRUE;
	}

	static gboolean unpack_simple_colocation(xmlNode xml_obj, pe_working_set_t data_set)
	{
	int score_i = 0;

	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);

	const char *id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
	const char *id_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
	const char *state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
	const char *state_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE);
	const char *instance_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_INSTANCE);
	const char *instance_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_INSTANCE);
	const char *attr = crm_element_value(xml_obj, XML_COLOC_ATTR_NODE_ATTR);

	const char *symmetrical = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);

	resource_t *rsc_lh = pe_find_resource(data_set->resources, id_lh);
	resource_t *rsc_rh = pe_find_resource(data_set->resources, id_rh);

	if(rsc_lh == NULL) {
	crm_config_err("Invalid constraint '%s': No resource named '%s'", id, id_lh);
	return FALSE;

	} else if(rsc_rh == NULL) {
	crm_config_err("Invalid constraint '%s': No resource named '%s'", id, id_rh);
	return FALSE;

	} else if(instance_lh && rsc_lh->variant < pe_clone) {
	crm_config_err("Invalid constraint '%s': Resource '%s' is not a clone but instance %s was requested", id, id_lh, instance_lh);
	return FALSE;

	} else if(instance_rh && rsc_rh->variant < pe_clone) {
	crm_config_err("Invalid constraint '%s': Resource '%s' is not a clone but instance %s was requested", id, id_rh, instance_rh);
	return FALSE;
	}

	if(instance_lh) {
	rsc_lh = find_clone_instance(rsc_lh, instance_lh, data_set);
	if(rsc_lh == NULL) {
	crm_config_warn("Invalid constraint '%s': No instance '%s' of '%s'", id, instance_lh, id_lh);
	return FALSE;
	}
	}

	if(instance_rh) {
	rsc_rh = find_clone_instance(rsc_rh, instance_rh, data_set);
	if(rsc_rh == NULL) {
	crm_config_warn("Invalid constraint '%s': No instance '%s' of '%s'", id, instance_rh, id_rh);
	return FALSE;
	}
	}

	if(crm_is_true(symmetrical)) {
	crm_config_warn("The %s colocation constraint attribute has been removed."
	" It didn't do what you think it did anyway.",
	XML_CONS_ATTR_SYMMETRICAL);
	}

	if(score) {
	score_i = char2score(score);
	}

	rsc_colocation_new(id, attr, score_i, rsc_lh, rsc_rh, state_lh, state_rh, data_set);
	return TRUE;
	}

	gboolean
	unpack_rsc_colocation(xmlNode xml_obj, pe_working_set_t data_set)
	{
	int score_i = 0;
	xmlNode *last = NULL;
	gboolean any_sets = FALSE;

	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);

	if(score) {
	score_i = char2score(score);
	}

	xml_child_iter_filter(
	xml_obj, set, XML_CONS_TAG_RSC_SET,

	any_sets = TRUE;
	set = expand_idref(set, data_set->input);
	if(unpack_colocation_set(set, score_i, data_set) == FALSE) {
	return FALSE;

	} else if(last && colocate_rsc_sets(id, last, set, score_i, data_set) == FALSE) {
	return FALSE;
	}
	last = set;
	);


	if(any_sets == FALSE) {
	return unpack_simple_colocation(xml_obj, data_set);
	}

	return TRUE;
	}

	gboolean is_active(rsc_to_node_t *cons)
	{
	return TRUE;
	}
	diff --git a/pengine/regression.sh b/pengine/regression.sh
	index 443ca33bc8..e826603164 100755
	--- a/pengine/regression.sh
	+++ b/pengine/regression.sh
	@@ -1,360 +1,363 @@
	#!/bin/bash

	# Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public
	# License as published by the Free Software Foundation; either
	# version 2.1 of the License, or (at your option) any later version.
	#
	# This software is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	# General Public License for more details.
	#
	# You should have received a copy of the GNU General Public
	# License along with this library; if not, write to the Free Software
	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	#

	core=`dirname $0`
	. $core/regression.core.sh
	io_dir=$test_home/test10

	create_mode="true"
	info Generating test outputs for these tests...
	# do_test file description
	info Done.
	echo ""

	info Performing the following tests from $io_dir
	create_mode="false"

	echo ""
	do_test simple1 "Offline "
	do_test simple2 "Start "
	do_test simple3 "Start 2 "
	do_test simple4 "Start Failed"
	do_test simple6 "Stop Start "
	do_test simple7 "Shutdown "
	#do_test simple8 "Stonith "
	#do_test simple9 "Lower version"
	#do_test simple10 "Higher version"
	do_test simple11 "Priority (ne)"
	do_test simple12 "Priority (eq)"
	do_test simple8 "Stickiness"

	echo ""
	do_test params-0 "Params: No change"
	do_test params-1 "Params: Changed"
	do_test params-2 "Params: Resource definition"
	do_test params-4 "Params: Reload"
	do_test novell-251689 "Resource definition change + target_role=stopped"
	do_test bug-lf-2106 "Restart all anonymous clone instances after config change"

	echo ""
	do_test orphan-0 "Orphan ignore"
	do_test orphan-1 "Orphan stop"

	echo ""
	do_test target-0 "Target Role : baseline"
	do_test target-1 "Target Role : master"
	do_test target-2 "Target Role : invalid"

	+echo ""
	+do_test domain "Failover domains"
	+
	echo ""
	do_test date-1 "Dates" -t "2005-020"
	do_test date-2 "Date Spec - Pass" -t "2005-020T12:30"
	do_test date-3 "Date Spec - Fail" -t "2005-020T11:30"
	do_test probe-0 "Probe (anon clone)"
	do_test probe-1 "Pending Probe"
	do_test probe-2 "Correctly re-probe cloned groups"
	do_test probe-3 "Probe (pending node)"
	do_test probe-4 "Probe (pending node + stopped resource)" --rc 4
	do_test standby "Standby"
	do_test comments "Comments"

	echo ""
	do_test rsc_dep1 "Must not "
	do_test rsc_dep3 "Must "
	do_test rsc_dep5 "Must not 3 "
	do_test rsc_dep7 "Must 3 "
	do_test rsc_dep10 "Must (but cant)"
	do_test rsc_dep2 "Must (running) "
	do_test rsc_dep8 "Must (running : alt) "
	do_test rsc_dep4 "Must (running + move)"
	do_test asymmetric "Asymmetric - require explicit location constraints"

	echo ""
	do_test order1 "Order start 1 "
	do_test order2 "Order start 2 "
	do_test order3 "Order stop "
	do_test order4 "Order (multiple) "
	do_test order5 "Order (move) "
	do_test order6 "Order (move w/ restart) "
	do_test order7 "Order (manditory) "
	do_test order-optional "Order (score=0) "
	do_test order-required "Order (score=INFINITY) "
	do_test bug-lf-2171 "Prevent group start when clone is stopped"
	do_test order-clone "Clone ordering should be able to prevent startup of dependant clones"
	do_test order-sets "Ordering for resource sets"
	do_test order-serialize "Serialize resources without inhibiting migration"
	do_test order-serialize-set "Serialize a set of resources without inhibiting migration"

	echo ""
	do_test coloc-loop "Colocation - loop"
	do_test coloc-many-one "Colocation - many-to-one"
	do_test coloc-list "Colocation - many-to-one with list"
	do_test coloc-group "Colocation - groups"
	do_test coloc-slave-anti "Anti-colocation with slave shouldn't prevent master colocation"
	do_test coloc-attr "Colocation based on node attributes"
	do_test coloc-negative-group "Negative colocation with a group"
	do_test coloc-intra-set "Intra-set colocation"

	#echo ""
	#do_test agent1 "version: lt (empty)"
	#do_test agent2 "version: eq "
	#do_test agent3 "version: gt "

	echo ""
	do_test attrs1 "string: eq (and) "
	do_test attrs2 "string: lt / gt (and)"
	do_test attrs3 "string: ne (or) "
	do_test attrs4 "string: exists "
	do_test attrs5 "string: not_exists "
	do_test attrs6 "is_dc: true "
	do_test attrs7 "is_dc: false "
	do_test attrs8 "score_attribute "

	echo ""
	do_test mon-rsc-1 "Schedule Monitor - start"
	do_test mon-rsc-2 "Schedule Monitor - move "
	do_test mon-rsc-3 "Schedule Monitor - pending start "
	do_test mon-rsc-4 "Schedule Monitor - move/pending start"

	echo ""
	do_test rec-rsc-0 "Resource Recover - no start "
	do_test rec-rsc-1 "Resource Recover - start "
	do_test rec-rsc-2 "Resource Recover - monitor "
	do_test rec-rsc-3 "Resource Recover - stop - ignore"
	do_test rec-rsc-4 "Resource Recover - stop - block "
	do_test rec-rsc-5 "Resource Recover - stop - fence "
	do_test rec-rsc-6 "Resource Recover - multiple - restart"
	do_test rec-rsc-7 "Resource Recover - multiple - stop "
	do_test rec-rsc-8 "Resource Recover - multiple - block "
	do_test rec-rsc-9 "Resource Recover - group/group"

	echo ""
	do_test quorum-1 "No quorum - ignore"
	do_test quorum-2 "No quorum - freeze"
	do_test quorum-3 "No quorum - stop "
	do_test quorum-4 "No quorum - start anyway"
	do_test quorum-5 "No quorum - start anyway (group)"
	do_test quorum-6 "No quorum - start anyway (clone)"

	echo ""
	do_test rec-node-1 "Node Recover - Startup - no fence"
	do_test rec-node-2 "Node Recover - Startup - fence "
	do_test rec-node-3 "Node Recover - HA down - no fence"
	do_test rec-node-4 "Node Recover - HA down - fence "
	do_test rec-node-5 "Node Recover - CRM down - no fence"
	do_test rec-node-6 "Node Recover - CRM down - fence "
	do_test rec-node-7 "Node Recover - no quorum - ignore "
	do_test rec-node-8 "Node Recover - no quorum - freeze "
	do_test rec-node-9 "Node Recover - no quorum - stop "
	do_test rec-node-10 "Node Recover - no quorum - stop w/fence"
	do_test rec-node-11 "Node Recover - CRM down w/ group - fence "
	do_test rec-node-12 "Node Recover - nothing active - fence "
	do_test rec-node-13 "Node Recover - failed resource + shutdown - fence "
	do_test rec-node-15 "Node Recover - unknown lrm section"
	do_test rec-node-14 "Serialize all stonith's"

	echo ""
	do_test multi1 "Multiple Active (stop/start)"

	echo ""
	do_test migrate-stop "Migration in a stopping stack"
	do_test migrate-start "Migration in a starting stack"
	do_test migrate-stop_start "Migration in a restarting stack"
	do_test migrate-stop-complex "Migration in a complex stopping stack"
	do_test migrate-start-complex "Migration in a complex starting stack"
	do_test migrate-stop-start-complex "Migration in a complex moving stack"

	do_test migrate-1 "Migrate (migrate)"
	do_test migrate-2 "Migrate (stable)"
	do_test migrate-3 "Migrate (failed migrate_to)"
	do_test migrate-4 "Migrate (failed migrate_from)"
	do_test novell-252693 "Migration in a stopping stack"
	do_test novell-252693-2 "Migration in a starting stack"
	do_test novell-252693-3 "Non-Migration in a starting and stopping stack"
	do_test bug-1820 "Migration in a group"
	do_test bug-1820-1 "Non-migration in a group"
	do_test migrate-5 "Primitive migration with a clone"

	#echo ""
	#do_test complex1 "Complex "

	echo ""
	do_test group1 "Group "
	do_test group2 "Group + Native "
	do_test group3 "Group + Group "
	do_test group4 "Group + Native (nothing)"
	do_test group5 "Group + Native (move) "
	do_test group6 "Group + Group (move) "
	do_test group7 "Group colocation"
	do_test group13 "Group colocation (cant run)"
	do_test group8 "Group anti-colocation"
	do_test group9 "Group recovery"
	do_test group10 "Group partial recovery"
	do_test group11 "Group target_role"
	do_test group14 "Group stop (graph terminated)"
	do_test group15 "-ve group colocation"
	do_test bug-1573 "Partial stop of a group with two children"
	do_test bug-1718 "Mandatory group ordering - Stop group_FUN"

	echo ""
	do_test clone-anon-probe-1 "Probe the correct (anonymous) clone instance for each node"
	do_test clone-anon-probe-2 "Avoid needless re-probing of anonymous clones"
	do_test clone-anon-failcount "Merge failcounts for anonymous clones"
	do_test inc0 "Incarnation start"
	do_test inc1 "Incarnation start order"
	do_test inc2 "Incarnation silent restart, stop, move"
	do_test inc3 "Inter-incarnation ordering, silent restart, stop, move"
	do_test inc4 "Inter-incarnation ordering, silent restart, stop, move (ordered)"
	do_test inc5 "Inter-incarnation ordering, silent restart, stop, move (restart 1)"
	do_test inc6 "Inter-incarnation ordering, silent restart, stop, move (restart 2)"
	do_test inc7 "Clone colocation"
	do_test inc8 "Clone anti-colocation"
	do_test inc9 "Non-unique clone"
	do_test inc10 "Non-unique clone (stop)"
	do_test inc11 "Primitive colocation with clones"
	do_test inc12 "Clone shutdown"
	do_test cloned-group "Make sure only the correct number of cloned groups are started"
	do_test clone-no-shuffle "Dont prioritize allocation of instances that must be moved"
	do_test clone-max-zero "Orphan processing with clone-max=0"
	do_test clone-anon-dup "Bug LF#2087 - Correctly parse the state of anonymous clones that are active more than once per node"
	do_test bug-lf-2160 "Dont shuffle clones due to colocation"
	do_test bug-lf-2213 "clone-node-max enforcement for cloned groups"
	do_test bug-lf-2153 "Clone ordering constraints"
	do_test bug-lf-2361 "Ensure clones observe mandatory ordering constraints if the LHS is unrunnable"
	do_test bug-lf-2317 "Avoid needless restart of primitive depending on a clone"
	do_test clone-colocate-instance-1 "Colocation with a specific clone instance (negative example)"
	do_test clone-colocate-instance-2 "Colocation with a specific clone instance"
	do_test clone-order-instance "Ordering with specific clone instances"

	echo ""
	do_test master-0 "Stopped -> Slave"
	do_test master-1 "Stopped -> Promote"
	do_test master-2 "Stopped -> Promote : notify"
	do_test master-3 "Stopped -> Promote : master location"
	do_test master-4 "Started -> Promote : master location"
	do_test master-5 "Promoted -> Promoted"
	do_test master-6 "Promoted -> Promoted (2)"
	do_test master-7 "Promoted -> Fenced"
	do_test master-8 "Promoted -> Fenced -> Moved"
	do_test master-9 "Stopped + Promotable + No quorum"
	do_test master-10 "Stopped -> Promotable : notify with monitor"
	do_test master-11 "Stopped -> Promote : colocation"
	do_test novell-239082 "Demote/Promote ordering"
	do_test novell-239087 "Stable master placement"
	do_test master-12 "Promotion based solely on rsc_location constraints"
	do_test master-13 "Include preferences of colocated resources when placing master"
	do_test master-demote "Ordering when actions depends on demoting a slave resource"
	do_test master-ordering "Prevent resources from starting that need a master"
	do_test bug-1765 "Master-Master Colocation (dont stop the slaves)"
	do_test master-group "Promotion of cloned groups"
	do_test bug-lf-1852 "Don't shuffle master/slave instances unnecessarily"
	do_test master-failed-demote "Dont retry failed demote actions"
	do_test master-failed-demote-2 "Dont retry failed demote actions (notify=false)"
	do_test master-depend "Ensure resources that depend on the master don't get allocated until the master does"
	do_test master-reattach "Re-attach to a running master"
	do_test master-allow-start "Don't include master score if it would prevent allocation"
	do_test master-colocation "Allow master instances placemaker to be influenced by colocation constraints"
	do_test master-pseudo "Make sure promote/demote pseudo actions are created correctly"
	do_test master-role "Prevent target-role from promoting more than master-max instances"
	do_test bug-lf-2358 "Master-Master anti-colocation"

	echo ""
	do_test managed-0 "Managed (reference)"
	do_test managed-1 "Not managed - down "
	do_test managed-2 "Not managed - up "

	echo ""
	do_test interleave-0 "Interleave (reference)"
	do_test interleave-1 "coloc - not interleaved"
	do_test interleave-2 "coloc - interleaved "
	do_test interleave-3 "coloc - interleaved (2)"
	do_test interleave-pseudo-stop "Interleaved clone during stonith"
	do_test interleave-stop "Interleaved clone during stop"
	do_test interleave-restart "Interleaved clone during dependancy restart"

	echo ""
	do_test notify-0 "Notify reference"
	do_test notify-1 "Notify simple"
	do_test notify-2 "Notify simple, confirm"
	do_test notify-3 "Notify move, confirm"
	do_test novell-239079 "Notification priority"
	#do_test notify-2 "Notify - 764"

	echo ""
	do_test 594 "OSDL #594"
	do_test 662 "OSDL #662"
	do_test 696 "OSDL #696"
	do_test 726 "OSDL #726"
	do_test 735 "OSDL #735"
	do_test 764 "OSDL #764"
	do_test 797 "OSDL #797"
	do_test 829 "OSDL #829"
	do_test 994 "OSDL #994"
	do_test 994-2 "OSDL #994 - with a dependant resource"
	do_test 1360 "OSDL #1360 - Clone stickiness"
	do_test 1484 "OSDL #1484 - on_fail=stop"
	do_test 1494 "OSDL #1494 - Clone stability"
	do_test unrunnable-1 "Unrunnable"
	do_test stonith-0 "Stonith loop - 1"
	do_test stonith-1 "Stonith loop - 2"
	do_test stonith-2 "Stonith loop - 3"
	do_test stonith-3 "Stonith startup"
	do_test bug-1572-1 "Recovery of groups depending on master/slave"
	do_test bug-1572-2 "Recovery of groups depending on master/slave when the master is never re-promoted"
	do_test bug-1685 "Depends-on-master ordering"
	do_test bug-1822 "Dont promote partially active groups"
	do_test bug-pm-11 "New resource added to a m/s group"
	do_test bug-pm-12 "Recover only the failed portion of a cloned group"
	do_test bug-n-387749 "Don't shuffle clone instances"
	do_test bug-n-385265 "Don't ignore the failure stickiness of group children - resource_idvscommon should stay stopped"
	do_test bug-n-385265-2 "Ensure groups are migrated instead of remaining partially active on the current node"
	do_test bug-lf-1920 "Correctly handle probes that find active resources"
	do_test bnc-515172 "Location constraint with multiple expressions"
	do_test colocate-primitive-with-clone "Optional colocation with a clone"
	do_test use-after-free-merge "Use-after-free in native_merge_weights"

	echo ""
	do_test systemhealth1 "System Health () #1"
	do_test systemhealth2 "System Health () #2"
	do_test systemhealth3 "System Health () #3"
	do_test systemhealthn1 "System Health (None) #1"
	do_test systemhealthn2 "System Health (None) #2"
	do_test systemhealthn3 "System Health (None) #3"
	do_test systemhealthm1 "System Health (Migrate On Red) #1"
	do_test systemhealthm2 "System Health (Migrate On Red) #2"
	do_test systemhealthm3 "System Health (Migrate On Red) #3"
	do_test systemhealtho1 "System Health (Only Green) #1"
	do_test systemhealtho2 "System Health (Only Green) #2"
	do_test systemhealtho3 "System Health (Only Green) #3"
	do_test systemhealthp1 "System Health (Progessive) #1"
	do_test systemhealthp2 "System Health (Progessive) #2"
	do_test systemhealthp3 "System Health (Progessive) #3"

	echo ""
	do_test utilization "Placement Strategy - utilization"
	do_test minimal "Placement Strategy - minimal"
	do_test balanced "Placement Strategy - balanced"

	echo ""
	do_test utilization-order1 "Utilization Order - Simple"
	do_test utilization-order2 "Utilization Order - Complex"
	do_test utilization-order3 "Utilization Order - Migrate"

	echo ""

	test_results
	diff --git a/pengine/test10/domain.dot b/pengine/test10/domain.dot
	new file mode 100644
	index 0000000000..b80581ccae
	--- /dev/null
	+++ b/pengine/test10/domain.dot
	@@ -0,0 +1,21 @@
	+digraph "g" {
	+"Dummy_monitor_0 puma1" -> "probe_complete puma1" [ style = bold]
	+"Dummy_monitor_0 puma1" [ style=bold color="green" fontcolor="black" ]
	+"Dummy_monitor_0 puma2" -> "probe_complete puma2" [ style = bold]
	+"Dummy_monitor_0 puma2" [ style=bold color="green" fontcolor="black" ]
	+"Dummy_monitor_0 puma3" -> "probe_complete puma3" [ style = bold]
	+"Dummy_monitor_0 puma3" [ style=bold color="green" fontcolor="black" ]
	+"Dummy_monitor_0 puma4" -> "probe_complete puma4" [ style = bold]
	+"Dummy_monitor_0 puma4" [ style=bold color="green" fontcolor="black" ]
	+"Dummy_start_0 puma2" [ style=bold color="green" fontcolor="black" ]
	+"probe_complete puma1" -> "probe_complete" [ style = bold]
	+"probe_complete puma1" [ style=bold color="green" fontcolor="black" ]
	+"probe_complete puma2" -> "probe_complete" [ style = bold]
	+"probe_complete puma2" [ style=bold color="green" fontcolor="black" ]
	+"probe_complete puma3" -> "probe_complete" [ style = bold]
	+"probe_complete puma3" [ style=bold color="green" fontcolor="black" ]
	+"probe_complete puma4" -> "probe_complete" [ style = bold]
	+"probe_complete puma4" [ style=bold color="green" fontcolor="black" ]
	+"probe_complete" -> "Dummy_start_0 puma2" [ style = bold]
	+"probe_complete" [ style=bold color="green" fontcolor="orange" ]
	+}
	diff --git a/pengine/test10/domain.exp b/pengine/test10/domain.exp
	new file mode 100644
	index 0000000000..53faad1f19
	--- /dev/null
	+++ b/pengine/test10/domain.exp
	@@ -0,0 +1,121 @@
	+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" batch-limit="30" transition_id="0">
	+ <synapse id="0">
	+ <action_set>
	+ <rsc_op id="4" operation="monitor" operation_key="Dummy_monitor_0" on_node="puma1" on_node_uuid="puma1">
	+ <primitive id="Dummy" long-id="Dummy" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.2"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="1">
	+ <action_set>
	+ <rsc_op id="6" operation="monitor" operation_key="Dummy_monitor_0" on_node="puma2" on_node_uuid="puma2">
	+ <primitive id="Dummy" long-id="Dummy" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.2"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="2">
	+ <action_set>
	+ <rsc_op id="8" operation="monitor" operation_key="Dummy_monitor_0" on_node="puma3" on_node_uuid="puma3">
	+ <primitive id="Dummy" long-id="Dummy" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.2"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="3">
	+ <action_set>
	+ <rsc_op id="10" operation="monitor" operation_key="Dummy_monitor_0" on_node="puma4" on_node_uuid="puma4">
	+ <primitive id="Dummy" long-id="Dummy" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.2"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="4">
	+ <action_set>
	+ <rsc_op id="11" operation="start" operation_key="Dummy_start_0" on_node="puma2" on_node_uuid="puma2">
	+ <primitive id="Dummy" long-id="Dummy" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.2"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="2" operation="probe_complete" operation_key="probe_complete"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="5">
	+ <action_set>
	+ <pseudo_event id="2" operation="probe_complete" operation_key="probe_complete">
	+ <attributes crm_feature_set="3.0.2"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="3" operation="probe_complete" operation_key="probe_complete" on_node="puma1" on_node_uuid="puma1"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="5" operation="probe_complete" operation_key="probe_complete" on_node="puma2" on_node_uuid="puma2"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="7" operation="probe_complete" operation_key="probe_complete" on_node="puma3" on_node_uuid="puma3"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="9" operation="probe_complete" operation_key="probe_complete" on_node="puma4" on_node_uuid="puma4"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="6" priority="1000000">
	+ <action_set>
	+ <rsc_op id="3" operation="probe_complete" operation_key="probe_complete" on_node="puma1" on_node_uuid="puma1">
	+ <attributes CRM_meta_op_no_wait="true" crm_feature_set="3.0.2"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="4" operation="monitor" operation_key="Dummy_monitor_0" on_node="puma1" on_node_uuid="puma1"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="7" priority="1000000">
	+ <action_set>
	+ <rsc_op id="5" operation="probe_complete" operation_key="probe_complete" on_node="puma2" on_node_uuid="puma2">
	+ <attributes CRM_meta_op_no_wait="true" crm_feature_set="3.0.2"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="6" operation="monitor" operation_key="Dummy_monitor_0" on_node="puma2" on_node_uuid="puma2"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="8" priority="1000000">
	+ <action_set>
	+ <rsc_op id="7" operation="probe_complete" operation_key="probe_complete" on_node="puma3" on_node_uuid="puma3">
	+ <attributes CRM_meta_op_no_wait="true" crm_feature_set="3.0.2"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="8" operation="monitor" operation_key="Dummy_monitor_0" on_node="puma3" on_node_uuid="puma3"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="9" priority="1000000">
	+ <action_set>
	+ <rsc_op id="9" operation="probe_complete" operation_key="probe_complete" on_node="puma4" on_node_uuid="puma4">
	+ <attributes CRM_meta_op_no_wait="true" crm_feature_set="3.0.2"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="10" operation="monitor" operation_key="Dummy_monitor_0" on_node="puma4" on_node_uuid="puma4"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+</transition_graph>
	+
	diff --git a/pengine/test10/domain.scores b/pengine/test10/domain.scores
	new file mode 100644
	index 0000000000..9e1b01cf8e
	--- /dev/null
	+++ b/pengine/test10/domain.scores
	@@ -0,0 +1,5 @@
	+Allocation scores:
	+native_color: Dummy allocation score on puma1: 100
	+native_color: Dummy allocation score on puma2: 2000
	+native_color: Dummy allocation score on puma3: -INFINITY
	+native_color: Dummy allocation score on puma4: 0
	diff --git a/pengine/test10/domain.xml b/pengine/test10/domain.xml
	new file mode 100644
	index 0000000000..c8b6c55622
	--- /dev/null
	+++ b/pengine/test10/domain.xml
	@@ -0,0 +1,40 @@
	+<cib crm_feature_set="2.1" admin_epoch="0" epoch="9698" num_updates="6" dc-uuid="c1b88737-8c7c-4824-81de-843fe7f2e88a" have-quorum="1" remote-tls-port="0" validate-with="pacemaker-1.1" cib-last-written="Fri Apr 30 17:14:33 2010">
	+ <configuration>
	+ <crm_config>
	+ <cluster_property_set id="cib-bootstrap-options">
	+ <nvpair id="nvpair.id21897" name="no-quorum-policy" value="ignore"/>
	+ <nvpair id="nvpair.id21906" name="stonith-enabled" value="false"/>
	+ </cluster_property_set>
	+ </crm_config>
	+ <nodes>
	+ <node id="puma1" uname="puma1" type="normal"/>
	+ <node id="puma2" uname="puma2" type="normal"/>
	+ <node id="puma3" uname="puma3" type="normal"/>
	+ <node id="puma4" uname="puma4" type="normal"/>
	+ </nodes>
	+ <domains>
	+ <domain id="dom1">
	+ <node name="puma1" score="100"/>
	+ <node name="puma2" score="2000"/>
	+ <node name="puma3" score="-INFINITY"/>
	+ </domain>
	+ </domains>
	+ <rsc_defaults>
	+ <meta_attributes id="foo">
	+ <nvpair id="nvpair.id21908" name="failure-timeout" value="10"/>
	+ </meta_attributes>
	+ </rsc_defaults>
	+ <resources>
	+ <primitive id="Dummy" class="ocf" provider="heartbeat" type="Dummy"/>
	+ </resources>
	+ <constraints>
	+ <rsc_location id="l-1" rsc="Dummy" domain="dom1"/>
	+ </constraints>
	+ </configuration>
	+ <status>
	+ <node_state id="puma1" uname="puma1" ha="active" in_ccm="true" crmd="online" join="member" expected="member" shutdown="0"/>
	+ <node_state id="puma2" uname="puma2" ha="active" in_ccm="true" crmd="online" join="member" expected="member" shutdown="0"/>
	+ <node_state id="puma3" uname="puma3" ha="active" in_ccm="true" crmd="online" join="member" expected="member" shutdown="0"/>
	+ <node_state id="puma4" uname="puma4" ha="active" in_ccm="true" crmd="online" join="member" expected="member" shutdown="0"/>
	+ </status>
	+</cib>
	diff --git a/xml/constraints-1.1.rng b/xml/constraints-1.1.rng
	index f493cf4951..44ab338959 100644
	--- a/xml/constraints-1.1.rng
	+++ b/xml/constraints-1.1.rng
	@@ -1,192 +1,202 @@
	<?xml version="1.0" encoding="utf-8"?>
	<grammar xmlns="http://relaxng.org/ns/structure/1.0"
	datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
	<start>
	<ref name="element-constraints"/>
	</start>

	<define name="element-constraints">
	<zeroOrMore>
	<choice>
	<ref name="element-location"/>
	<ref name="element-colocation"/>
	<ref name="element-order"/>
	</choice>
	</zeroOrMore>
	</define>

	<define name="element-location">
	<element name="rsc_location">
	<attribute name="id"><data type="ID"/></attribute>
	<attribute name="rsc"><data type="IDREF"/></attribute>
	<choice>
	<group>
	- <externalRef href="score.rng"/>
	- <attribute name="node"><text/></attribute>
	+ <optional>
	+ <attribute name="role">
	+ <ref name="attribute-roles"/>
	+ </attribute>
	+ </optional>
	+ <choice>
	+ <attribute name="domain"><data type="IDREF"/></attribute>
	+ <group>
	+ <attribute name="node"><text/></attribute>
	+ <externalRef href="score.rng"/>
	+ </group>
	+ </choice>
	</group>
	<oneOrMore>
	<externalRef href="rule.rng"/>
	</oneOrMore>
	</choice>
	<optional>
	<ref name="element-lifetime"/>
	</optional>
	</element>
	</define>

	<define name="element-resource-set">
	<element name="resource_set">
	<choice>
	<attribute name="id-ref"><data type="IDREF"/></attribute>
	<group>
	<attribute name="id"><data type="ID"/></attribute>
	<optional>
	<attribute name="sequential"><data type="boolean"/></attribute>
	</optional>
	<optional>
	<attribute name="action">
	<ref name="attribute-actions"/>
	</attribute>
	</optional>
	<optional>
	<attribute name="role">
	<ref name="attribute-roles"/>
	</attribute>
	</optional>
	<optional>
	<externalRef href="score.rng"/>
	</optional>
	<oneOrMore>
	<element name="resource_ref">
	<attribute name="id"><data type="IDREF"/></attribute>
	</element>
	</oneOrMore>
	</group>
	</choice>
	</element>
	</define>

	<define name="element-colocation">
	<element name="rsc_colocation">
	<attribute name="id"><data type="ID"/></attribute>
	<optional>
	<choice>
	<externalRef href="score.rng"/>
	<attribute name="score-attribute"><text/></attribute>
	<attribute name="score-attribute-mangle"><text/></attribute>
	</choice>
	</optional>
	<optional>
	<ref name="element-lifetime"/>
	</optional>
	<choice>
	<oneOrMore>
	<ref name="element-resource-set"/>
	</oneOrMore>
	<group>
	<attribute name="rsc"><data type="IDREF"/></attribute>
	<attribute name="with-rsc"><data type="IDREF"/></attribute>
	<optional>
	<attribute name="node-attribute"><text/></attribute>
	</optional>
	<optional>
	<attribute name="rsc-role">
	<ref name="attribute-roles"/>
	</attribute>
	</optional>
	<optional>
	<attribute name="with-rsc-role">
	<ref name="attribute-roles"/>
	</attribute>
	</optional>
	<optional>
	<attribute name="rsc-instance"><data type="integer"/></attribute>
	</optional>
	<optional>
	<attribute name="with-rsc-instance"><data type="integer"/></attribute>
	</optional>
	</group>
	</choice>
	</element>
	</define>

	<define name="element-order">
	<element name="rsc_order">
	<attribute name="id"><data type="ID"/></attribute>
	<optional>
	<ref name="element-lifetime"/>
	</optional>
	<optional>
	<attribute name="symmetrical"><data type="boolean"/></attribute>
	</optional>
	<optional>
	<choice>
	<externalRef href="score.rng"/>
	<attribute name="kind">
	<ref name="order-types"/>
	</attribute>
	</choice>
	</optional>
	<choice>
	<oneOrMore>
	<ref name="element-resource-set"/>
	</oneOrMore>
	<group>
	<attribute name="first"><data type="IDREF"/></attribute>
	<attribute name="then"><data type="IDREF"/></attribute>
	<optional>
	<attribute name="first-action">
	<ref name="attribute-actions"/>
	</attribute>
	</optional>
	<optional>
	<attribute name="then-action">
	<ref name="attribute-actions"/>
	</attribute>
	</optional>
	<optional>
	<attribute name="first-instance"><data type="integer"/></attribute>
	</optional>
	<optional>
	<attribute name="then-instance"><data type="integer"/></attribute>
	</optional>
	</group>
	</choice>
	</element>
	</define>

	<define name="attribute-actions">
	<choice>
	<value>start</value>
	<value>promote</value>
	<value>demote</value>
	<value>stop</value>
	</choice>
	</define>

	<define name="attribute-roles">
	<choice>
	<value>Stopped</value>
	<value>Started</value>
	<value>Master</value>
	<value>Slave</value>
	</choice>
	</define>

	<define name="order-types">
	<choice>
	<value>Optional</value>
	<value>Mandatory</value>
	<value>Serialize</value>
	</choice>
	</define>

	<define name="element-lifetime">
	<element name="lifetime">
	<oneOrMore>
	<externalRef href="rule.rng"/>
	</oneOrMore>
	</element>
	</define>

	</grammar>
	diff --git a/xml/pacemaker-1.1.rng b/xml/pacemaker-1.1.rng
	index 70a34ea1f9..5ce6ec40e7 100644
	--- a/xml/pacemaker-1.1.rng
	+++ b/xml/pacemaker-1.1.rng
	@@ -1,126 +1,141 @@
	<?xml version="1.0" encoding="utf-8"?>
	<!-- types: http://www.w3.org/TR/xmlschema-2/#dateTime -->
	<grammar xmlns="http://relaxng.org/ns/structure/1.0"
	datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
	<start>
	<element name="cib">
	<ref name="element-cib"/>
	</element>
	</start>

	<define name="element-cib">
	<ref name="attribute-options"/>
	<element name="configuration">
	<interleave>
	<element name="crm_config">
	<zeroOrMore>
	<element name="cluster_property_set">
	<externalRef href="nvset.rng"/>
	</element>
	</zeroOrMore>
	</element>
	<optional>
	<element name="rsc_defaults">
	<zeroOrMore>
	<element name="meta_attributes">
	<externalRef href="nvset.rng"/>
	</element>
	</zeroOrMore>
	</element>
	</optional>
	<optional>
	<element name="op_defaults">
	<zeroOrMore>
	<element name="meta_attributes">
	<externalRef href="nvset.rng"/>
	</element>
	</zeroOrMore>
	</element>
	</optional>
	<ref name="element-nodes"/>
	<element name="resources">
	<externalRef href="resources-1.1.rng"/>
	</element>
	+ <optional>
	+ <element name="domains">
	+ <zeroOrMore>
	+ <element name="domain">
	+ <attribute name="id"><data type="ID"/></attribute>
	+ <zeroOrMore>
	+ <element name="node">
	+ <attribute name="name"><text/></attribute>
	+ <externalRef href="score.rng"/>
	+ </element>
	+ </zeroOrMore>
	+ </element>
	+ </zeroOrMore>
	+ </element>
	+ </optional>
	<element name="constraints">
	<externalRef href="constraints-1.1.rng"/>
	</element>
	</interleave>
	</element>
	<element name="status">
	<ref name="element-status"/>
	</element>
	</define>

	<define name="attribute-options">
	<externalRef href="versions.rng"/>
	<optional>
	<attribute name="crm_feature_set"><text/></attribute>
	</optional>
	<optional>
	<attribute name="remote-tls-port"><data type="nonNegativeInteger"/></attribute>
	</optional>
	<optional>
	<attribute name="remote-clear-port"><data type="nonNegativeInteger"/></attribute>
	</optional>
	<optional>
	<attribute name="have-quorum"><data type="boolean"/></attribute>
	</optional>
	<optional>
	<attribute name="dc-uuid"><text/></attribute>
	</optional>
	<optional>
	<attribute name="cib-last-written"><text/></attribute>
	</optional>
	<optional>
	<attribute name="no-quorum-panic"><data type="boolean"/></attribute>
	</optional>
	</define>

	<define name="element-nodes">
	<element name="nodes">
	<zeroOrMore>
	<element name="node">
	<attribute name="id"><text/></attribute>
	<attribute name="uname"><text/></attribute>
	<attribute name="type">
	<choice>
	<value>normal</value>
	<value>member</value>
	<value>ping</value>
	</choice>
	</attribute>
	<optional>
	<attribute name="description"><text/></attribute>
	</optional>
	<zeroOrMore>
	<choice>
	<element name="instance_attributes">
	<externalRef href="nvset.rng"/>
	</element>
	<element name="utilization">
	<externalRef href="nvset.rng"/>
	</element>
	</choice>
	</zeroOrMore>
	</element>
	</zeroOrMore>
	</element>
	</define>

	<define name="element-status">
	<zeroOrMore>
	<choice>
	<attribute>
	<anyName/>
	<text/>
	</attribute>
	<element>
	<anyName/>
	<ref name="element-status"/>
	</element>
	<text/>
	</choice>
	</zeroOrMore>
	</define>

	</grammar>

File Metadata

Mime Type: text/x-diff
Expires: Sat, Jan 25, 6:37 AM (1 d, 7 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1321535
Default Alt Text: (179 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions