No OneTemporary
Actions

Size

194 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/pengine/regression.sh b/pengine/regression.sh
	index 05ab20b85f..82418d6b31 100755
	--- a/pengine/regression.sh
	+++ b/pengine/regression.sh
	@@ -1,712 +1,719 @@
	#!/bin/bash

	# Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public
	# License as published by the Free Software Foundation; either
	# version 2 of the License, or (at your option) any later version.
	#
	# This software is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	# General Public License for more details.
	#
	# You should have received a copy of the GNU General Public
	# License along with this library; if not, write to the Free Software
	# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	#

	core=`dirname $0`
	. $core/regression.core.sh \|\| exit 1

	create_mode="true"
	info Generating test outputs for these tests...
	# do_test file description

	info Done.
	echo ""

	info Performing the following tests from $io_dir
	create_mode="false"

	echo ""
	do_test simple1 "Offline "
	do_test simple2 "Start "
	do_test simple3 "Start 2 "
	do_test simple4 "Start Failed"
	do_test simple6 "Stop Start "
	do_test simple7 "Shutdown "
	#do_test simple8 "Stonith "
	#do_test simple9 "Lower version"
	#do_test simple10 "Higher version"
	do_test simple11 "Priority (ne)"
	do_test simple12 "Priority (eq)"
	do_test simple8 "Stickiness"

	echo ""
	do_test group1 "Group "
	do_test group2 "Group + Native "
	do_test group3 "Group + Group "
	do_test group4 "Group + Native (nothing)"
	do_test group5 "Group + Native (move) "
	do_test group6 "Group + Group (move) "
	do_test group7 "Group colocation"
	do_test group13 "Group colocation (cant run)"
	do_test group8 "Group anti-colocation"
	do_test group9 "Group recovery"
	do_test group10 "Group partial recovery"
	do_test group11 "Group target_role"
	do_test group14 "Group stop (graph terminated)"
	do_test group15 "-ve group colocation"
	do_test bug-1573 "Partial stop of a group with two children"
	do_test bug-1718 "Mandatory group ordering - Stop group_FUN"
	do_test bug-lf-2613 "Move group on failure"
	do_test bug-lf-2619 "Move group on clone failure"
	do_test group-fail "Ensure stop order is preserved for partially active groups"
	do_test group-unmanaged "No need to restart r115 because r114 is unmanaged"
	do_test group-unmanaged-stopped "Make sure r115 is stopped when r114 fails"
	do_test group-dependants "Account for the location preferences of things colocated with a group"

	echo ""
	do_test rsc_dep1 "Must not "
	do_test rsc_dep3 "Must "
	do_test rsc_dep5 "Must not 3 "
	do_test rsc_dep7 "Must 3 "
	do_test rsc_dep10 "Must (but cant)"
	do_test rsc_dep2 "Must (running) "
	do_test rsc_dep8 "Must (running : alt) "
	do_test rsc_dep4 "Must (running + move)"
	do_test asymmetric "Asymmetric - require explicit location constraints"

	echo ""
	do_test orphan-0 "Orphan ignore"
	do_test orphan-1 "Orphan stop"
	do_test orphan-2 "Orphan stop, remove failcount"

	echo ""
	do_test params-0 "Params: No change"
	do_test params-1 "Params: Changed"
	do_test params-2 "Params: Resource definition"
	do_test params-4 "Params: Reload"
	do_test params-5 "Params: Restart based on probe digest"
	do_test novell-251689 "Resource definition change + target_role=stopped"
	do_test bug-lf-2106 "Restart all anonymous clone instances after config change"
	do_test params-6 "Params: Detect reload in previously migrated resource"

	echo ""
	do_test target-0 "Target Role : baseline"
	do_test target-1 "Target Role : master"
	do_test target-2 "Target Role : invalid"

	echo ""
	do_test domain "Failover domains"
	do_test base-score "Set a node's default score for all nodes"

	echo ""
	do_test date-1 "Dates" -t "2005-020"
	do_test date-2 "Date Spec - Pass" -t "2005-020T12:30"
	do_test date-3 "Date Spec - Fail" -t "2005-020T11:30"
	do_test probe-0 "Probe (anon clone)"
	do_test probe-1 "Pending Probe"
	do_test probe-2 "Correctly re-probe cloned groups"
	do_test probe-3 "Probe (pending node)"
	do_test probe-4 "Probe (pending node + stopped resource)" --rc 4
	do_test standby "Standby"
	do_test comments "Comments"

	echo ""
	do_test one-or-more-0 "Everything starts"
	do_test one-or-more-1 "Nothing starts because of A"
	do_test one-or-more-2 "D can start because of C"
	do_test one-or-more-3 "D cannot start because of B and C"
	do_test one-or-more-4 "D cannot start because of target-role"
	do_test one-or-more-5 "Start A and F even though C and D are stopped"
	do_test one-or-more-6 "Leave A running even though B is stopped"
	do_test one-or-more-7 "Leave A running even though C is stopped"
	do_test bug-5140-require-all-false "Allow basegrp:0 to stop"

	echo ""
	do_test order1 "Order start 1 "
	do_test order2 "Order start 2 "
	do_test order3 "Order stop "
	do_test order4 "Order (multiple) "
	do_test order5 "Order (move) "
	do_test order6 "Order (move w/ restart) "
	do_test order7 "Order (manditory) "
	do_test order-optional "Order (score=0) "
	do_test order-required "Order (score=INFINITY) "
	do_test bug-lf-2171 "Prevent group start when clone is stopped"
	do_test order-clone "Clone ordering should be able to prevent startup of dependant clones"
	do_test order-sets "Ordering for resource sets"
	do_test order-serialize "Serialize resources without inhibiting migration"
	do_test order-serialize-set "Serialize a set of resources without inhibiting migration"
	do_test clone-order-primitive "Order clone start after a primitive"
	do_test order-optional-keyword "Order (optional keyword)"
	do_test order-mandatory "Order (mandatory keyword)"
	do_test bug-lf-2493 "Don't imply colocation requirements when applying ordering constraints with clones"
	do_test ordered-set-basic-startup "Constraint set with default order settings."
	do_test order-wrong-kind "Order (error)"

	echo ""
	do_test coloc-loop "Colocation - loop"
	do_test coloc-many-one "Colocation - many-to-one"
	do_test coloc-list "Colocation - many-to-one with list"
	do_test coloc-group "Colocation - groups"
	do_test coloc-slave-anti "Anti-colocation with slave shouldn't prevent master colocation"
	do_test coloc-attr "Colocation based on node attributes"
	do_test coloc-negative-group "Negative colocation with a group"
	do_test coloc-intra-set "Intra-set colocation"
	do_test bug-lf-2435 "Colocation sets with a negative score"
	do_test coloc-clone-stays-active "Ensure clones don't get stopped/demoted because a dependant must stop"
	do_test coloc_fp_logic "Verify floating point calculations in colocation are working"
	do_test colo_master_w_native "cl#5070 - Verify promotion order is affected when colocating master to native rsc."
	do_test colo_slave_w_native "cl#5070 - Verify promotion order is affected when colocating slave to native rsc."

	echo ""
	do_test rsc-sets-seq-true "Resource Sets - sequential=false"
	do_test rsc-sets-seq-false "Resource Sets - sequential=true"
	do_test rsc-sets-clone "Resource Sets - Clone"
	do_test rsc-sets-master "Resource Sets - Master"
	do_test rsc-sets-clone-1 "Resource Sets - Clone (lf#2404)"

	#echo ""
	#do_test agent1 "version: lt (empty)"
	#do_test agent2 "version: eq "
	#do_test agent3 "version: gt "

	echo ""
	do_test attrs1 "string: eq (and) "
	do_test attrs2 "string: lt / gt (and)"
	do_test attrs3 "string: ne (or) "
	do_test attrs4 "string: exists "
	do_test attrs5 "string: not_exists "
	do_test attrs6 "is_dc: true "
	do_test attrs7 "is_dc: false "
	do_test attrs8 "score_attribute "
	do_test per-node-attrs "Per node resource parameters"

	echo ""
	do_test mon-rsc-1 "Schedule Monitor - start"
	do_test mon-rsc-2 "Schedule Monitor - move "
	do_test mon-rsc-3 "Schedule Monitor - pending start "
	do_test mon-rsc-4 "Schedule Monitor - move/pending start"

	echo ""
	do_test rec-rsc-0 "Resource Recover - no start "
	do_test rec-rsc-1 "Resource Recover - start "
	do_test rec-rsc-2 "Resource Recover - monitor "
	do_test rec-rsc-3 "Resource Recover - stop - ignore"
	do_test rec-rsc-4 "Resource Recover - stop - block "
	do_test rec-rsc-5 "Resource Recover - stop - fence "
	do_test rec-rsc-6 "Resource Recover - multiple - restart"
	do_test rec-rsc-7 "Resource Recover - multiple - stop "
	do_test rec-rsc-8 "Resource Recover - multiple - block "
	do_test rec-rsc-9 "Resource Recover - group/group"
	do_test monitor-recovery "on-fail=block + resource recovery detected by recurring monitor"
	do_test stop-failure-no-quorum "Stop failure without quorum"
	do_test stop-failure-no-fencing "Stop failure without fencing available"
	do_test stop-failure-with-fencing "Stop failure with fencing available"

	echo ""
	do_test quorum-1 "No quorum - ignore"
	do_test quorum-2 "No quorum - freeze"
	do_test quorum-3 "No quorum - stop "
	do_test quorum-4 "No quorum - start anyway"
	do_test quorum-5 "No quorum - start anyway (group)"
	do_test quorum-6 "No quorum - start anyway (clone)"

	echo ""
	do_test rec-node-1 "Node Recover - Startup - no fence"
	do_test rec-node-2 "Node Recover - Startup - fence "
	do_test rec-node-3 "Node Recover - HA down - no fence"
	do_test rec-node-4 "Node Recover - HA down - fence "
	do_test rec-node-5 "Node Recover - CRM down - no fence"
	do_test rec-node-6 "Node Recover - CRM down - fence "
	do_test rec-node-7 "Node Recover - no quorum - ignore "
	do_test rec-node-8 "Node Recover - no quorum - freeze "
	do_test rec-node-9 "Node Recover - no quorum - stop "
	do_test rec-node-10 "Node Recover - no quorum - stop w/fence"
	do_test rec-node-11 "Node Recover - CRM down w/ group - fence "
	do_test rec-node-12 "Node Recover - nothing active - fence "
	do_test rec-node-13 "Node Recover - failed resource + shutdown - fence "
	do_test rec-node-15 "Node Recover - unknown lrm section"
	do_test rec-node-14 "Serialize all stonith's"

	echo ""
	do_test multi1 "Multiple Active (stop/start)"

	echo ""
	do_test migrate-begin "Normal migration"
	do_test migrate-success "Completed migration"
	do_test migrate-partial-1 "Completed migration, missing stop on source"
	do_test migrate-partial-2 "Successful migrate_to only"
	do_test migrate-partial-3 "Successful migrate_to only, target down"
	do_test migrate-partial-4 "Migrate from the correct host after migrate_to+migrate_from"

	do_test migrate-fail-2 "Failed migrate_from"
	do_test migrate-fail-3 "Failed migrate_from + stop on source"
	do_test migrate-fail-4 "Failed migrate_from + stop on target - ideally we wouldn't need to re-stop on target"
	do_test migrate-fail-5 "Failed migrate_from + stop on source and target"

	do_test migrate-fail-6 "Failed migrate_to"
	do_test migrate-fail-7 "Failed migrate_to + stop on source"
	do_test migrate-fail-8 "Failed migrate_to + stop on target - ideally we wouldn't need to re-stop on target"
	do_test migrate-fail-9 "Failed migrate_to + stop on source and target"

	do_test migrate-stop "Migration in a stopping stack"
	do_test migrate-start "Migration in a starting stack"
	do_test migrate-stop_start "Migration in a restarting stack"
	do_test migrate-stop-complex "Migration in a complex stopping stack"
	do_test migrate-start-complex "Migration in a complex starting stack"
	do_test migrate-stop-start-complex "Migration in a complex moving stack"
	do_test migrate-shutdown "Order the post-migration 'stop' before node shutdown"

	do_test migrate-1 "Migrate (migrate)"
	do_test migrate-2 "Migrate (stable)"
	do_test migrate-3 "Migrate (failed migrate_to)"
	do_test migrate-4 "Migrate (failed migrate_from)"
	do_test novell-252693 "Migration in a stopping stack"
	do_test novell-252693-2 "Migration in a starting stack"
	do_test novell-252693-3 "Non-Migration in a starting and stopping stack"
	do_test bug-1820 "Migration in a group"
	do_test bug-1820-1 "Non-migration in a group"
	do_test migrate-5 "Primitive migration with a clone"
	do_test migrate-fencing "Migration after Fencing"
	do_test migrate-both-vms "Migrate two VMs that have no colocation"

	#echo ""
	#do_test complex1 "Complex "

	do_test bug-lf-2422 "Dependancy on partially active group - stop ocfs:*"

	echo ""
	do_test clone-anon-probe-1 "Probe the correct (anonymous) clone instance for each node"
	do_test clone-anon-probe-2 "Avoid needless re-probing of anonymous clones"
	do_test clone-anon-failcount "Merge failcounts for anonymous clones"
	do_test inc0 "Incarnation start"
	do_test inc1 "Incarnation start order"
	do_test inc2 "Incarnation silent restart, stop, move"
	do_test inc3 "Inter-incarnation ordering, silent restart, stop, move"
	do_test inc4 "Inter-incarnation ordering, silent restart, stop, move (ordered)"
	do_test inc5 "Inter-incarnation ordering, silent restart, stop, move (restart 1)"
	do_test inc6 "Inter-incarnation ordering, silent restart, stop, move (restart 2)"
	do_test inc7 "Clone colocation"
	do_test inc8 "Clone anti-colocation"
	do_test inc9 "Non-unique clone"
	do_test inc10 "Non-unique clone (stop)"
	do_test inc11 "Primitive colocation with clones"
	do_test inc12 "Clone shutdown"
	do_test cloned-group "Make sure only the correct number of cloned groups are started"
	do_test clone-no-shuffle "Dont prioritize allocation of instances that must be moved"
	do_test clone-max-zero "Orphan processing with clone-max=0"
	do_test clone-anon-dup "Bug LF#2087 - Correctly parse the state of anonymous clones that are active more than once per node"
	do_test bug-lf-2160 "Dont shuffle clones due to colocation"
	do_test bug-lf-2213 "clone-node-max enforcement for cloned groups"
	do_test bug-lf-2153 "Clone ordering constraints"
	do_test bug-lf-2361 "Ensure clones observe mandatory ordering constraints if the LHS is unrunnable"
	do_test bug-lf-2317 "Avoid needless restart of primitive depending on a clone"
	do_test clone-colocate-instance-1 "Colocation with a specific clone instance (negative example)"
	do_test clone-colocate-instance-2 "Colocation with a specific clone instance"
	do_test clone-order-instance "Ordering with specific clone instances"
	do_test bug-lf-2453 "Enforce mandatory clone ordering without colocation"
	do_test bug-lf-2508 "Correctly reconstruct the status of anonymous cloned groups"
	do_test bug-lf-2544 "Balanced clone placement"
	do_test bug-lf-2445 "Redistribute clones with node-max > 1 and stickiness = 0"
	do_test bug-lf-2574 "Avoid clone shuffle"
	do_test bug-lf-2581 "Avoid group restart due to unrelated clone (re)start"
	do_test bug-cl-5168 "Don't shuffle clones"
	do_test bug-cl-5170 "Prevent clone from starting with on-fail=block"
	do_test clone-fail-block-colocation "Move colocated group when failed clone has on-fail=block"

	echo ""
	do_test master-0 "Stopped -> Slave"
	do_test master-1 "Stopped -> Promote"
	do_test master-2 "Stopped -> Promote : notify"
	do_test master-3 "Stopped -> Promote : master location"
	do_test master-4 "Started -> Promote : master location"
	do_test master-5 "Promoted -> Promoted"
	do_test master-6 "Promoted -> Promoted (2)"
	do_test master-7 "Promoted -> Fenced"
	do_test master-8 "Promoted -> Fenced -> Moved"
	do_test master-9 "Stopped + Promotable + No quorum"
	do_test master-10 "Stopped -> Promotable : notify with monitor"
	do_test master-11 "Stopped -> Promote : colocation"
	do_test novell-239082 "Demote/Promote ordering"
	do_test novell-239087 "Stable master placement"
	do_test master-12 "Promotion based solely on rsc_location constraints"
	do_test master-13 "Include preferences of colocated resources when placing master"
	do_test master-demote "Ordering when actions depends on demoting a slave resource"
	do_test master-ordering "Prevent resources from starting that need a master"
	do_test bug-1765 "Master-Master Colocation (dont stop the slaves)"
	do_test master-group "Promotion of cloned groups"
	do_test bug-lf-1852 "Don't shuffle master/slave instances unnecessarily"
	do_test master-failed-demote "Dont retry failed demote actions"
	do_test master-failed-demote-2 "Dont retry failed demote actions (notify=false)"
	do_test master-depend "Ensure resources that depend on the master don't get allocated until the master does"
	do_test master-reattach "Re-attach to a running master"
	do_test master-allow-start "Don't include master score if it would prevent allocation"
	do_test master-colocation "Allow master instances placemaker to be influenced by colocation constraints"
	do_test master-pseudo "Make sure promote/demote pseudo actions are created correctly"
	do_test master-role "Prevent target-role from promoting more than master-max instances"
	do_test bug-lf-2358 "Master-Master anti-colocation"
	do_test master-promotion-constraint "Mandatory master colocation constraints"
	do_test unmanaged-master "Ensure role is preserved for unmanaged resources"
	do_test master-unmanaged-monitor "Start the correct monitor operation for unmanaged masters"
	do_test master-demote-2 "Demote does not clear past failure"
	do_test master-move "Move master based on failure of colocated group"
	do_test master-probed-score "Observe the promotion score of probed resources"
	do_test colocation_constraint_stops_master "cl#5054 - Ensure master is demoted when stopped by colocation constraint"
	do_test colocation_constraint_stops_slave "cl#5054 - Ensure slave is not demoted when stopped by colocation constraint"
	do_test order_constraint_stops_master "cl#5054 - Ensure master is demoted when stopped by order constraint"
	do_test order_constraint_stops_slave "cl#5054 - Ensure slave is not demoted when stopped by order constraint"
	do_test master_monitor_restart "cl#5072 - Ensure master monitor operation will start after promotion."
	do_test bug-rh-880249 "Handle replacement of an m/s resource with a primitive"
	do_test bug-5143-ms-shuffle "Prevent master shuffling due to promotion score"
	do_test master-demote-block "Block promotion if demote fails with on-fail=block"
	do_test master-dependant-ban "Don't stop instances from being active because a dependant is banned from that host"

	echo ""
	do_test history-1 "Correctly parse stateful-1 resource state"

	echo ""
	do_test managed-0 "Managed (reference)"
	do_test managed-1 "Not managed - down "
	do_test managed-2 "Not managed - up "
	do_test bug-5028 "Shutdown should block if anything depends on an unmanaged resource"
	do_test bug-5028-detach "Ensure detach still works"
	do_test bug-5028-bottom "Ensure shutdown still blocks if the blocked resource is at the bottom of the stack"
	do_test unmanaged-stop-1 "cl#5155 - Block the stop of resources if any depending resource is unmanaged "
	do_test unmanaged-stop-2 "cl#5155 - Block the stop of resources if the first resource in a mandatory stop order is unmanaged "
	do_test unmanaged-stop-3 "cl#5155 - Block the stop of resources if any depending resource in a group is unmanaged "
	do_test unmanaged-stop-4 "cl#5155 - Block the stop of resources if any depending resource in the middle of a group is unmanaged "

	echo ""
	do_test interleave-0 "Interleave (reference)"
	do_test interleave-1 "coloc - not interleaved"
	do_test interleave-2 "coloc - interleaved "
	do_test interleave-3 "coloc - interleaved (2)"
	do_test interleave-pseudo-stop "Interleaved clone during stonith"
	do_test interleave-stop "Interleaved clone during stop"
	do_test interleave-restart "Interleaved clone during dependancy restart"

	echo ""
	do_test notify-0 "Notify reference"
	do_test notify-1 "Notify simple"
	do_test notify-2 "Notify simple, confirm"
	do_test notify-3 "Notify move, confirm"
	do_test novell-239079 "Notification priority"
	#do_test notify-2 "Notify - 764"

	echo ""
	do_test 594 "OSDL #594 - Unrunnable actions scheduled in transition"
	do_test 662 "OSDL #662 - Two resources start on one node when incarnation_node_max = 1"
	do_test 696 "OSDL #696 - CRM starts stonith RA without monitor"
	do_test 726 "OSDL #726 - Attempting to schedule rsc_posic041_monitor_5000 _after_ a stop"
	do_test 735 "OSDL #735 - Correctly detect that rsc_hadev1 is stopped on hadev3"
	do_test 764 "OSDL #764 - Missing monitor op for DoFencing:child_DoFencing:1"
	do_test 797 "OSDL #797 - Assert triggered: task_id_i > max_call_id"
	do_test 829 "OSDL #829"
	do_test 994 "OSDL #994 - Stopping the last resource in a resource group causes the entire group to be restarted"
	do_test 994-2 "OSDL #994 - with a dependant resource"
	do_test 1360 "OSDL #1360 - Clone stickiness"
	do_test 1484 "OSDL #1484 - on_fail=stop"
	do_test 1494 "OSDL #1494 - Clone stability"
	do_test unrunnable-1 "Unrunnable"
	do_test stonith-0 "Stonith loop - 1"
	do_test stonith-1 "Stonith loop - 2"
	do_test stonith-2 "Stonith loop - 3"
	do_test stonith-3 "Stonith startup"
	do_test stonith-4 "Stonith node state" --rc 4
	do_test bug-1572-1 "Recovery of groups depending on master/slave"
	do_test bug-1572-2 "Recovery of groups depending on master/slave when the master is never re-promoted"
	do_test bug-1685 "Depends-on-master ordering"
	do_test bug-1822 "Dont promote partially active groups"
	do_test bug-pm-11 "New resource added to a m/s group"
	do_test bug-pm-12 "Recover only the failed portion of a cloned group"
	do_test bug-n-387749 "Don't shuffle clone instances"
	do_test bug-n-385265 "Don't ignore the failure stickiness of group children - resource_idvscommon should stay stopped"
	do_test bug-n-385265-2 "Ensure groups are migrated instead of remaining partially active on the current node"
	do_test bug-lf-1920 "Correctly handle probes that find active resources"
	do_test bnc-515172 "Location constraint with multiple expressions"
	do_test colocate-primitive-with-clone "Optional colocation with a clone"
	do_test use-after-free-merge "Use-after-free in native_merge_weights"
	do_test bug-lf-2551 "STONITH ordering for stop"
	do_test bug-lf-2606 "Stonith implies demote"
	do_test bug-lf-2474 "Ensure resource op timeout takes precedence over op_defaults"
	do_test bug-suse-707150 "Prevent vm-01 from starting due to colocation/ordering"
	do_test bug-5014-A-start-B-start "Verify when A starts B starts using symmetrical=false"
	do_test bug-5014-A-stop-B-started "Verify when A stops B does not stop if it has already started using symmetric=false"
	do_test bug-5014-A-stopped-B-stopped "Verify when A is stopped and B has not started, B does not start before A using symmetric=false"
	do_test bug-5014-CthenAthenB-C-stopped "Verify when C then A is symmetrical=true, A then B is symmetric=false, and C is stopped that nothing starts."
	do_test bug-5014-CLONE-A-start-B-start "Verify when A starts B starts using clone resources with symmetric=false"
	do_test bug-5014-CLONE-A-stop-B-started "Verify when A stops B does not stop if it has already started using clone resources with symmetric=false."
	do_test bug-5014-GROUP-A-start-B-start "Verify when A starts B starts when using group resources with symmetric=false."
	do_test bug-5014-GROUP-A-stopped-B-started "Verify when A stops B does not stop if it has already started using group resources with symmetric=false."
	do_test bug-5014-GROUP-A-stopped-B-stopped "Verify when A is stopped and B has not started, B does not start before A using group resources with symmetric=false."
	do_test bug-5014-ordered-set-symmetrical-false "Verify ordered sets work with symmetrical=false"
	do_test bug-5014-ordered-set-symmetrical-true "Verify ordered sets work with symmetrical=true"
	do_test bug-5007-masterslave_colocation "Verify use of colocation scores other than INFINITY and -INFINITY work on multi-state resources."
	do_test bug-5038 "Prevent restart of anonymous clones when clone-max decreases"
	do_test bug-5025-1 "Automatically clean up failcount after resource config change with reload"
	do_test bug-5025-2 "Make sure clear failcount action isn't set when config does not change."
	do_test bug-5025-3 "Automatically clean up failcount after resource config change with restart"
	do_test bug-5025-4 "Clear failcount when last failure is a start op and rsc attributes changed."
	do_test failcount "Ensure failcounts are correctly expired"
	do_test failcount-block "Ensure failcounts are not expired when on-fail=block is present"
	do_test monitor-onfail-restart "bug-5058 - Monitor failure with on-fail set to restart"
	do_test monitor-onfail-stop "bug-5058 - Monitor failure wiht on-fail set to stop"
	do_test bug-5059 "No need to restart p_stateful1:*"
	do_test bug-5069-op-enabled "Test on-fail=ignore with failure when monitor is enabled."
	do_test bug-5069-op-disabled "Test on-fail-ignore with failure when monitor is disabled."
	do_test obsolete-lrm-resource "cl#5115 - Do not use obsolete lrm_resource sections"

	do_test ignore_stonith_rsc_order1 "cl#5056- Ignore order constraint between stonith and non-stonith rsc."
	do_test ignore_stonith_rsc_order2 "cl#5056- Ignore order constraint with group rsc containing mixed stonith and non-stonith."
	do_test ignore_stonith_rsc_order3 "cl#5056- Ignore order constraint, stonith clone and mixed group"
	do_test ignore_stonith_rsc_order4 "cl#5056- Ignore order constraint, stonith clone and clone with nested mixed group"
	do_test honor_stonith_rsc_order1 "cl#5056- Honor order constraint, stonith clone and pure stonith group(single rsc)."
	do_test honor_stonith_rsc_order2 "cl#5056- Honor order constraint, stonith clone and pure stonith group(multiple rsc)"
	do_test honor_stonith_rsc_order3 "cl#5056- Honor order constraint, stonith clones with nested pure stonith group."
	do_test honor_stonith_rsc_order4 "cl#5056- Honor order constraint, between two native stonith rscs."
	do_test probe-timeout "cl#5099 - Default probe timeout"

	echo ""
	do_test systemhealth1 "System Health () #1"
	do_test systemhealth2 "System Health () #2"
	do_test systemhealth3 "System Health () #3"
	do_test systemhealthn1 "System Health (None) #1"
	do_test systemhealthn2 "System Health (None) #2"
	do_test systemhealthn3 "System Health (None) #3"
	do_test systemhealthm1 "System Health (Migrate On Red) #1"
	do_test systemhealthm2 "System Health (Migrate On Red) #2"
	do_test systemhealthm3 "System Health (Migrate On Red) #3"
	do_test systemhealtho1 "System Health (Only Green) #1"
	do_test systemhealtho2 "System Health (Only Green) #2"
	do_test systemhealtho3 "System Health (Only Green) #3"
	do_test systemhealthp1 "System Health (Progessive) #1"
	do_test systemhealthp2 "System Health (Progessive) #2"
	do_test systemhealthp3 "System Health (Progessive) #3"

	echo ""
	do_test utilization "Placement Strategy - utilization"
	do_test minimal "Placement Strategy - minimal"
	do_test balanced "Placement Strategy - balanced"

	echo ""
	do_test placement-stickiness "Optimized Placement Strategy - stickiness"
	do_test placement-priority "Optimized Placement Strategy - priority"
	do_test placement-location "Optimized Placement Strategy - location"
	do_test placement-capacity "Optimized Placement Strategy - capacity"

	echo ""
	do_test utilization-order1 "Utilization Order - Simple"
	do_test utilization-order2 "Utilization Order - Complex"
	do_test utilization-order3 "Utilization Order - Migrate"
	do_test utilization-order4 "Utilization Order - Live Mirgration (bnc#695440)"
	do_test utilization-shuffle "Don't displace prmExPostgreSQLDB2 on act2, Start prmExPostgreSQLDB1 on act3"
	do_test load-stopped-loop "Avoid transition loop due to load_stopped (cl#5044)"

	echo ""
	do_test reprobe-target_rc "Ensure correct target_rc for reprobe of inactive resources"
	do_test node-maintenance-1 "cl#5128 - Node maintenance"
	do_test node-maintenance-2 "cl#5128 - Node maintenance (coming out of maintenance mode)"

	echo ""
	do_test not-installed-agent "The resource agent is missing"
	do_test not-installed-tools "Something the resource agent needs is missing"

	echo ""
	do_test stopped-monitor-00 "Stopped Monitor - initial start"
	do_test stopped-monitor-01 "Stopped Monitor - failed started"
	do_test stopped-monitor-02 "Stopped Monitor - started multi-up"
	do_test stopped-monitor-03 "Stopped Monitor - stop started"
	do_test stopped-monitor-04 "Stopped Monitor - failed stop"
	do_test stopped-monitor-05 "Stopped Monitor - start unmanaged"
	do_test stopped-monitor-06 "Stopped Monitor - unmanaged multi-up"
	do_test stopped-monitor-07 "Stopped Monitor - start unmanaged multi-up"
	do_test stopped-monitor-08 "Stopped Monitor - migrate"
	do_test stopped-monitor-09 "Stopped Monitor - unmanage started"
	do_test stopped-monitor-10 "Stopped Monitor - unmanaged started multi-up"
	do_test stopped-monitor-11 "Stopped Monitor - stop unmanaged started"
	do_test stopped-monitor-12 "Stopped Monitor - unmanaged started multi-up (targer-role="Stopped")"
	do_test stopped-monitor-20 "Stopped Monitor - initial stop"
	do_test stopped-monitor-21 "Stopped Monitor - stopped single-up"
	do_test stopped-monitor-22 "Stopped Monitor - stopped multi-up"
	do_test stopped-monitor-23 "Stopped Monitor - start stopped"
	do_test stopped-monitor-24 "Stopped Monitor - unmanage stopped"
	do_test stopped-monitor-25 "Stopped Monitor - unmanaged stopped multi-up"
	do_test stopped-monitor-26 "Stopped Monitor - start unmanaged stopped"
	do_test stopped-monitor-27 "Stopped Monitor - unmanaged stopped multi-up (target-role="Started")"
	do_test stopped-monitor-30 "Stopped Monitor - new node started"
	do_test stopped-monitor-31 "Stopped Monitor - new node stopped"

	echo""
	do_test ticket-primitive-1 "Ticket - Primitive (loss-policy=stop, initial)"
	do_test ticket-primitive-2 "Ticket - Primitive (loss-policy=stop, granted)"
	do_test ticket-primitive-3 "Ticket - Primitive (loss-policy-stop, revoked)"
	do_test ticket-primitive-4 "Ticket - Primitive (loss-policy=demote, initial)"
	do_test ticket-primitive-5 "Ticket - Primitive (loss-policy=demote, granted)"
	do_test ticket-primitive-6 "Ticket - Primitive (loss-policy=demote, revoked)"
	do_test ticket-primitive-7 "Ticket - Primitive (loss-policy=fence, initial)"
	do_test ticket-primitive-8 "Ticket - Primitive (loss-policy=fence, granted)"
	do_test ticket-primitive-9 "Ticket - Primitive (loss-policy=fence, revoked)"
	do_test ticket-primitive-10 "Ticket - Primitive (loss-policy=freeze, initial)"
	do_test ticket-primitive-11 "Ticket - Primitive (loss-policy=freeze, granted)"
	do_test ticket-primitive-12 "Ticket - Primitive (loss-policy=freeze, revoked)"

	do_test ticket-primitive-13 "Ticket - Primitive (loss-policy=stop, standby, granted)"
	do_test ticket-primitive-14 "Ticket - Primitive (loss-policy=stop, granted, standby)"
	do_test ticket-primitive-15 "Ticket - Primitive (loss-policy=stop, standby, revoked)"
	do_test ticket-primitive-16 "Ticket - Primitive (loss-policy=demote, standby, granted)"
	do_test ticket-primitive-17 "Ticket - Primitive (loss-policy=demote, granted, standby)"
	do_test ticket-primitive-18 "Ticket - Primitive (loss-policy=demote, standby, revoked)"
	do_test ticket-primitive-19 "Ticket - Primitive (loss-policy=fence, standby, granted)"
	do_test ticket-primitive-20 "Ticket - Primitive (loss-policy=fence, granted, standby)"
	do_test ticket-primitive-21 "Ticket - Primitive (loss-policy=fence, standby, revoked)"
	do_test ticket-primitive-22 "Ticket - Primitive (loss-policy=freeze, standby, granted)"
	do_test ticket-primitive-23 "Ticket - Primitive (loss-policy=freeze, granted, standby)"
	do_test ticket-primitive-24 "Ticket - Primitive (loss-policy=freeze, standby, revoked)"

	echo""
	do_test ticket-group-1 "Ticket - Group (loss-policy=stop, initial)"
	do_test ticket-group-2 "Ticket - Group (loss-policy=stop, granted)"
	do_test ticket-group-3 "Ticket - Group (loss-policy-stop, revoked)"
	do_test ticket-group-4 "Ticket - Group (loss-policy=demote, initial)"
	do_test ticket-group-5 "Ticket - Group (loss-policy=demote, granted)"
	do_test ticket-group-6 "Ticket - Group (loss-policy=demote, revoked)"
	do_test ticket-group-7 "Ticket - Group (loss-policy=fence, initial)"
	do_test ticket-group-8 "Ticket - Group (loss-policy=fence, granted)"
	do_test ticket-group-9 "Ticket - Group (loss-policy=fence, revoked)"
	do_test ticket-group-10 "Ticket - Group (loss-policy=freeze, initial)"
	do_test ticket-group-11 "Ticket - Group (loss-policy=freeze, granted)"
	do_test ticket-group-12 "Ticket - Group (loss-policy=freeze, revoked)"

	do_test ticket-group-13 "Ticket - Group (loss-policy=stop, standby, granted)"
	do_test ticket-group-14 "Ticket - Group (loss-policy=stop, granted, standby)"
	do_test ticket-group-15 "Ticket - Group (loss-policy=stop, standby, revoked)"
	do_test ticket-group-16 "Ticket - Group (loss-policy=demote, standby, granted)"
	do_test ticket-group-17 "Ticket - Group (loss-policy=demote, granted, standby)"
	do_test ticket-group-18 "Ticket - Group (loss-policy=demote, standby, revoked)"
	do_test ticket-group-19 "Ticket - Group (loss-policy=fence, standby, granted)"
	do_test ticket-group-20 "Ticket - Group (loss-policy=fence, granted, standby)"
	do_test ticket-group-21 "Ticket - Group (loss-policy=fence, standby, revoked)"
	do_test ticket-group-22 "Ticket - Group (loss-policy=freeze, standby, granted)"
	do_test ticket-group-23 "Ticket - Group (loss-policy=freeze, granted, standby)"
	do_test ticket-group-24 "Ticket - Group (loss-policy=freeze, standby, revoked)"

	echo""
	do_test ticket-clone-1 "Ticket - Clone (loss-policy=stop, initial)"
	do_test ticket-clone-2 "Ticket - Clone (loss-policy=stop, granted)"
	do_test ticket-clone-3 "Ticket - Clone (loss-policy-stop, revoked)"
	do_test ticket-clone-4 "Ticket - Clone (loss-policy=demote, initial)"
	do_test ticket-clone-5 "Ticket - Clone (loss-policy=demote, granted)"
	do_test ticket-clone-6 "Ticket - Clone (loss-policy=demote, revoked)"
	do_test ticket-clone-7 "Ticket - Clone (loss-policy=fence, initial)"
	do_test ticket-clone-8 "Ticket - Clone (loss-policy=fence, granted)"
	do_test ticket-clone-9 "Ticket - Clone (loss-policy=fence, revoked)"
	do_test ticket-clone-10 "Ticket - Clone (loss-policy=freeze, initial)"
	do_test ticket-clone-11 "Ticket - Clone (loss-policy=freeze, granted)"
	do_test ticket-clone-12 "Ticket - Clone (loss-policy=freeze, revoked)"

	do_test ticket-clone-13 "Ticket - Clone (loss-policy=stop, standby, granted)"
	do_test ticket-clone-14 "Ticket - Clone (loss-policy=stop, granted, standby)"
	do_test ticket-clone-15 "Ticket - Clone (loss-policy=stop, standby, revoked)"
	do_test ticket-clone-16 "Ticket - Clone (loss-policy=demote, standby, granted)"
	do_test ticket-clone-17 "Ticket - Clone (loss-policy=demote, granted, standby)"
	do_test ticket-clone-18 "Ticket - Clone (loss-policy=demote, standby, revoked)"
	do_test ticket-clone-19 "Ticket - Clone (loss-policy=fence, standby, granted)"
	do_test ticket-clone-20 "Ticket - Clone (loss-policy=fence, granted, standby)"
	do_test ticket-clone-21 "Ticket - Clone (loss-policy=fence, standby, revoked)"
	do_test ticket-clone-22 "Ticket - Clone (loss-policy=freeze, standby, granted)"
	do_test ticket-clone-23 "Ticket - Clone (loss-policy=freeze, granted, standby)"
	do_test ticket-clone-24 "Ticket - Clone (loss-policy=freeze, standby, revoked)"

	echo""
	do_test ticket-master-1 "Ticket - Master (loss-policy=stop, initial)"
	do_test ticket-master-2 "Ticket - Master (loss-policy=stop, granted)"
	do_test ticket-master-3 "Ticket - Master (loss-policy-stop, revoked)"
	do_test ticket-master-4 "Ticket - Master (loss-policy=demote, initial)"
	do_test ticket-master-5 "Ticket - Master (loss-policy=demote, granted)"
	do_test ticket-master-6 "Ticket - Master (loss-policy=demote, revoked)"
	do_test ticket-master-7 "Ticket - Master (loss-policy=fence, initial)"
	do_test ticket-master-8 "Ticket - Master (loss-policy=fence, granted)"
	do_test ticket-master-9 "Ticket - Master (loss-policy=fence, revoked)"
	do_test ticket-master-10 "Ticket - Master (loss-policy=freeze, initial)"
	do_test ticket-master-11 "Ticket - Master (loss-policy=freeze, granted)"
	do_test ticket-master-12 "Ticket - Master (loss-policy=freeze, revoked)"

	do_test ticket-master-13 "Ticket - Master (loss-policy=stop, standby, granted)"
	do_test ticket-master-14 "Ticket - Master (loss-policy=stop, granted, standby)"
	do_test ticket-master-15 "Ticket - Master (loss-policy=stop, standby, revoked)"
	do_test ticket-master-16 "Ticket - Master (loss-policy=demote, standby, granted)"
	do_test ticket-master-17 "Ticket - Master (loss-policy=demote, granted, standby)"
	do_test ticket-master-18 "Ticket - Master (loss-policy=demote, standby, revoked)"
	do_test ticket-master-19 "Ticket - Master (loss-policy=fence, standby, granted)"
	do_test ticket-master-20 "Ticket - Master (loss-policy=fence, granted, standby)"
	do_test ticket-master-21 "Ticket - Master (loss-policy=fence, standby, revoked)"
	do_test ticket-master-22 "Ticket - Master (loss-policy=freeze, standby, granted)"
	do_test ticket-master-23 "Ticket - Master (loss-policy=freeze, granted, standby)"
	do_test ticket-master-24 "Ticket - Master (loss-policy=freeze, standby, revoked)"

	echo ""
	do_test ticket-rsc-sets-1 "Ticket - Resource sets (1 ticket, initial)"
	do_test ticket-rsc-sets-2 "Ticket - Resource sets (1 ticket, granted)"
	do_test ticket-rsc-sets-3 "Ticket - Resource sets (1 ticket, revoked)"
	do_test ticket-rsc-sets-4 "Ticket - Resource sets (2 tickets, initial)"
	do_test ticket-rsc-sets-5 "Ticket - Resource sets (2 tickets, granted)"
	do_test ticket-rsc-sets-6 "Ticket - Resource sets (2 tickets, granted)"
	do_test ticket-rsc-sets-7 "Ticket - Resource sets (2 tickets, revoked)"

	do_test ticket-rsc-sets-8 "Ticket - Resource sets (1 ticket, standby, granted)"
	do_test ticket-rsc-sets-9 "Ticket - Resource sets (1 ticket, granted, standby)"
	do_test ticket-rsc-sets-10 "Ticket - Resource sets (1 ticket, standby, revoked)"
	do_test ticket-rsc-sets-11 "Ticket - Resource sets (2 tickets, standby, granted)"
	do_test ticket-rsc-sets-12 "Ticket - Resource sets (2 tickets, standby, granted)"
	do_test ticket-rsc-sets-13 "Ticket - Resource sets (2 tickets, granted, standby)"
	do_test ticket-rsc-sets-14 "Ticket - Resource sets (2 tickets, standby, revoked)"

	echo ""
	do_test template-1 "Template - 1"
	do_test template-2 "Template - 2"
	do_test template-3 "Template - 3 (merge operations)"

	do_test template-coloc-1 "Template - Colocation 1"
	do_test template-coloc-2 "Template - Colocation 2"
	do_test template-coloc-3 "Template - Colocation 3"
	do_test template-order-1 "Template - Order 1"
	do_test template-order-2 "Template - Order 2"
	do_test template-order-3 "Template - Order 3"
	do_test template-ticket "Template - Ticket"

	do_test template-rsc-sets-1 "Template - Resource Sets 1"
	do_test template-rsc-sets-2 "Template - Resource Sets 2"
	do_test template-rsc-sets-3 "Template - Resource Sets 3"
	do_test template-rsc-sets-4 "Template - Resource Sets 4"

	do_test template-clone-primitive "Cloned primitive from template"
	do_test template-clone-group "Cloned group from template"

	echo ""
	do_test container-1 "Container - initial"
	do_test container-2 "Container - monitor failed"
	do_test container-3 "Container - stop failed"
	do_test container-4 "Container - reached migration-threshold"
	do_test container-group-1 "Container in group - initial"
	do_test container-group-2 "Container in group - monitor failed"
	do_test container-group-3 "Container in group - stop failed"
	do_test container-group-4 "Container in group - reached migration-threshold"

	echo ""
	do_test whitebox-fail1 "Fail whitebox container rsc."
	do_test whitebox-fail2 "Fail whitebox container rsc lrmd connection."
	do_test whitebox-fail3 "Failed containers should not run nested on remote nodes."
	do_test whitebox-start "Start whitebox container with resources assigned to it"
	do_test whitebox-stop "Stop whitebox container with resources assigned to it"
	do_test whitebox-move "Move whitebox container with resources assigned to it"
	do_test whitebox-asymmetric "Verify connection rsc opts-in based on container resource"
	do_test whitebox-ms-ordering "Verify promote/demote can not occur before connection is established"
	do_test whitebox-ms-ordering "Verify promote/demote can not occur before connection is established"

	+echo ""
	+do_test remote-startup-probes "Baremetal remote-node startup probes"
	+do_test remote-startup "Startup a newly discovered remote-nodes with no status."
	+do_test remote-fence-unclean "Fence unclean baremetal remote-node"
	+do_test remote-move "Move remote-node connection resource"
	+do_test remote-disable "Disable a baremetal remote-node"
	+
	echo ""
	test_results
	diff --git a/pengine/test10/remote-disable.dot b/pengine/test10/remote-disable.dot
	new file mode 100644
	index 0000000000..2e21fefdc2
	--- /dev/null
	+++ b/pengine/test10/remote-disable.dot
	@@ -0,0 +1,12 @@
	+ digraph "g" {
	+"FAKE2_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"]
	+"FAKE2_start_0 18builder" -> "FAKE2_monitor_60000 18builder" [ style = bold]
	+"FAKE2_start_0 18builder" [ style=bold color="green" fontcolor="black"]
	+"FAKE2_stop_0 remote1" -> "FAKE2_start_0 18builder" [ style = bold]
	+"FAKE2_stop_0 remote1" -> "all_stopped" [ style = bold]
	+"FAKE2_stop_0 remote1" -> "remote1_stop_0 18builder" [ style = bold]
	+"FAKE2_stop_0 remote1" [ style=bold color="green" fontcolor="black"]
	+"all_stopped" [ style=bold color="green" fontcolor="orange"]
	+"remote1_stop_0 18builder" -> "all_stopped" [ style = bold]
	+"remote1_stop_0 18builder" [ style=bold color="green" fontcolor="black"]
	+}
	diff --git a/pengine/test10/remote-disable.exp b/pengine/test10/remote-disable.exp
	new file mode 100644
	index 0000000000..752cca6336
	--- /dev/null
	+++ b/pengine/test10/remote-disable.exp
	@@ -0,0 +1,65 @@
	+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" batch-limit="30" transition_id="0">
	+ <synapse id="0">
	+ <action_set>
	+ <rsc_op id="16" operation="stop" operation_key="remote1_stop_0" on_node="18builder" on_node_uuid="5">
	+ <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="19" operation="stop" operation_key="FAKE2_stop_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="1">
	+ <action_set>
	+ <rsc_op id="21" operation="monitor" operation_key="FAKE2_monitor_60000" on_node="18builder" on_node_uuid="5">
	+ <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="20" operation="start" operation_key="FAKE2_start_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="2">
	+ <action_set>
	+ <rsc_op id="20" operation="start" operation_key="FAKE2_start_0" on_node="18builder" on_node_uuid="5">
	+ <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="19" operation="stop" operation_key="FAKE2_stop_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="3">
	+ <action_set>
	+ <rsc_op id="19" operation="stop" operation_key="FAKE2_stop_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
	+ <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="4">
	+ <action_set>
	+ <pseudo_event id="7" operation="all_stopped" operation_key="all_stopped">
	+ <attributes crm_feature_set="3.0.7"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="16" operation="stop" operation_key="remote1_stop_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="19" operation="stop" operation_key="FAKE2_stop_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+</transition_graph>
	diff --git a/pengine/test10/remote-disable.scores b/pengine/test10/remote-disable.scores
	new file mode 100644
	index 0000000000..4efd7d8dfa
	--- /dev/null
	+++ b/pengine/test10/remote-disable.scores
	@@ -0,0 +1,25 @@
	+Allocation scores:
	+native_color: FAKE1 allocation score on 18builder: 0
	+native_color: FAKE1 allocation score on 18node1: 0
	+native_color: FAKE1 allocation score on 18node2: 0
	+native_color: FAKE1 allocation score on remote1: 0
	+native_color: FAKE2 allocation score on 18builder: 0
	+native_color: FAKE2 allocation score on 18node1: 0
	+native_color: FAKE2 allocation score on 18node2: 0
	+native_color: FAKE2 allocation score on remote1: 0
	+native_color: FAKE3 allocation score on 18builder: 0
	+native_color: FAKE3 allocation score on 18node1: 0
	+native_color: FAKE3 allocation score on 18node2: 0
	+native_color: FAKE3 allocation score on remote1: 0
	+native_color: FAKE4 allocation score on 18builder: 0
	+native_color: FAKE4 allocation score on 18node1: 0
	+native_color: FAKE4 allocation score on 18node2: 0
	+native_color: FAKE4 allocation score on remote1: 0
	+native_color: remote1 allocation score on 18builder: -INFINITY
	+native_color: remote1 allocation score on 18node1: -INFINITY
	+native_color: remote1 allocation score on 18node2: -INFINITY
	+native_color: remote1 allocation score on remote1: -INFINITY
	+native_color: shooter allocation score on 18builder: 0
	+native_color: shooter allocation score on 18node1: 0
	+native_color: shooter allocation score on 18node2: 0
	+native_color: shooter allocation score on remote1: -INFINITY
	diff --git a/pengine/test10/remote-disable.summary b/pengine/test10/remote-disable.summary
	new file mode 100644
	index 0000000000..57b06fe532
	--- /dev/null
	+++ b/pengine/test10/remote-disable.summary
	@@ -0,0 +1,34 @@
	+
	+Current cluster status:
	+Online: [ 18builder 18node1 18node2 ]
	+RemoteOnline: [ remote1 ]
	+
	+ shooter (stonith:fence_xvm): Started 18node1
	+ remote1 (ocf::pacemaker:remote): Started 18builder
	+ FAKE1 (ocf::heartbeat:Dummy): Started 18node2
	+ FAKE2 (ocf::heartbeat:Dummy): Started remote1
	+ FAKE3 (ocf::heartbeat:Dummy): Started 18builder
	+ FAKE4 (ocf::heartbeat:Dummy): Started 18node1
	+
	+Transition Summary:
	+ * Stop remote1 (18builder)
	+ * Move FAKE2 (Started remote1 -> 18builder)
	+
	+Executing cluster transition:
	+ * Resource action: FAKE2 stop on remote1
	+ * Resource action: remote1 stop on 18builder
	+ * Resource action: FAKE2 start on 18builder
	+ * Pseudo action: all_stopped
	+ * Resource action: FAKE2 monitor=60000 on 18builder
	+
	+Revised cluster status:
	+Online: [ 18builder 18node1 18node2 ]
	+RemoteOFFLINE: [ remote1 ]
	+
	+ shooter (stonith:fence_xvm): Started 18node1
	+ remote1 (ocf::pacemaker:remote): Stopped
	+ FAKE1 (ocf::heartbeat:Dummy): Started 18node2
	+ FAKE2 (ocf::heartbeat:Dummy): Started 18builder
	+ FAKE3 (ocf::heartbeat:Dummy): Started 18builder
	+ FAKE4 (ocf::heartbeat:Dummy): Started 18node1
	+
	diff --git a/pengine/test10/remote-disable.xml b/pengine/test10/remote-disable.xml
	new file mode 100644
	index 0000000000..eae0055f3c
	--- /dev/null
	+++ b/pengine/test10/remote-disable.xml
	@@ -0,0 +1,180 @@
	+<cib epoch="12" num_updates="1" admin_epoch="0" validate-with="pacemaker-1.2" cib-last-written="Tue Sep 3 18:01:09 2013" update-origin="18builder" update-client="crm_resource" crm_feature_set="3.0.7" have-quorum="1" dc-uuid="5">
	+ <configuration>
	+ <crm_config>
	+ <cluster_property_set id="cib-bootstrap-options">
	+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.10-ab50afe"/>
	+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
	+ </cluster_property_set>
	+ </crm_config>
	+ <nodes>
	+ <node id="1" uname="18node1"/>
	+ <node id="2" uname="18node2"/>
	+ <node id="5" uname="18builder"/>
	+ </nodes>
	+ <resources>
	+ <primitive class="stonith" id="shooter" type="fence_xvm">
	+ <instance_attributes id="shooter-instance_attributes"/>
	+ <operations>
	+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="remote1" provider="pacemaker" type="remote">
	+ <instance_attributes id="remote1-instance_attributes"/>
	+ <operations>
	+ <op id="remote1-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ <meta_attributes id="remote1-meta_attributes">
	+ <nvpair id="remote1-meta_attributes-target-role" name="target-role" value="Stopped"/>
	+ </meta_attributes>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE1" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE1-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE1-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE2" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE2-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE2-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE3" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE3-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE3-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE4" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE4-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE4-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ </resources>
	+ <constraints/>
	+ </configuration>
	+ <status>
	+ <node_state id="5" uname="18builder" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
	+ <transient_attributes id="5">
	+ <instance_attributes id="status-5">
	+ <nvpair id="status-5-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ <lrm id="5">
	+ <lrm_resources>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="11:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;11:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="16" rc-code="0" op-status="0" interval="0" last-run="1378248545" last-rc-change="1378248545" exec-time="1" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="10:3:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;10:3:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="11" rc-code="0" op-status="0" interval="60000" last-rc-change="1378248534" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="remote1_last_0" operation_key="remote1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="19:13:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;19:13:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="14" rc-code="0" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="remote1_monitor_60000" operation_key="remote1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="21:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;21:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="15" rc-code="0" op-status="0" interval="60000" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="22:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;22:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="65" rc-code="0" op-status="0" interval="0" last-run="1378249167" last-rc-change="1378249167" exec-time="31" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE1_monitor_60000" operation_key="FAKE1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="13:8:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;13:8:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="34" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249143" exec-time="9" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="7:9:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;7:9:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="39" rc-code="7" op-status="0" interval="0" last-run="1378249145" last-rc-change="1378249145" exec-time="61" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="18:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;18:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="53" rc-code="0" op-status="0" interval="0" last-run="1378249147" last-rc-change="1378249147" exec-time="11" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE3_monitor_60000" operation_key="FAKE3_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="19:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;19:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="56" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249147" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="9:12:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;9:12:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="61" rc-code="7" op-status="0" interval="0" last-run="1378249149" last-rc-change="1378249149" exec-time="65" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ </node_state>
	+ <node_state id="1" uname="18node1" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
	+ <lrm id="1">
	+ <lrm_resources>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="12:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;12:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="9" rc-code="0" op-status="0" interval="0" last-run="1378248547" last-rc-change="1378248547" exec-time="19" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="13:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;13:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="12" rc-code="0" op-status="0" interval="60000" last-rc-change="1378248548" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="remote1_last_0" operation_key="remote1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="8:4:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;8:4:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="1" rc-code="7" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_failure_0" operation_key="FAKE1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="8:7:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;8:7:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="17" rc-code="0" op-status="0" interval="0" last-run="1378249142" last-rc-change="1378249142" exec-time="50" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="11:8:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;11:8:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="20" rc-code="0" op-status="0" interval="0" last-run="1378249142" last-rc-change="1378249142" exec-time="18" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="9:9:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;9:9:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="25" rc-code="7" op-status="0" interval="0" last-run="1378249144" last-rc-change="1378249144" exec-time="28" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_failure_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="10:10:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;10:10:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="32" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="17:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;17:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="33" rc-code="0" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="15" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="22:12:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;22:12:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="40" rc-code="0" op-status="0" interval="0" last-run="1378249148" last-rc-change="1378249148" exec-time="36" queue-time="38" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE4_monitor_60000" operation_key="FAKE4_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="23:12:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;23:12:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="44" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249148" exec-time="14" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ <transient_attributes id="1">
	+ <instance_attributes id="status-1">
	+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ </node_state>
	+ <node_state id="2" uname="18node2" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
	+ <lrm id="2">
	+ <lrm_resources>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="8:3:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;8:3:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1378248530" last-rc-change="1378248530" exec-time="980" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="remote1_last_0" operation_key="remote1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="10:4:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;10:4:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="1" rc-code="7" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="23:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;23:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="40" rc-code="0" op-status="0" interval="0" last-run="1378249166" last-rc-change="1378249166" exec-time="28" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE1_monitor_60000" operation_key="FAKE1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="24:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;24:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="43" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249166" exec-time="16" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="25:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;25:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="36" rc-code="0" op-status="0" interval="0" last-run="1378249166" last-rc-change="1378249166" exec-time="30" queue-time="1" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE2_monitor_60000" operation_key="FAKE2_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="17:9:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;17:9:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="22" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249144" exec-time="20" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="12:10:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;12:10:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="27" rc-code="7" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="35" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="13:12:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;13:12:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="32" rc-code="7" op-status="0" interval="0" last-run="1378249148" last-rc-change="1378249148" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ <transient_attributes id="2">
	+ <instance_attributes id="status-2">
	+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ </node_state>
	+ <node_state remote_node="true" id="remote1" uname="remote1" crm-debug-origin="do_update_resource">
	+ <lrm id="remote1">
	+ <lrm_resources>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="13:14:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;13:14:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="16" rc-code="7" op-status="0" interval="0" last-run="1378231164" last-rc-change="1378231164" exec-time="84" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="26:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;26:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="35" rc-code="0" op-status="0" interval="0" last-run="1378231166" last-rc-change="1378231166" exec-time="17" queue-time="1" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE2_monitor_60000" operation_key="FAKE2_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="27:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;27:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="38" rc-code="0" op-status="0" interval="60000" last-rc-change="1378231166" exec-time="10" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="15:14:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;15:14:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="24" rc-code="7" op-status="0" interval="0" last-run="1378231165" last-rc-change="1378231165" exec-time="8" queue-time="1" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="16:14:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;16:14:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="28" rc-code="7" op-status="0" interval="0" last-run="1378231165" last-rc-change="1378231165" exec-time="14" queue-time="1" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ <transient_attributes id="remote1">
	+ <instance_attributes id="status-remote1">
	+ <nvpair id="status-remote1-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ </node_state>
	+ </status>
	+</cib>
	diff --git a/pengine/test10/remote-fence-unclean.dot b/pengine/test10/remote-fence-unclean.dot
	new file mode 100644
	index 0000000000..2ddf63ee4b
	--- /dev/null
	+++ b/pengine/test10/remote-fence-unclean.dot
	@@ -0,0 +1,36 @@
	+ digraph "g" {
	+"FAKE2_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"]
	+"FAKE2_start_0 18builder" -> "FAKE2_monitor_60000 18builder" [ style = bold]
	+"FAKE2_start_0 18builder" [ style=bold color="green" fontcolor="black"]
	+"FAKE2_stop_0 remote1" -> "FAKE2_start_0 18builder" [ style = bold]
	+"FAKE2_stop_0 remote1" -> "all_stopped" [ style = bold]
	+"FAKE2_stop_0 remote1" [ style=bold color="green" fontcolor="orange"]
	+"FAKE3_monitor_60000 18node1" [ style=bold color="green" fontcolor="black"]
	+"FAKE3_start_0 18node1" -> "FAKE3_monitor_60000 18node1" [ style = bold]
	+"FAKE3_start_0 18node1" [ style=bold color="green" fontcolor="black"]
	+"FAKE3_stop_0 18builder" -> "FAKE3_start_0 18node1" [ style = bold]
	+"FAKE3_stop_0 18builder" -> "all_stopped" [ style = bold]
	+"FAKE3_stop_0 18builder" [ style=bold color="green" fontcolor="black"]
	+"FAKE4_monitor_60000 18node2" [ style=bold color="green" fontcolor="black"]
	+"FAKE4_start_0 18node2" -> "FAKE4_monitor_60000 18node2" [ style = bold]
	+"FAKE4_start_0 18node2" [ style=bold color="green" fontcolor="black"]
	+"FAKE4_stop_0 18node1" -> "FAKE4_start_0 18node2" [ style = bold]
	+"FAKE4_stop_0 18node1" -> "all_stopped" [ style = bold]
	+"FAKE4_stop_0 18node1" [ style=bold color="green" fontcolor="black"]
	+"all_stopped" [ style=bold color="green" fontcolor="orange"]
	+"remote1_monitor_60000 18node1" [ style=bold color="green" fontcolor="black"]
	+"remote1_start_0 18node1" -> "remote1_monitor_60000 18node1" [ style = bold]
	+"remote1_start_0 18node1" [ style=bold color="green" fontcolor="black"]
	+"remote1_stop_0 18node1" -> "all_stopped" [ style = bold]
	+"remote1_stop_0 18node1" -> "remote1_start_0 18node1" [ style = bold]
	+"remote1_stop_0 18node1" [ style=bold color="green" fontcolor="black"]
	+"stonith remote1" -> "FAKE2_stop_0 remote1" [ style = bold]
	+"stonith remote1" -> "stonith_complete" [ style = bold]
	+"stonith remote1" [ style=bold color="green" fontcolor="black"]
	+"stonith_complete" -> "FAKE2_start_0 18builder" [ style = bold]
	+"stonith_complete" -> "FAKE3_start_0 18node1" [ style = bold]
	+"stonith_complete" -> "FAKE4_start_0 18node2" [ style = bold]
	+"stonith_complete" -> "all_stopped" [ style = bold]
	+"stonith_complete" -> "remote1_start_0 18node1" [ style = bold]
	+"stonith_complete" [ style=bold color="green" fontcolor="orange"]
	+}
	diff --git a/pengine/test10/remote-fence-unclean.exp b/pengine/test10/remote-fence-unclean.exp
	new file mode 100644
	index 0000000000..f9a8fa82e1
	--- /dev/null
	+++ b/pengine/test10/remote-fence-unclean.exp
	@@ -0,0 +1,201 @@
	+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" batch-limit="30" transition_id="0">
	+ <synapse id="0">
	+ <action_set>
	+ <rsc_op id="15" operation="start" operation_key="remote1_start_0" on_node="18node1" on_node_uuid="1">
	+ <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="5" operation="stop" operation_key="remote1_stop_0" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ <trigger>
	+ <pseudo_event id="27" operation="stonith_complete" operation_key="stonith_complete"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="1">
	+ <action_set>
	+ <rsc_op id="5" operation="stop" operation_key="remote1_stop_0" on_node="18node1" on_node_uuid="1">
	+ <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="2">
	+ <action_set>
	+ <rsc_op id="4" operation="monitor" operation_key="remote1_monitor_60000" on_node="18node1" on_node_uuid="1">
	+ <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
	+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="15" operation="start" operation_key="remote1_start_0" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="3">
	+ <action_set>
	+ <rsc_op id="20" operation="monitor" operation_key="FAKE2_monitor_60000" on_node="18builder" on_node_uuid="5">
	+ <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="19" operation="start" operation_key="FAKE2_start_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="4">
	+ <action_set>
	+ <rsc_op id="19" operation="start" operation_key="FAKE2_start_0" on_node="18builder" on_node_uuid="5">
	+ <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="18" operation="stop" operation_key="FAKE2_stop_0"/>
	+ </trigger>
	+ <trigger>
	+ <pseudo_event id="27" operation="stonith_complete" operation_key="stonith_complete"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="5">
	+ <action_set>
	+ <pseudo_event id="18" operation="stop" operation_key="FAKE2_stop_0">
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <crm_event id="28" operation="stonith" operation_key="stonith-remote1" on_node="remote1" on_node_uuid="remote1" router_node="18node1"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="6">
	+ <action_set>
	+ <rsc_op id="23" operation="monitor" operation_key="FAKE3_monitor_60000" on_node="18node1" on_node_uuid="1">
	+ <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="22" operation="start" operation_key="FAKE3_start_0" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="7">
	+ <action_set>
	+ <rsc_op id="22" operation="start" operation_key="FAKE3_start_0" on_node="18node1" on_node_uuid="1">
	+ <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="21" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ <trigger>
	+ <pseudo_event id="27" operation="stonith_complete" operation_key="stonith_complete"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="8">
	+ <action_set>
	+ <rsc_op id="21" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5">
	+ <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="9">
	+ <action_set>
	+ <rsc_op id="26" operation="monitor" operation_key="FAKE4_monitor_60000" on_node="18node2" on_node_uuid="2">
	+ <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="25" operation="start" operation_key="FAKE4_start_0" on_node="18node2" on_node_uuid="2"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="10">
	+ <action_set>
	+ <rsc_op id="25" operation="start" operation_key="FAKE4_start_0" on_node="18node2" on_node_uuid="2">
	+ <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="24" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ <trigger>
	+ <pseudo_event id="27" operation="stonith_complete" operation_key="stonith_complete"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="11">
	+ <action_set>
	+ <rsc_op id="24" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1">
	+ <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="12">
	+ <action_set>
	+ <crm_event id="28" operation="stonith" operation_key="stonith-remote1" on_node="remote1" on_node_uuid="remote1" router_node="18node1">
	+ <attributes CRM_meta_on_node="remote1" CRM_meta_on_node_uuid="remote1" CRM_meta_probe_complete="true" CRM_meta_stonith_action="reboot" crm_feature_set="3.0.7"/>
	+ </crm_event>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="13">
	+ <action_set>
	+ <pseudo_event id="27" operation="stonith_complete" operation_key="stonith_complete">
	+ <attributes crm_feature_set="3.0.7"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <crm_event id="28" operation="stonith" operation_key="stonith-remote1" on_node="remote1" on_node_uuid="remote1" router_node="18node1"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="14">
	+ <action_set>
	+ <pseudo_event id="7" operation="all_stopped" operation_key="all_stopped">
	+ <attributes crm_feature_set="3.0.7"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="5" operation="stop" operation_key="remote1_stop_0" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ <trigger>
	+ <pseudo_event id="18" operation="stop" operation_key="FAKE2_stop_0"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="21" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="24" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ <trigger>
	+ <pseudo_event id="27" operation="stonith_complete" operation_key="stonith_complete"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+</transition_graph>
	diff --git a/pengine/test10/remote-fence-unclean.scores b/pengine/test10/remote-fence-unclean.scores
	new file mode 100644
	index 0000000000..8d296627e9
	--- /dev/null
	+++ b/pengine/test10/remote-fence-unclean.scores
	@@ -0,0 +1,25 @@
	+Allocation scores:
	+native_color: FAKE1 allocation score on 18builder: 0
	+native_color: FAKE1 allocation score on 18node1: 0
	+native_color: FAKE1 allocation score on 18node2: 0
	+native_color: FAKE1 allocation score on remote1: 0
	+native_color: FAKE2 allocation score on 18builder: 0
	+native_color: FAKE2 allocation score on 18node1: 0
	+native_color: FAKE2 allocation score on 18node2: 0
	+native_color: FAKE2 allocation score on remote1: 0
	+native_color: FAKE3 allocation score on 18builder: 0
	+native_color: FAKE3 allocation score on 18node1: 0
	+native_color: FAKE3 allocation score on 18node2: 0
	+native_color: FAKE3 allocation score on remote1: 0
	+native_color: FAKE4 allocation score on 18builder: 0
	+native_color: FAKE4 allocation score on 18node1: 0
	+native_color: FAKE4 allocation score on 18node2: 0
	+native_color: FAKE4 allocation score on remote1: 0
	+native_color: remote1 allocation score on 18builder: -INFINITY
	+native_color: remote1 allocation score on 18node1: 0
	+native_color: remote1 allocation score on 18node2: 0
	+native_color: remote1 allocation score on remote1: -INFINITY
	+native_color: shooter allocation score on 18builder: 0
	+native_color: shooter allocation score on 18node1: 0
	+native_color: shooter allocation score on 18node2: 0
	+native_color: shooter allocation score on remote1: -INFINITY
	diff --git a/pengine/test10/remote-fence-unclean.summary b/pengine/test10/remote-fence-unclean.summary
	new file mode 100644
	index 0000000000..052b7db4b4
	--- /dev/null
	+++ b/pengine/test10/remote-fence-unclean.summary
	@@ -0,0 +1,46 @@
	+
	+Current cluster status:
	+RemoteNode remote1: UNCLEAN (offline)
	+Online: [ 18builder 18node1 18node2 ]
	+
	+ shooter (stonith:fence_xvm): Started 18builder
	+ remote1 (ocf::pacemaker:remote): FAILED 18node1
	+ FAKE1 (ocf::heartbeat:Dummy): Started 18node2
	+ FAKE2 (ocf::heartbeat:Dummy): Started remote1
	+ FAKE3 (ocf::heartbeat:Dummy): Started 18builder
	+ FAKE4 (ocf::heartbeat:Dummy): Started 18node1
	+
	+Transition Summary:
	+ * Recover remote1 (Started 18node1)
	+ * Move FAKE2 (Started remote1 -> 18builder)
	+ * Move FAKE3 (Started 18builder -> 18node1)
	+ * Move FAKE4 (Started 18node1 -> 18node2)
	+
	+Executing cluster transition:
	+ * Resource action: remote1 stop on 18node1
	+ * Resource action: FAKE3 stop on 18builder
	+ * Resource action: FAKE4 stop on 18node1
	+ * Fencing remote1
	+ * Pseudo action: stonith_complete
	+ * Resource action: remote1 start on 18node1
	+ * Resource action: remote1 monitor=60000 on 18node1
	+ * Pseudo action: FAKE2_stop_0
	+ * Resource action: FAKE3 start on 18node1
	+ * Resource action: FAKE4 start on 18node2
	+ * Pseudo action: all_stopped
	+ * Resource action: FAKE2 start on 18builder
	+ * Resource action: FAKE3 monitor=60000 on 18node1
	+ * Resource action: FAKE4 monitor=60000 on 18node2
	+ * Resource action: FAKE2 monitor=60000 on 18builder
	+
	+Revised cluster status:
	+Online: [ 18builder 18node1 18node2 ]
	+RemoteOnline: [ remote1 ]
	+
	+ shooter (stonith:fence_xvm): Started 18builder
	+ remote1 (ocf::pacemaker:remote): Started 18node1
	+ FAKE1 (ocf::heartbeat:Dummy): Started 18node2
	+ FAKE2 (ocf::heartbeat:Dummy): Started 18builder
	+ FAKE3 (ocf::heartbeat:Dummy): Started 18node1
	+ FAKE4 (ocf::heartbeat:Dummy): Started 18node2
	+
	diff --git a/pengine/test10/remote-fence-unclean.xml b/pengine/test10/remote-fence-unclean.xml
	new file mode 100644
	index 0000000000..f1fc479e5b
	--- /dev/null
	+++ b/pengine/test10/remote-fence-unclean.xml
	@@ -0,0 +1,182 @@
	+<cib epoch="14" num_updates="21" admin_epoch="0" validate-with="pacemaker-1.2" cib-last-written="Tue Sep 3 18:02:57 2013" update-origin="18builder" update-client="cibadmin" crm_feature_set="3.0.7" have-quorum="1" dc-uuid="5">
	+ <configuration>
	+ <crm_config>
	+ <cluster_property_set id="cib-bootstrap-options">
	+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.10-ab50afe"/>
	+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
	+ </cluster_property_set>
	+ </crm_config>
	+ <nodes>
	+ <node id="1" uname="18node1"/>
	+ <node id="2" uname="18node2"/>
	+ <node id="5" uname="18builder"/>
	+ </nodes>
	+ <resources>
	+ <primitive class="stonith" id="shooter" type="fence_xvm">
	+ <instance_attributes id="shooter-instance_attributes"/>
	+ <operations>
	+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="remote1" provider="pacemaker" type="remote">
	+ <instance_attributes id="remote1-instance_attributes"/>
	+ <operations>
	+ <op id="remote1-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ <meta_attributes id="remote1-meta_attributes"/>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE1" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE1-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE1-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE2" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE2-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE2-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE3" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE3-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE3-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE4" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE4-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE4-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ </resources>
	+ <constraints>
	+ <rsc_location id="location-remote1-18builder--INFINITY" node="18builder" rsc="remote1" score="-INFINITY"/>
	+ </constraints>
	+ </configuration>
	+ <status>
	+ <node_state id="5" uname="18builder" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
	+ <transient_attributes id="5">
	+ <instance_attributes id="status-5">
	+ <nvpair id="status-5-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ <lrm id="5">
	+ <lrm_resources>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="18:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;18:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="53" rc-code="0" op-status="0" interval="0" last-run="1378249147" last-rc-change="1378249147" exec-time="11" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE3_monitor_60000" operation_key="FAKE3_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="19:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;19:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="56" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249147" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="9:12:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;9:12:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="61" rc-code="7" op-status="0" interval="0" last-run="1378249149" last-rc-change="1378249149" exec-time="65" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="15:18:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;15:18:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="80" rc-code="0" op-status="0" interval="0" last-run="1378249379" last-rc-change="1378249379" exec-time="24" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="16:18:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;16:18:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="83" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249380" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="remote1_last_0" operation_key="remote1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="17:18:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;17:18:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="20" rc-code="0" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="remote1_monitor_60000" operation_key="remote1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="20:17:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;20:17:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="19" rc-code="0" op-status="0" interval="60000" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="22:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;22:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="65" rc-code="0" op-status="0" interval="0" last-run="1378249167" last-rc-change="1378249167" exec-time="31" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="19:16:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;19:16:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="76" rc-code="0" op-status="0" interval="0" last-run="1378249334" last-rc-change="1378249334" exec-time="45" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ </node_state>
	+ <node_state id="1" uname="18node1" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
	+ <transient_attributes id="1">
	+ <instance_attributes id="status-1">
	+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
	+ <nvpair id="status-1-fail-count-remote1" name="fail-count-remote1" value="1"/>
	+ <nvpair id="status-1-last-failure-remote1" name="last-failure-remote1" value="1378251122"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ <lrm id="1">
	+ <lrm_resources>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="17:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;17:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="33" rc-code="0" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="15" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE3_last_failure_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="10:10:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;10:10:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="32" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="22:12:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;22:12:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="40" rc-code="0" op-status="0" interval="0" last-run="1378249148" last-rc-change="1378249148" exec-time="36" queue-time="38" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE4_monitor_60000" operation_key="FAKE4_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="23:12:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;23:12:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="44" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249148" exec-time="14" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="14:18:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;14:18:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="48" rc-code="0" op-status="0" interval="0" last-run="1378249378" last-rc-change="1378249378" exec-time="1" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="13:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;13:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="12" rc-code="0" op-status="0" interval="60000" last-rc-change="1378248548" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="remote1_last_0" operation_key="remote1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="18:18:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;18:18:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="2" rc-code="0" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="remote1_monitor_60000" operation_key="remote1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="19:18:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;19:18:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="3" rc-code="0" op-status="0" interval="60000" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ <lrm_rsc_op id="remote1_last_failure_0" operation_key="remote1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="19:18:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;19:18:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="32" rc-code="7" op-status="0" interval="60000" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="11:8:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;11:8:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="20" rc-code="0" op-status="0" interval="0" last-run="1378249142" last-rc-change="1378249142" exec-time="18" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE1_last_failure_0" operation_key="FAKE1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="8:7:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;8:7:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="17" rc-code="0" op-status="0" interval="0" last-run="1378249142" last-rc-change="1378249142" exec-time="50" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="9:9:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;9:9:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="25" rc-code="7" op-status="0" interval="0" last-run="1378249144" last-rc-change="1378249144" exec-time="28" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ </node_state>
	+ <node_state id="2" uname="18node2" crmd="online" crm-debug-origin="do_state_transition" in_ccm="true" join="member" expected="member">
	+ <transient_attributes id="2">
	+ <instance_attributes id="status-2">
	+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ <lrm id="2">
	+ <lrm_resources>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="12:10:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;12:10:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="27" rc-code="7" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="35" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="13:12:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;13:12:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="32" rc-code="7" op-status="0" interval="0" last-run="1378249148" last-rc-change="1378249148" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="8:3:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;8:3:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1378248530" last-rc-change="1378248530" exec-time="980" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="remote1_last_0" operation_key="remote1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="10:4:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;10:4:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="1" rc-code="7" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="23:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;23:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="40" rc-code="0" op-status="0" interval="0" last-run="1378249166" last-rc-change="1378249166" exec-time="28" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE1_monitor_60000" operation_key="FAKE1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="24:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;24:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="43" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249166" exec-time="16" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="25:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;25:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="36" rc-code="0" op-status="0" interval="0" last-run="1378249166" last-rc-change="1378249166" exec-time="30" queue-time="1" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ </node_state>
	+ <node_state remote_node="true" id="remote1" uname="remote1" crm-debug-origin="do_update_resource">
	+ <lrm id="remote1">
	+ <lrm_resources>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="12:17:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;12:17:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="48" rc-code="7" op-status="0" interval="0" last-run="1378231334" last-rc-change="1378231334" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="23:18:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;23:18:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="74" rc-code="0" op-status="0" interval="0" last-run="1378231379" last-rc-change="1378231379" exec-time="11" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE2_monitor_60000" operation_key="FAKE2_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="6:18:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;6:18:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="77" rc-code="0" op-status="0" interval="60000" last-rc-change="1378231380" exec-time="13" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="14:17:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;14:17:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="52" rc-code="7" op-status="0" interval="0" last-run="1378231334" last-rc-change="1378231334" exec-time="13" queue-time="1" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="15:17:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;15:17:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="54" rc-code="7" op-status="0" interval="0" last-run="1378231334" last-rc-change="1378231334" exec-time="7" queue-time="1" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ <transient_attributes id="remote1">
	+ <instance_attributes id="status-remote1">
	+ <nvpair id="status-remote1-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ </node_state>
	+ </status>
	+</cib>
	diff --git a/pengine/test10/remote-move.dot b/pengine/test10/remote-move.dot
	new file mode 100644
	index 0000000000..8cef9d8bf4
	--- /dev/null
	+++ b/pengine/test10/remote-move.dot
	@@ -0,0 +1,24 @@
	+ digraph "g" {
	+"FAKE2_monitor_60000 remote1" [ style=bold color="green" fontcolor="black"]
	+"FAKE2_start_0 remote1" -> "FAKE2_monitor_60000 remote1" [ style = bold]
	+"FAKE2_start_0 remote1" [ style=bold color="green" fontcolor="black"]
	+"FAKE2_stop_0 remote1" -> "FAKE2_start_0 remote1" [ style = bold]
	+"FAKE2_stop_0 remote1" -> "all_stopped" [ style = bold]
	+"FAKE2_stop_0 remote1" -> "remote1_stop_0 18builder" [ style = bold]
	+"FAKE2_stop_0 remote1" [ style=bold color="green" fontcolor="black"]
	+"all_stopped" [ style=bold color="green" fontcolor="orange"]
	+"remote1_monitor_60000 18node1" [ style=bold color="green" fontcolor="black"]
	+"remote1_start_0 18node1" -> "FAKE2_monitor_60000 remote1" [ style = bold]
	+"remote1_start_0 18node1" -> "FAKE2_start_0 remote1" [ style = bold]
	+"remote1_start_0 18node1" -> "remote1_monitor_60000 18node1" [ style = bold]
	+"remote1_start_0 18node1" [ style=bold color="green" fontcolor="black"]
	+"remote1_stop_0 18builder" -> "all_stopped" [ style = bold]
	+"remote1_stop_0 18builder" -> "remote1_start_0 18node1" [ style = bold]
	+"remote1_stop_0 18builder" [ style=bold color="green" fontcolor="black"]
	+"shooter_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"]
	+"shooter_start_0 18builder" -> "shooter_monitor_60000 18builder" [ style = bold]
	+"shooter_start_0 18builder" [ style=bold color="green" fontcolor="black"]
	+"shooter_stop_0 18node1" -> "all_stopped" [ style = bold]
	+"shooter_stop_0 18node1" -> "shooter_start_0 18builder" [ style = bold]
	+"shooter_stop_0 18node1" [ style=bold color="green" fontcolor="black"]
	+}
	diff --git a/pengine/test10/remote-move.exp b/pengine/test10/remote-move.exp
	new file mode 100644
	index 0000000000..4f1a80ce14
	--- /dev/null
	+++ b/pengine/test10/remote-move.exp
	@@ -0,0 +1,135 @@
	+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" batch-limit="30" transition_id="0">
	+ <synapse id="0">
	+ <action_set>
	+ <rsc_op id="16" operation="monitor" operation_key="shooter_monitor_60000" on_node="18builder" on_node_uuid="5">
	+ <primitive id="shooter" class="stonith" type="fence_xvm"/>
	+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="15" operation="start" operation_key="shooter_start_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="1">
	+ <action_set>
	+ <rsc_op id="15" operation="start" operation_key="shooter_start_0" on_node="18builder" on_node_uuid="5">
	+ <primitive id="shooter" class="stonith" type="fence_xvm"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="14" operation="stop" operation_key="shooter_stop_0" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="2">
	+ <action_set>
	+ <rsc_op id="14" operation="stop" operation_key="shooter_stop_0" on_node="18node1" on_node_uuid="1">
	+ <primitive id="shooter" class="stonith" type="fence_xvm"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="3">
	+ <action_set>
	+ <rsc_op id="19" operation="monitor" operation_key="remote1_monitor_60000" on_node="18node1" on_node_uuid="1">
	+ <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
	+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="18" operation="start" operation_key="remote1_start_0" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="4">
	+ <action_set>
	+ <rsc_op id="18" operation="start" operation_key="remote1_start_0" on_node="18node1" on_node_uuid="1">
	+ <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="17" operation="stop" operation_key="remote1_stop_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="5">
	+ <action_set>
	+ <rsc_op id="17" operation="stop" operation_key="remote1_stop_0" on_node="18builder" on_node_uuid="5">
	+ <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="22" operation="stop" operation_key="FAKE2_stop_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="6">
	+ <action_set>
	+ <rsc_op id="23" operation="start" operation_key="FAKE2_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18node1">
	+ <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="18" operation="start" operation_key="remote1_start_0" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="22" operation="stop" operation_key="FAKE2_stop_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="7">
	+ <action_set>
	+ <rsc_op id="22" operation="stop" operation_key="FAKE2_stop_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
	+ <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="8">
	+ <action_set>
	+ <rsc_op id="6" operation="monitor" operation_key="FAKE2_monitor_60000" on_node="remote1" on_node_uuid="remote1" router_node="18node1">
	+ <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="18" operation="start" operation_key="remote1_start_0" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="23" operation="start" operation_key="FAKE2_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18node1"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="9">
	+ <action_set>
	+ <pseudo_event id="7" operation="all_stopped" operation_key="all_stopped">
	+ <attributes crm_feature_set="3.0.7"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="14" operation="stop" operation_key="shooter_stop_0" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="17" operation="stop" operation_key="remote1_stop_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="22" operation="stop" operation_key="FAKE2_stop_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+</transition_graph>
	diff --git a/pengine/test10/remote-move.scores b/pengine/test10/remote-move.scores
	new file mode 100644
	index 0000000000..8d296627e9
	--- /dev/null
	+++ b/pengine/test10/remote-move.scores
	@@ -0,0 +1,25 @@
	+Allocation scores:
	+native_color: FAKE1 allocation score on 18builder: 0
	+native_color: FAKE1 allocation score on 18node1: 0
	+native_color: FAKE1 allocation score on 18node2: 0
	+native_color: FAKE1 allocation score on remote1: 0
	+native_color: FAKE2 allocation score on 18builder: 0
	+native_color: FAKE2 allocation score on 18node1: 0
	+native_color: FAKE2 allocation score on 18node2: 0
	+native_color: FAKE2 allocation score on remote1: 0
	+native_color: FAKE3 allocation score on 18builder: 0
	+native_color: FAKE3 allocation score on 18node1: 0
	+native_color: FAKE3 allocation score on 18node2: 0
	+native_color: FAKE3 allocation score on remote1: 0
	+native_color: FAKE4 allocation score on 18builder: 0
	+native_color: FAKE4 allocation score on 18node1: 0
	+native_color: FAKE4 allocation score on 18node2: 0
	+native_color: FAKE4 allocation score on remote1: 0
	+native_color: remote1 allocation score on 18builder: -INFINITY
	+native_color: remote1 allocation score on 18node1: 0
	+native_color: remote1 allocation score on 18node2: 0
	+native_color: remote1 allocation score on remote1: -INFINITY
	+native_color: shooter allocation score on 18builder: 0
	+native_color: shooter allocation score on 18node1: 0
	+native_color: shooter allocation score on 18node2: 0
	+native_color: shooter allocation score on remote1: -INFINITY
	diff --git a/pengine/test10/remote-move.summary b/pengine/test10/remote-move.summary
	new file mode 100644
	index 0000000000..4f8b5c2578
	--- /dev/null
	+++ b/pengine/test10/remote-move.summary
	@@ -0,0 +1,40 @@
	+
	+Current cluster status:
	+Online: [ 18builder 18node1 18node2 ]
	+RemoteOnline: [ remote1 ]
	+
	+ shooter (stonith:fence_xvm): Started 18node1
	+ remote1 (ocf::pacemaker:remote): Started 18builder
	+ FAKE1 (ocf::heartbeat:Dummy): Started 18node2
	+ FAKE2 (ocf::heartbeat:Dummy): Started remote1
	+ FAKE3 (ocf::heartbeat:Dummy): Started 18builder
	+ FAKE4 (ocf::heartbeat:Dummy): Started 18node1
	+
	+Transition Summary:
	+ * Move shooter (Started 18node1 -> 18builder)
	+ * Move remote1 (Started 18builder -> 18node1)
	+ * Restart FAKE2 (Started remote1)
	+
	+Executing cluster transition:
	+ * Resource action: shooter stop on 18node1
	+ * Resource action: FAKE2 stop on remote1
	+ * Resource action: shooter start on 18builder
	+ * Resource action: remote1 stop on 18builder
	+ * Pseudo action: all_stopped
	+ * Resource action: shooter monitor=60000 on 18builder
	+ * Resource action: remote1 start on 18node1
	+ * Resource action: FAKE2 start on remote1
	+ * Resource action: FAKE2 monitor=60000 on remote1
	+ * Resource action: remote1 monitor=60000 on 18node1
	+
	+Revised cluster status:
	+Online: [ 18builder 18node1 18node2 ]
	+RemoteOnline: [ remote1 ]
	+
	+ shooter (stonith:fence_xvm): Started 18builder
	+ remote1 (ocf::pacemaker:remote): Started 18node1
	+ FAKE1 (ocf::heartbeat:Dummy): Started 18node2
	+ FAKE2 (ocf::heartbeat:Dummy): Started remote1
	+ FAKE3 (ocf::heartbeat:Dummy): Started 18builder
	+ FAKE4 (ocf::heartbeat:Dummy): Started 18node1
	+
	diff --git a/pengine/test10/remote-move.xml b/pengine/test10/remote-move.xml
	new file mode 100644
	index 0000000000..3f95d94aca
	--- /dev/null
	+++ b/pengine/test10/remote-move.xml
	@@ -0,0 +1,177 @@
	+<cib epoch="14" num_updates="9" admin_epoch="0" validate-with="pacemaker-1.2" cib-last-written="Tue Sep 3 18:02:57 2013" update-origin="18builder" update-client="cibadmin" crm_feature_set="3.0.7" have-quorum="1" dc-uuid="5">
	+ <configuration>
	+ <crm_config>
	+ <cluster_property_set id="cib-bootstrap-options">
	+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.10-ab50afe"/>
	+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
	+ </cluster_property_set>
	+ </crm_config>
	+ <nodes>
	+ <node id="1" uname="18node1"/>
	+ <node id="2" uname="18node2"/>
	+ <node id="5" uname="18builder"/>
	+ </nodes>
	+ <resources>
	+ <primitive class="stonith" id="shooter" type="fence_xvm">
	+ <instance_attributes id="shooter-instance_attributes"/>
	+ <operations>
	+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="remote1" provider="pacemaker" type="remote">
	+ <instance_attributes id="remote1-instance_attributes"/>
	+ <operations>
	+ <op id="remote1-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ <meta_attributes id="remote1-meta_attributes"/>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE1" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE1-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE1-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE2" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE2-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE2-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE3" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE3-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE3-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE4" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE4-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE4-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ </resources>
	+ <constraints>
	+ <rsc_location id="location-remote1-18builder--INFINITY" node="18builder" rsc="remote1" score="-INFINITY"/>
	+ </constraints>
	+ </configuration>
	+ <status>
	+ <node_state id="5" uname="18builder" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
	+ <transient_attributes id="5">
	+ <instance_attributes id="status-5">
	+ <nvpair id="status-5-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ <lrm id="5">
	+ <lrm_resources>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="18:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;18:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="53" rc-code="0" op-status="0" interval="0" last-run="1378249147" last-rc-change="1378249147" exec-time="11" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE3_monitor_60000" operation_key="FAKE3_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="19:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;19:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="56" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249147" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="9:12:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;9:12:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="61" rc-code="7" op-status="0" interval="0" last-run="1378249149" last-rc-change="1378249149" exec-time="65" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="11:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;11:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="16" rc-code="0" op-status="0" interval="0" last-run="1378248545" last-rc-change="1378248545" exec-time="1" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="remote1_last_0" operation_key="remote1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="15:16:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;15:16:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="18" rc-code="0" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="remote1_monitor_60000" operation_key="remote1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="20:17:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;20:17:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="19" rc-code="0" op-status="0" interval="60000" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="22:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;22:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="65" rc-code="0" op-status="0" interval="0" last-run="1378249167" last-rc-change="1378249167" exec-time="31" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="19:16:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;19:16:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="76" rc-code="0" op-status="0" interval="0" last-run="1378249334" last-rc-change="1378249334" exec-time="45" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ </node_state>
	+ <node_state id="1" uname="18node1" crmd="online" crm-debug-origin="do_state_transition" in_ccm="true" join="member" expected="member">
	+ <transient_attributes id="1">
	+ <instance_attributes id="status-1">
	+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ <lrm id="1">
	+ <lrm_resources>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="17:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;17:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="33" rc-code="0" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="15" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE3_last_failure_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="10:10:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;10:10:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="32" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="22:12:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;22:12:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="40" rc-code="0" op-status="0" interval="0" last-run="1378249148" last-rc-change="1378249148" exec-time="36" queue-time="38" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE4_monitor_60000" operation_key="FAKE4_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="23:12:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;23:12:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="44" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249148" exec-time="14" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="12:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;12:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="9" rc-code="0" op-status="0" interval="0" last-run="1378248547" last-rc-change="1378248547" exec-time="19" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="13:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;13:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="12" rc-code="0" op-status="0" interval="60000" last-rc-change="1378248548" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="remote1_last_0" operation_key="remote1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="8:4:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;8:4:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="1" rc-code="7" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="11:8:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;11:8:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="20" rc-code="0" op-status="0" interval="0" last-run="1378249142" last-rc-change="1378249142" exec-time="18" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE1_last_failure_0" operation_key="FAKE1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="8:7:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;8:7:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="17" rc-code="0" op-status="0" interval="0" last-run="1378249142" last-rc-change="1378249142" exec-time="50" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="9:9:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;9:9:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="25" rc-code="7" op-status="0" interval="0" last-run="1378249144" last-rc-change="1378249144" exec-time="28" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ </node_state>
	+ <node_state id="2" uname="18node2" crmd="online" crm-debug-origin="do_state_transition" in_ccm="true" join="member" expected="member">
	+ <transient_attributes id="2">
	+ <instance_attributes id="status-2">
	+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ <lrm id="2">
	+ <lrm_resources>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="12:10:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;12:10:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="27" rc-code="7" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="35" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="13:12:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;13:12:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="32" rc-code="7" op-status="0" interval="0" last-run="1378249148" last-rc-change="1378249148" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="8:3:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;8:3:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1378248530" last-rc-change="1378248530" exec-time="980" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="remote1_last_0" operation_key="remote1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="10:4:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;10:4:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="1" rc-code="7" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="23:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;23:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="40" rc-code="0" op-status="0" interval="0" last-run="1378249166" last-rc-change="1378249166" exec-time="28" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE1_monitor_60000" operation_key="FAKE1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="24:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;24:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="43" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249166" exec-time="16" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.7" transition-key="25:14:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;25:14:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="36" rc-code="0" op-status="0" interval="0" last-run="1378249166" last-rc-change="1378249166" exec-time="30" queue-time="1" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ </node_state>
	+ <node_state remote_node="true" id="remote1" uname="remote1" crm-debug-origin="do_update_resource">
	+ <lrm id="remote1">
	+ <lrm_resources>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="12:17:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;12:17:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="48" rc-code="7" op-status="0" interval="0" last-run="1378231334" last-rc-change="1378231334" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="23:17:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;23:17:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="61" rc-code="0" op-status="0" interval="0" last-run="1378231335" last-rc-change="1378231335" exec-time="14" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE2_monitor_60000" operation_key="FAKE2_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="24:17:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;24:17:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="64" rc-code="0" op-status="0" interval="60000" last-rc-change="1378231335" exec-time="19" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="14:17:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;14:17:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="52" rc-code="7" op-status="0" interval="0" last-run="1378231334" last-rc-change="1378231334" exec-time="13" queue-time="1" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="15:17:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;15:17:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="54" rc-code="7" op-status="0" interval="0" last-run="1378231334" last-rc-change="1378231334" exec-time="7" queue-time="1" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ <transient_attributes id="remote1">
	+ <instance_attributes id="status-remote1">
	+ <nvpair id="status-remote1-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ </node_state>
	+ </status>
	+</cib>
	diff --git a/pengine/test10/remote-startup-probes.dot b/pengine/test10/remote-startup-probes.dot
	new file mode 100644
	index 0000000000..13065d9580
	--- /dev/null
	+++ b/pengine/test10/remote-startup-probes.dot
	@@ -0,0 +1,39 @@
	+ digraph "g" {
	+"FAKE1_monitor_0 remote1" -> "probe_complete remote1" [ style = bold]
	+"FAKE1_monitor_0 remote1" [ style=bold color="green" fontcolor="black"]
	+"FAKE1_monitor_60000 18node2" [ style=bold color="green" fontcolor="black"]
	+"FAKE1_start_0 18node2" -> "FAKE1_monitor_60000 18node2" [ style = bold]
	+"FAKE1_start_0 18node2" [ style=bold color="green" fontcolor="black"]
	+"FAKE1_stop_0 18builder" -> "FAKE1_start_0 18node2" [ style = bold]
	+"FAKE1_stop_0 18builder" -> "all_stopped" [ style = bold]
	+"FAKE1_stop_0 18builder" [ style=bold color="green" fontcolor="black"]
	+"FAKE2_monitor_0 remote1" -> "probe_complete remote1" [ style = bold]
	+"FAKE2_monitor_0 remote1" [ style=bold color="green" fontcolor="black"]
	+"FAKE2_monitor_60000 remote1" [ style=bold color="green" fontcolor="black"]
	+"FAKE2_start_0 remote1" -> "FAKE2_monitor_60000 remote1" [ style = bold]
	+"FAKE2_start_0 remote1" [ style=bold color="green" fontcolor="black"]
	+"FAKE2_stop_0 18node2" -> "FAKE2_start_0 remote1" [ style = bold]
	+"FAKE2_stop_0 18node2" -> "all_stopped" [ style = bold]
	+"FAKE2_stop_0 18node2" [ style=bold color="green" fontcolor="black"]
	+"FAKE3_monitor_0 remote1" -> "probe_complete remote1" [ style = bold]
	+"FAKE3_monitor_0 remote1" [ style=bold color="green" fontcolor="black"]
	+"FAKE4_monitor_0 remote1" -> "probe_complete remote1" [ style = bold]
	+"FAKE4_monitor_0 remote1" [ style=bold color="green" fontcolor="black"]
	+"all_stopped" [ style=bold color="green" fontcolor="orange"]
	+"probe_complete remote1" -> "probe_complete" [ style = bold]
	+"probe_complete remote1" [ style=bold color="green" fontcolor="black"]
	+"probe_complete" -> "FAKE1_start_0 18node2" [ style = bold]
	+"probe_complete" -> "FAKE1_stop_0 18builder" [ style = bold]
	+"probe_complete" -> "FAKE2_start_0 remote1" [ style = bold]
	+"probe_complete" -> "FAKE2_stop_0 18node2" [ style = bold]
	+"probe_complete" [ style=bold color="green" fontcolor="orange"]
	+"remote1_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"]
	+"remote1_start_0 18builder" -> "FAKE1_monitor_0 remote1" [ style = bold]
	+"remote1_start_0 18builder" -> "FAKE2_monitor_0 remote1" [ style = bold]
	+"remote1_start_0 18builder" -> "FAKE2_monitor_60000 remote1" [ style = bold]
	+"remote1_start_0 18builder" -> "FAKE2_start_0 remote1" [ style = bold]
	+"remote1_start_0 18builder" -> "FAKE3_monitor_0 remote1" [ style = bold]
	+"remote1_start_0 18builder" -> "FAKE4_monitor_0 remote1" [ style = bold]
	+"remote1_start_0 18builder" -> "remote1_monitor_60000 18builder" [ style = bold]
	+"remote1_start_0 18builder" [ style=bold color="green" fontcolor="black"]
	+}
	diff --git a/pengine/test10/remote-startup-probes.exp b/pengine/test10/remote-startup-probes.exp
	new file mode 100644
	index 0000000000..3af16fe5f3
	--- /dev/null
	+++ b/pengine/test10/remote-startup-probes.exp
	@@ -0,0 +1,214 @@
	+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" batch-limit="30" transition_id="0">
	+ <synapse id="0">
	+ <action_set>
	+ <rsc_op id="20" operation="monitor" operation_key="remote1_monitor_60000" on_node="18builder" on_node_uuid="5">
	+ <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
	+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="19" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="1">
	+ <action_set>
	+ <rsc_op id="19" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5">
	+ <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="2">
	+ <action_set>
	+ <rsc_op id="23" operation="monitor" operation_key="FAKE1_monitor_60000" on_node="18node2" on_node_uuid="2">
	+ <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="22" operation="start" operation_key="FAKE1_start_0" on_node="18node2" on_node_uuid="2"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="3">
	+ <action_set>
	+ <rsc_op id="22" operation="start" operation_key="FAKE1_start_0" on_node="18node2" on_node_uuid="2">
	+ <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="7" operation="probe_complete" operation_key="probe_complete"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="21" operation="stop" operation_key="FAKE1_stop_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="4">
	+ <action_set>
	+ <rsc_op id="21" operation="stop" operation_key="FAKE1_stop_0" on_node="18builder" on_node_uuid="5">
	+ <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="7" operation="probe_complete" operation_key="probe_complete"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="5">
	+ <action_set>
	+ <rsc_op id="13" operation="monitor" operation_key="FAKE1_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
	+ <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="19" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="6">
	+ <action_set>
	+ <rsc_op id="26" operation="monitor" operation_key="FAKE2_monitor_60000" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
	+ <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="19" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="25" operation="start" operation_key="FAKE2_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="7">
	+ <action_set>
	+ <rsc_op id="25" operation="start" operation_key="FAKE2_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
	+ <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="7" operation="probe_complete" operation_key="probe_complete"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="19" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="24" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="8">
	+ <action_set>
	+ <rsc_op id="24" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2">
	+ <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="7" operation="probe_complete" operation_key="probe_complete"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="9">
	+ <action_set>
	+ <rsc_op id="14" operation="monitor" operation_key="FAKE2_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
	+ <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="19" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="10">
	+ <action_set>
	+ <rsc_op id="15" operation="monitor" operation_key="FAKE3_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
	+ <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="19" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="11">
	+ <action_set>
	+ <rsc_op id="16" operation="monitor" operation_key="FAKE4_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
	+ <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="19" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="12" priority="1000000">
	+ <action_set>
	+ <rsc_op id="12" operation="probe_complete" operation_key="probe_complete" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
	+ <attributes CRM_meta_op_no_wait="true" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="13" operation="monitor" operation_key="FAKE1_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="14" operation="monitor" operation_key="FAKE2_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="15" operation="monitor" operation_key="FAKE3_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="16" operation="monitor" operation_key="FAKE4_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="13">
	+ <action_set>
	+ <pseudo_event id="7" operation="probe_complete" operation_key="probe_complete">
	+ <attributes crm_feature_set="3.0.7"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="12" operation="probe_complete" operation_key="probe_complete" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="14">
	+ <action_set>
	+ <pseudo_event id="6" operation="all_stopped" operation_key="all_stopped">
	+ <attributes crm_feature_set="3.0.7"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="21" operation="stop" operation_key="FAKE1_stop_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="24" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+</transition_graph>
	diff --git a/pengine/test10/remote-startup-probes.scores b/pengine/test10/remote-startup-probes.scores
	new file mode 100644
	index 0000000000..27d226369b
	--- /dev/null
	+++ b/pengine/test10/remote-startup-probes.scores
	@@ -0,0 +1,25 @@
	+Allocation scores:
	+native_color: FAKE1 allocation score on 18builder: 0
	+native_color: FAKE1 allocation score on 18node1: 0
	+native_color: FAKE1 allocation score on 18node2: 0
	+native_color: FAKE1 allocation score on remote1: 0
	+native_color: FAKE2 allocation score on 18builder: 0
	+native_color: FAKE2 allocation score on 18node1: 0
	+native_color: FAKE2 allocation score on 18node2: 0
	+native_color: FAKE2 allocation score on remote1: 0
	+native_color: FAKE3 allocation score on 18builder: 0
	+native_color: FAKE3 allocation score on 18node1: 0
	+native_color: FAKE3 allocation score on 18node2: 0
	+native_color: FAKE3 allocation score on remote1: 0
	+native_color: FAKE4 allocation score on 18builder: 0
	+native_color: FAKE4 allocation score on 18node1: 0
	+native_color: FAKE4 allocation score on 18node2: 0
	+native_color: FAKE4 allocation score on remote1: 0
	+native_color: remote1 allocation score on 18builder: 0
	+native_color: remote1 allocation score on 18node1: 0
	+native_color: remote1 allocation score on 18node2: 0
	+native_color: remote1 allocation score on remote1: -INFINITY
	+native_color: shooter allocation score on 18builder: 0
	+native_color: shooter allocation score on 18node1: 0
	+native_color: shooter allocation score on 18node2: 0
	+native_color: shooter allocation score on remote1: -INFINITY
	diff --git a/pengine/test10/remote-startup-probes.summary b/pengine/test10/remote-startup-probes.summary
	new file mode 100644
	index 0000000000..14335ddf01
	--- /dev/null
	+++ b/pengine/test10/remote-startup-probes.summary
	@@ -0,0 +1,44 @@
	+
	+Current cluster status:
	+Online: [ 18builder 18node1 18node2 ]
	+RemoteOFFLINE: [ remote1 ]
	+
	+ shooter (stonith:fence_xvm): Started 18node1
	+ remote1 (ocf::pacemaker:remote): Stopped
	+ FAKE1 (ocf::heartbeat:Dummy): Started 18builder
	+ FAKE2 (ocf::heartbeat:Dummy): Started 18node2
	+ FAKE3 (ocf::heartbeat:Dummy): Started 18builder
	+ FAKE4 (ocf::heartbeat:Dummy): Started 18node1
	+
	+Transition Summary:
	+ * Start remote1 (18builder)
	+ * Move FAKE1 (Started 18builder -> 18node2)
	+ * Move FAKE2 (Started 18node2 -> remote1)
	+
	+Executing cluster transition:
	+ * Resource action: remote1 start on 18builder
	+ * Resource action: FAKE1 monitor on remote1
	+ * Resource action: FAKE2 monitor on remote1
	+ * Resource action: FAKE3 monitor on remote1
	+ * Resource action: FAKE4 monitor on remote1
	+ * Pseudo action: probe_complete
	+ * Resource action: remote1 monitor=60000 on 18builder
	+ * Resource action: FAKE1 stop on 18builder
	+ * Resource action: FAKE2 stop on 18node2
	+ * Pseudo action: all_stopped
	+ * Resource action: FAKE1 start on 18node2
	+ * Resource action: FAKE2 start on remote1
	+ * Resource action: FAKE1 monitor=60000 on 18node2
	+ * Resource action: FAKE2 monitor=60000 on remote1
	+
	+Revised cluster status:
	+Online: [ 18builder 18node1 18node2 ]
	+RemoteOnline: [ remote1 ]
	+
	+ shooter (stonith:fence_xvm): Started 18node1
	+ remote1 (ocf::pacemaker:remote): Started 18builder
	+ FAKE1 (ocf::heartbeat:Dummy): Started 18node2
	+ FAKE2 (ocf::heartbeat:Dummy): Started remote1
	+ FAKE3 (ocf::heartbeat:Dummy): Started 18builder
	+ FAKE4 (ocf::heartbeat:Dummy): Started 18node1
	+
	diff --git a/pengine/test10/remote-startup-probes.xml b/pengine/test10/remote-startup-probes.xml
	new file mode 100644
	index 0000000000..70606217d7
	--- /dev/null
	+++ b/pengine/test10/remote-startup-probes.xml
	@@ -0,0 +1,160 @@
	+<cib epoch="11" num_updates="1" admin_epoch="0" validate-with="pacemaker-1.2" cib-last-written="Tue Sep 3 17:59:24 2013" update-origin="18builder" update-client="crm_resource" crm_feature_set="3.0.7" have-quorum="1" dc-uuid="5">
	+ <configuration>
	+ <crm_config>
	+ <cluster_property_set id="cib-bootstrap-options">
	+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.10-ab50afe"/>
	+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
	+ </cluster_property_set>
	+ </crm_config>
	+ <nodes>
	+ <node id="1" uname="18node1"/>
	+ <node id="2" uname="18node2"/>
	+ <node id="5" uname="18builder"/>
	+ </nodes>
	+ <resources>
	+ <primitive class="stonith" id="shooter" type="fence_xvm">
	+ <instance_attributes id="shooter-instance_attributes"/>
	+ <operations>
	+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="remote1" provider="pacemaker" type="remote">
	+ <instance_attributes id="remote1-instance_attributes"/>
	+ <operations>
	+ <op id="remote1-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ <meta_attributes id="remote1-meta_attributes"/>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE1" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE1-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE1-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE2" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE2-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE2-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE3" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE3-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE3-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="FAKE4" provider="heartbeat" type="Dummy">
	+ <instance_attributes id="FAKE4-instance_attributes"/>
	+ <operations>
	+ <op id="FAKE4-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ </resources>
	+ <constraints/>
	+ </configuration>
	+ <status>
	+ <node_state id="5" uname="18builder" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
	+ <transient_attributes id="5">
	+ <instance_attributes id="status-5">
	+ <nvpair id="status-5-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ <lrm id="5">
	+ <lrm_resources>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="11:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;11:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="16" rc-code="0" op-status="0" interval="0" last-run="1378248545" last-rc-change="1378248545" exec-time="1" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="10:3:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;10:3:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="11" rc-code="0" op-status="0" interval="60000" last-rc-change="1378248534" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="remote1_last_0" operation_key="remote1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="12:6:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;12:6:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="13" rc-code="0" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="remote1_monitor_60000" operation_key="remote1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="13:5:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;13:5:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="3" rc-code="0" op-status="0" interval="60000" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="12:8:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;12:8:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="31" rc-code="0" op-status="0" interval="0" last-run="1378249143" last-rc-change="1378249143" exec-time="14" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE1_monitor_60000" operation_key="FAKE1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="13:8:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;13:8:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="34" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249143" exec-time="9" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="7:9:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;7:9:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="39" rc-code="7" op-status="0" interval="0" last-run="1378249145" last-rc-change="1378249145" exec-time="61" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="18:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;18:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="53" rc-code="0" op-status="0" interval="0" last-run="1378249147" last-rc-change="1378249147" exec-time="11" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE3_monitor_60000" operation_key="FAKE3_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="19:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;19:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="56" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249147" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="9:12:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;9:12:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="61" rc-code="7" op-status="0" interval="0" last-run="1378249149" last-rc-change="1378249149" exec-time="65" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ </node_state>
	+ <node_state id="1" uname="18node1" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
	+ <lrm id="1">
	+ <lrm_resources>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="12:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;12:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="9" rc-code="0" op-status="0" interval="0" last-run="1378248547" last-rc-change="1378248547" exec-time="19" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="13:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;13:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="12" rc-code="0" op-status="0" interval="60000" last-rc-change="1378248548" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="remote1_last_0" operation_key="remote1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="8:4:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;8:4:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="1" rc-code="7" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_failure_0" operation_key="FAKE1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="8:7:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;8:7:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="17" rc-code="0" op-status="0" interval="0" last-run="1378249142" last-rc-change="1378249142" exec-time="50" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="11:8:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;11:8:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="20" rc-code="0" op-status="0" interval="0" last-run="1378249142" last-rc-change="1378249142" exec-time="18" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="9:9:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;9:9:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="25" rc-code="7" op-status="0" interval="0" last-run="1378249144" last-rc-change="1378249144" exec-time="28" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_failure_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="10:10:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;10:10:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="32" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="17:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;17:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="33" rc-code="0" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="15" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="22:12:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;22:12:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="40" rc-code="0" op-status="0" interval="0" last-run="1378249148" last-rc-change="1378249148" exec-time="36" queue-time="38" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE4_monitor_60000" operation_key="FAKE4_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="23:12:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;23:12:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="44" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249148" exec-time="14" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ <transient_attributes id="1">
	+ <instance_attributes id="status-1">
	+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ </node_state>
	+ <node_state id="2" uname="18node2" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
	+ <lrm id="2">
	+ <lrm_resources>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="8:3:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;8:3:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1378248530" last-rc-change="1378248530" exec-time="980" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="remote1_last_0" operation_key="remote1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="10:4:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;10:4:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="1" rc-code="7" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="10:7:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;10:7:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="11" rc-code="7" op-status="0" interval="0" last-run="1378249142" last-rc-change="1378249142" exec-time="42" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="16:9:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;16:9:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1378249144" last-rc-change="1378249144" exec-time="18" queue-time="9" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="FAKE2_monitor_60000" operation_key="FAKE2_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="17:9:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;17:9:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="22" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249144" exec-time="20" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="12:10:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;12:10:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="27" rc-code="7" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="35" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="13:12:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;13:12:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="32" rc-code="7" op-status="0" interval="0" last-run="1378249148" last-rc-change="1378249148" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ <transient_attributes id="2">
	+ <instance_attributes id="status-2">
	+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ </node_state>
	+ <node_state remote_node="true" id="remote1" uname="remote1" crm-debug-origin="remote_init_cib_status">
	+ <transient_attributes id="remote1">
	+ <instance_attributes id="status-remote1">
	+ <nvpair id="status-remote1-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ </node_state>
	+ </status>
	+</cib>
	diff --git a/pengine/test10/remote-startup.dot b/pengine/test10/remote-startup.dot
	new file mode 100644
	index 0000000000..50e53319f8
	--- /dev/null
	+++ b/pengine/test10/remote-startup.dot
	@@ -0,0 +1,41 @@
	+ digraph "g" {
	+"all_stopped" [ style=bold color="green" fontcolor="orange"]
	+"fake_monitor_0 18builder" -> "probe_complete 18builder" [ style = bold]
	+"fake_monitor_0 18builder" [ style=bold color="green" fontcolor="black"]
	+"fake_monitor_0 18node1" -> "probe_complete 18node1" [ style = bold]
	+"fake_monitor_0 18node1" [ style=bold color="green" fontcolor="black"]
	+"fake_monitor_0 18node2" -> "probe_complete 18node2" [ style = bold]
	+"fake_monitor_0 18node2" [ style=bold color="green" fontcolor="black"]
	+"fake_monitor_0 remote1" -> "probe_complete remote1" [ style = bold]
	+"fake_monitor_0 remote1" [ style=bold color="green" fontcolor="black"]
	+"fake_start_0 18node2" [ style=bold color="green" fontcolor="black"]
	+"probe_complete 18builder" -> "probe_nodes_complete" [ style = bold]
	+"probe_complete 18builder" [ style=bold color="green" fontcolor="black"]
	+"probe_complete 18node1" -> "probe_nodes_complete" [ style = bold]
	+"probe_complete 18node1" [ style=bold color="green" fontcolor="black"]
	+"probe_complete 18node2" -> "probe_nodes_complete" [ style = bold]
	+"probe_complete 18node2" [ style=bold color="green" fontcolor="black"]
	+"probe_complete remote1" -> "probe_complete" [ style = bold]
	+"probe_complete remote1" [ style=bold color="green" fontcolor="black"]
	+"probe_complete" -> "fake_start_0 18node2" [ style = bold]
	+"probe_complete" -> "shooter_stop_0 18builder" [ style = bold]
	+"probe_complete" [ style=bold color="green" fontcolor="orange"]
	+"probe_nodes_complete" -> "remote1_start_0 18builder" [ style = bold]
	+"probe_nodes_complete" [ style=bold color="green" fontcolor="orange"]
	+"remote1_monitor_0 18builder" -> "probe_complete 18builder" [ style = bold]
	+"remote1_monitor_0 18builder" [ style=bold color="green" fontcolor="black"]
	+"remote1_monitor_0 18node1" -> "probe_complete 18node1" [ style = bold]
	+"remote1_monitor_0 18node1" [ style=bold color="green" fontcolor="black"]
	+"remote1_monitor_0 18node2" -> "probe_complete 18node2" [ style = bold]
	+"remote1_monitor_0 18node2" [ style=bold color="green" fontcolor="black"]
	+"remote1_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"]
	+"remote1_start_0 18builder" -> "fake_monitor_0 remote1" [ style = bold]
	+"remote1_start_0 18builder" -> "remote1_monitor_60000 18builder" [ style = bold]
	+"remote1_start_0 18builder" [ style=bold color="green" fontcolor="black"]
	+"shooter_monitor_60000 18node1" [ style=bold color="green" fontcolor="black"]
	+"shooter_start_0 18node1" -> "shooter_monitor_60000 18node1" [ style = bold]
	+"shooter_start_0 18node1" [ style=bold color="green" fontcolor="black"]
	+"shooter_stop_0 18builder" -> "all_stopped" [ style = bold]
	+"shooter_stop_0 18builder" -> "shooter_start_0 18node1" [ style = bold]
	+"shooter_stop_0 18builder" [ style=bold color="green" fontcolor="black"]
	+}
	diff --git a/pengine/test10/remote-startup.exp b/pengine/test10/remote-startup.exp
	new file mode 100644
	index 0000000000..34476e4083
	--- /dev/null
	+++ b/pengine/test10/remote-startup.exp
	@@ -0,0 +1,246 @@
	+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" batch-limit="30" transition_id="0">
	+ <synapse id="0">
	+ <action_set>
	+ <rsc_op id="18" operation="monitor" operation_key="shooter_monitor_60000" on_node="18node1" on_node_uuid="1">
	+ <primitive id="shooter" class="stonith" type="fence_xvm"/>
	+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="17" operation="start" operation_key="shooter_start_0" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="1">
	+ <action_set>
	+ <rsc_op id="17" operation="start" operation_key="shooter_start_0" on_node="18node1" on_node_uuid="1">
	+ <primitive id="shooter" class="stonith" type="fence_xvm"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="16" operation="stop" operation_key="shooter_stop_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="2">
	+ <action_set>
	+ <rsc_op id="16" operation="stop" operation_key="shooter_stop_0" on_node="18builder" on_node_uuid="5">
	+ <primitive id="shooter" class="stonith" type="fence_xvm"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="3" operation="probe_complete" operation_key="probe_complete"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="3">
	+ <action_set>
	+ <rsc_op id="19" operation="start" operation_key="fake_start_0" on_node="18node2" on_node_uuid="2">
	+ <primitive id="fake" class="ocf" provider="pacemaker" type="Dummy"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="3" operation="probe_complete" operation_key="probe_complete"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="4">
	+ <action_set>
	+ <rsc_op id="15" operation="monitor" operation_key="fake_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
	+ <primitive id="fake" class="ocf" provider="pacemaker" type="Dummy"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="20" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="5">
	+ <action_set>
	+ <rsc_op id="12" operation="monitor" operation_key="fake_monitor_0" on_node="18node2" on_node_uuid="2">
	+ <primitive id="fake" class="ocf" provider="pacemaker" type="Dummy"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="6">
	+ <action_set>
	+ <rsc_op id="9" operation="monitor" operation_key="fake_monitor_0" on_node="18node1" on_node_uuid="1">
	+ <primitive id="fake" class="ocf" provider="pacemaker" type="Dummy"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="7">
	+ <action_set>
	+ <rsc_op id="6" operation="monitor" operation_key="fake_monitor_0" on_node="18builder" on_node_uuid="5">
	+ <primitive id="fake" class="ocf" provider="pacemaker" type="Dummy"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="8">
	+ <action_set>
	+ <rsc_op id="21" operation="monitor" operation_key="remote1_monitor_60000" on_node="18builder" on_node_uuid="5">
	+ <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
	+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="20" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="9">
	+ <action_set>
	+ <rsc_op id="20" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5">
	+ <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
	+ <attributes CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="4" operation="probe_nodes_complete" operation_key="probe_nodes_complete"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="10">
	+ <action_set>
	+ <rsc_op id="13" operation="monitor" operation_key="remote1_monitor_0" on_node="18node2" on_node_uuid="2">
	+ <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="11">
	+ <action_set>
	+ <rsc_op id="10" operation="monitor" operation_key="remote1_monitor_0" on_node="18node1" on_node_uuid="1">
	+ <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="12">
	+ <action_set>
	+ <rsc_op id="7" operation="monitor" operation_key="remote1_monitor_0" on_node="18builder" on_node_uuid="5">
	+ <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
	+ <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="13" priority="1000000">
	+ <action_set>
	+ <rsc_op id="14" operation="probe_complete" operation_key="probe_complete" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
	+ <attributes CRM_meta_op_no_wait="true" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="15" operation="monitor" operation_key="fake_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="14" priority="1000000">
	+ <action_set>
	+ <rsc_op id="11" operation="probe_complete" operation_key="probe_complete" on_node="18node2" on_node_uuid="2">
	+ <attributes CRM_meta_op_no_wait="true" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="12" operation="monitor" operation_key="fake_monitor_0" on_node="18node2" on_node_uuid="2"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="13" operation="monitor" operation_key="remote1_monitor_0" on_node="18node2" on_node_uuid="2"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="15" priority="1000000">
	+ <action_set>
	+ <rsc_op id="8" operation="probe_complete" operation_key="probe_complete" on_node="18node1" on_node_uuid="1">
	+ <attributes CRM_meta_op_no_wait="true" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="9" operation="monitor" operation_key="fake_monitor_0" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="10" operation="monitor" operation_key="remote1_monitor_0" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="16" priority="1000000">
	+ <action_set>
	+ <rsc_op id="5" operation="probe_complete" operation_key="probe_complete" on_node="18builder" on_node_uuid="5">
	+ <attributes CRM_meta_op_no_wait="true" crm_feature_set="3.0.7"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="6" operation="monitor" operation_key="fake_monitor_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="7" operation="monitor" operation_key="remote1_monitor_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="17">
	+ <action_set>
	+ <pseudo_event id="4" operation="probe_nodes_complete" operation_key="probe_nodes_complete">
	+ <attributes crm_feature_set="3.0.7"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="5" operation="probe_complete" operation_key="probe_complete" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="8" operation="probe_complete" operation_key="probe_complete" on_node="18node1" on_node_uuid="1"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="11" operation="probe_complete" operation_key="probe_complete" on_node="18node2" on_node_uuid="2"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="18">
	+ <action_set>
	+ <pseudo_event id="3" operation="probe_complete" operation_key="probe_complete">
	+ <attributes crm_feature_set="3.0.7"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="14" operation="probe_complete" operation_key="probe_complete" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="19">
	+ <action_set>
	+ <pseudo_event id="2" operation="all_stopped" operation_key="all_stopped">
	+ <attributes crm_feature_set="3.0.7"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="16" operation="stop" operation_key="shooter_stop_0" on_node="18builder" on_node_uuid="5"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+</transition_graph>
	diff --git a/pengine/test10/remote-startup.scores b/pengine/test10/remote-startup.scores
	new file mode 100644
	index 0000000000..06c32ea75f
	--- /dev/null
	+++ b/pengine/test10/remote-startup.scores
	@@ -0,0 +1,13 @@
	+Allocation scores:
	+native_color: fake allocation score on 18builder: 0
	+native_color: fake allocation score on 18node1: 0
	+native_color: fake allocation score on 18node2: 0
	+native_color: fake allocation score on remote1: 0
	+native_color: remote1 allocation score on 18builder: 0
	+native_color: remote1 allocation score on 18node1: 0
	+native_color: remote1 allocation score on 18node2: 0
	+native_color: remote1 allocation score on remote1: -INFINITY
	+native_color: shooter allocation score on 18builder: 0
	+native_color: shooter allocation score on 18node1: 0
	+native_color: shooter allocation score on 18node2: 0
	+native_color: shooter allocation score on remote1: -INFINITY
	diff --git a/pengine/test10/remote-startup.summary b/pengine/test10/remote-startup.summary
	new file mode 100644
	index 0000000000..19f18edd30
	--- /dev/null
	+++ b/pengine/test10/remote-startup.summary
	@@ -0,0 +1,40 @@
	+
	+Current cluster status:
	+RemoteNode remote1: UNCLEAN (offline)
	+Online: [ 18builder 18node1 18node2 ]
	+
	+ shooter (stonith:fence_xvm): Started 18builder
	+ fake (ocf::pacemaker:Dummy): Stopped
	+ remote1 (ocf::pacemaker:remote): Stopped
	+
	+Transition Summary:
	+ * Move shooter (Started 18builder -> 18node1)
	+ * Start fake (18node2)
	+ * Start remote1 (18builder)
	+
	+Executing cluster transition:
	+ * Resource action: fake monitor on 18node2
	+ * Resource action: fake monitor on 18node1
	+ * Resource action: fake monitor on 18builder
	+ * Resource action: remote1 monitor on 18node2
	+ * Resource action: remote1 monitor on 18node1
	+ * Resource action: remote1 monitor on 18builder
	+ * Pseudo action: probe_nodes_complete
	+ * Resource action: remote1 start on 18builder
	+ * Resource action: fake monitor on remote1
	+ * Resource action: remote1 monitor=60000 on 18builder
	+ * Pseudo action: probe_complete
	+ * Resource action: shooter stop on 18builder
	+ * Resource action: fake start on 18node2
	+ * Pseudo action: all_stopped
	+ * Resource action: shooter start on 18node1
	+ * Resource action: shooter monitor=60000 on 18node1
	+
	+Revised cluster status:
	+Online: [ 18builder 18node1 18node2 ]
	+RemoteOnline: [ remote1 ]
	+
	+ shooter (stonith:fence_xvm): Started 18node1
	+ fake (ocf::pacemaker:Dummy): Started 18node2
	+ remote1 (ocf::pacemaker:remote): Started 18builder
	+
	diff --git a/pengine/test10/remote-startup.xml b/pengine/test10/remote-startup.xml
	new file mode 100644
	index 0000000000..9ff27f9335
	--- /dev/null
	+++ b/pengine/test10/remote-startup.xml
	@@ -0,0 +1,76 @@
	+<cib epoch="5" num_updates="1" admin_epoch="0" validate-with="pacemaker-1.2" cib-last-written="Tue Sep 3 17:49:05 2013" update-origin="18builder" update-client="cibadmin" crm_feature_set="3.0.7" have-quorum="1" dc-uuid="5">
	+ <configuration>
	+ <crm_config>
	+ <cluster_property_set id="cib-bootstrap-options">
	+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.10-ab50afe"/>
	+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
	+ </cluster_property_set>
	+ </crm_config>
	+ <nodes>
	+ <node id="1" uname="18node1"/>
	+ <node id="2" uname="18node2"/>
	+ <node id="5" uname="18builder"/>
	+ </nodes>
	+ <resources>
	+ <primitive class="stonith" id="shooter" type="fence_xvm">
	+ <instance_attributes id="shooter-instance_attributes"/>
	+ <operations>
	+ <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="fake" provider="pacemaker" type="Dummy"/>
	+ <primitive class="ocf" id="remote1" provider="pacemaker" type="remote">
	+ <instance_attributes id="remote1-instance_attributes"/>
	+ <operations>
	+ <op id="remote1-monitor-interval-60s" interval="60s" name="monitor"/>
	+ </operations>
	+ </primitive>
	+ </resources>
	+ <constraints/>
	+ </configuration>
	+ <status>
	+ <node_state id="5" uname="18builder" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
	+ <transient_attributes id="5">
	+ <instance_attributes id="status-5">
	+ <nvpair id="status-5-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ <lrm id="5">
	+ <lrm_resources>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="9:3:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;9:3:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="8" rc-code="0" op-status="0" interval="0" last-run="1378248533" last-rc-change="1378248533" exec-time="1034" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="10:3:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;10:3:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="11" rc-code="0" op-status="0" interval="60000" last-rc-change="1378248534" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ </node_state>
	+ <node_state id="1" uname="18node1" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
	+ <lrm id="1">
	+ <lrm_resources>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="6:3:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;6:3:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1378248530" last-rc-change="1378248530" exec-time="993" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ <transient_attributes id="1">
	+ <instance_attributes id="status-1">
	+ <nvpair id="status-1-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ </node_state>
	+ <node_state id="2" uname="18node2" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
	+ <lrm id="2">
	+ <lrm_resources>
	+ <lrm_resource id="shooter" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="8:3:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;8:3:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1378248530" last-rc-change="1378248530" exec-time="980" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ <transient_attributes id="2">
	+ <instance_attributes id="status-2">
	+ <nvpair id="status-2-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ </node_state>
	+ </status>
	+</cib>

File Metadata

Mime Type: text/x-diff
Expires: Sat, Oct 25, 2:12 AM (1 d, 16 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2619974
Default Alt Text: (194 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions