No OneTemporary
Actions

Size

221 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/cts/CTS.py b/cts/CTS.py
	index 4c7448cbea..4f5ed5557a 100644
	--- a/cts/CTS.py
	+++ b/cts/CTS.py
	@@ -1,1002 +1,1008 @@
	'''CTS: Cluster Testing System: Main module

	Classes related to testing high-availability clusters...
	'''

	__copyright__ = '''
	Copyright (C) 2000, 2001 Alan Robertson <alanr@unix.sh>
	Licensed under the GNU GPL.
	'''

	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public License
	# as published by the Free Software Foundation; either version 2
	# of the License, or (at your option) any later version.
	#
	# This program is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	# GNU General Public License for more details.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write to the Free Software
	# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

	import string, sys, time, re, os, traceback

	from UserDict import UserDict

	from cts.CTSvars import *
	from cts.logging import LogFactory
	from cts.watcher import LogWatcher
	from cts.remote import RemoteFactory
	from cts.environment import EnvFactory
	from cts.patterns import PatternSelector

	has_log_stats = {}
	log_stats_bin = CTSvars.CRM_DAEMON_DIR + "/cts_log_stats.sh"
	log_stats = """
	#!/bin/bash
	# Tool for generating system load reports while CTS runs

	trap "" 1

	f=$1; shift
	action=$1; shift
	base=`basename $0`

	if [ ! -e $f ]; then
	echo "Time, Load 1, Load 5, Load 15, Test Marker" > $f
	fi

	function killpid() {
	if [ -e $f.pid ]; then
	kill -9 `cat $f.pid`
	rm -f $f.pid
	fi
	}

	function status() {
	if [ -e $f.pid ]; then
	kill -0 `cat $f.pid`
	return $?
	else
	return 1
	fi
	}

	function start() {
	# Is it already running?
	if
	status
	then
	return
	fi

	echo Active as $$
	echo $$ > $f.pid

	while [ 1 = 1 ]; do
	uptime \| sed s/up.*:/,/ \| tr '\\n' ',' >> $f
	#top -b -c -n1 \| grep -e usr/libexec/pacemaker \| grep -v -e grep -e python \| head -n 1 \| sed s@/usr/libexec/pacemaker/@@ \| awk '{print " 0, "$9", "$10", "$12}' \| tr '\\n' ',' >> $f
	echo 0 >> $f
	sleep 5
	done
	}

	case $action in
	start)
	start
	;;
	start-bg\|bg)
	# Use c --ssh -- ./stats.sh file start-bg
	nohup $0 $f start >/dev/null 2>&1 </dev/null &
	;;
	stop)
	killpid
	;;
	delete)
	killpid
	rm -f $f
	;;
	mark)
	uptime \| sed s/up.*:/,/ \| tr '\\n' ',' >> $f
	echo " $*" >> $f
	start
	;;
	*)
	echo "Unknown action: $action."
	;;
	esac
	"""

	class CtsLab:
	'''This class defines the Lab Environment for the Cluster Test System.
	It defines those things which are expected to change from test
	environment to test environment for the same cluster manager.

	It is where you define the set of nodes that are in your test lab
	what kind of reset mechanism you use, etc.

	This class is derived from a UserDict because we hold many
	different parameters of different kinds, and this provides
	provide a uniform and extensible interface useful for any kind of
	communication between the user/administrator/tester and CTS.

	At this point in time, it is the intent of this class to model static
	configuration and/or environmental data about the environment which
	doesn't change as the tests proceed.

	Well-known names (keys) are an important concept in this class.
	The HasMinimalKeys member function knows the minimal set of
	well-known names for the class.

	The following names are standard (well-known) at this time:

	nodes An array of the nodes in the cluster
	reset A ResetMechanism object
	logger An array of objects that log strings...
	CMclass The type of ClusterManager we are running
	(This is a class object, not a class instance)
	RandSeed Random seed. It is a triple of bytes. (optional)

	The CTS code ignores names it doesn't know about/need.
	The individual tests have access to this information, and it is
	perfectly acceptable to provide hints, tweaks, fine-tuning
	directions or other information to the tests through this mechanism.
	'''

	def __init__(self, args=None):
	self.Env = EnvFactory().getInstance(args)
	self.Scenario = None
	self.logger = LogFactory()
	self.rsh = RemoteFactory().getInstance()

	def dump(self):
	self.Env.dump()

	def has_key(self, key):
	return key in self.Env.keys()

	def __getitem__(self, key):
	return self.Env[key]

	def __setitem__(self, key, value):
	self.Env[key] = value

	def run(self, Scenario, Iterations):
	if not Scenario:
	self.logger.log("No scenario was defined")
	return 1

	self.logger.log("Cluster nodes: ")
	for node in self.Env["nodes"]:
	self.logger.log(" * %s" % (node))

	if not Scenario.SetUp():
	return 1

	try :
	Scenario.run(Iterations)
	except :
	self.logger.log("Exception by %s" % sys.exc_info()[0])
	self.logger.traceback(traceback)

	Scenario.summarize()
	Scenario.TearDown()
	return 1

	#ClusterManager.oprofileSave(Iterations)
	Scenario.TearDown()

	Scenario.summarize()
	if Scenario.Stats["failure"] > 0:
	return Scenario.Stats["failure"]

	elif Scenario.Stats["success"] != Iterations:
	self.logger.log("No failure count but success != requested iterations")
	return 1

	return 0

	def __CheckNode(self, node):
	"Raise a ValueError if the given node isn't valid"

	if not self.IsValidNode(node):
	raise ValueError("Invalid node [%s] in CheckNode" % node)

	class NodeStatus:
	def __init__(self, env):
	self.Env = env

	def IsNodeBooted(self, node):
	'''Return TRUE if the given node is booted (responds to pings)'''
	if self.Env["docker"]:
	return RemoteFactory().getInstance()("localhost", "docker inspect --format {{.State.Running}} %s \| grep -q true" % node, silent=True) == 0

	return RemoteFactory().getInstance()("localhost", "ping -nq -c1 -w1 %s" % node, silent=True) == 0

	def IsSshdUp(self, node):
	rc = RemoteFactory().getInstance()(node, "true", silent=True)
	return rc == 0

	def WaitForNodeToComeUp(self, node, Timeout=300):
	'''Return TRUE when given node comes up, or None/FALSE if timeout'''
	timeout = Timeout
	anytimeouts = 0
	while timeout > 0:
	if self.IsNodeBooted(node) and self.IsSshdUp(node):
	if anytimeouts:
	# Fudge to wait for the system to finish coming up
	time.sleep(30)
	LogFactory().debug("Node %s now up" % node)
	return 1

	time.sleep(30)
	if (not anytimeouts):
	LogFactory().debug("Waiting for node %s to come up" % node)

	anytimeouts = 1
	timeout = timeout - 1

	LogFactory().log("%s did not come up within %d tries" % (node, Timeout))
	- answer = raw_input('Continue? [nY]')
	+ if self.Env["continue"] == 1:
	+ answer = "Y"
	+ else:
	+ try:
	+ answer = raw_input('Continue? [nY]')
	+ except EOFError, e:
	+ answer = "n"
	if answer and answer == "n":
	raise ValueError("%s did not come up within %d tries" % (node, Timeout))

	def WaitForAllNodesToComeUp(self, nodes, timeout=300):
	'''Return TRUE when all nodes come up, or FALSE if timeout'''

	for node in nodes:
	if not self.WaitForNodeToComeUp(node, timeout):
	return None
	return 1


	class ClusterManager(UserDict):
	'''The Cluster Manager class.
	This is an subclass of the Python dictionary class.
	(this is because it contains lots of {name,value} pairs,
	not because it's behavior is that terribly similar to a
	dictionary in other ways.)

	This is an abstract class which class implements high-level
	operations on the cluster and/or its cluster managers.
	Actual cluster managers classes are subclassed from this type.

	One of the things we do is track the state we think every node should
	be in.
	'''

	def __InitialConditions(self):
	#if os.geteuid() != 0:
	# raise ValueError("Must Be Root!")
	None

	def _finalConditions(self):
	for key in list(self.keys()):
	if self[key] == None:
	raise ValueError("Improper derivation: self[" + key + "] must be overridden by subclass.")

	def __init__(self, Environment, randseed=None):
	self.Env = EnvFactory().getInstance()
	self.templates = PatternSelector(self.Env["Name"])
	self.__InitialConditions()
	self.logger = LogFactory()
	self.clear_cache = 0
	self.TestLoggingLevel=0
	self.data = {}
	self.name = self.Env["Name"]

	self.rsh = RemoteFactory().getInstance()
	self.ShouldBeStatus={}
	self.ns = NodeStatus(self.Env)
	self.OurNode = string.lower(os.uname()[1])
	self.__instance_errorstoignore = []

	def __getitem__(self, key):
	if key == "Name":
	return self.name

	print("FIXME: Getting %s from %s" % (key, repr(self)))
	if key in self.data:
	return self.data[key]

	return self.templates.get_patterns(self.Env["Name"], key)

	def __setitem__(self, key, value):
	print("FIXME: Setting %s=%s on %s" % (key, value, repr(self)))
	self.data[key] = value

	def key_for_node(self, node):
	return node

	def instance_errorstoignore_clear(self):
	'''Allows the test scenario to reset instance errors to ignore on each iteration.'''
	self.__instance_errorstoignore = []

	def instance_errorstoignore(self):
	'''Return list of errors which are 'normal' for a specific test instance'''
	return self.__instance_errorstoignore

	def errorstoignore(self):
	'''Return list of errors which are 'normal' and should be ignored'''
	return []

	def log(self, args):
	self.logger.log(args)

	def debug(self, args):
	self.logger.debug(args)

	def prepare(self):
	'''Finish the Initialization process. Prepare to test...'''

	print(repr(self)+"prepare")
	for node in self.Env["nodes"]:
	if self.StataCM(node):
	self.ShouldBeStatus[node] = "up"
	else:
	self.ShouldBeStatus[node] = "down"

	self.unisolate_node(node)

	def upcount(self):
	'''How many nodes are up?'''
	count = 0
	for node in self.Env["nodes"]:
	if self.ShouldBeStatus[node] == "up":
	count = count + 1
	return count

	def install_helper(self, filename, destdir=None, nodes=None, sourcedir=None):
	if sourcedir == None:
	sourcedir = CTSvars.CTS_home
	file_with_path = "%s/%s" % (sourcedir, filename)
	if not nodes:
	nodes = self.Env["nodes"]

	if not destdir:
	destdir = CTSvars.CTS_home

	self.debug("Installing %s to %s on %s" % (filename, destdir, repr(self.Env["nodes"])))
	for node in nodes:
	self.rsh(node, "mkdir -p %s" % destdir)
	self.rsh.cp(file_with_path, "root@%s:%s/%s" % (node, destdir, filename))
	return file_with_path

	def install_config(self, node):
	return None

	def clear_all_caches(self):
	if self.clear_cache:
	for node in self.Env["nodes"]:
	if self.ShouldBeStatus[node] == "down":
	self.debug("Removing cache file on: "+node)
	self.rsh(node, "rm -f "+CTSvars.HA_VARLIBHBDIR+"/hostcache")
	else:
	self.debug("NOT Removing cache file on: "+node)

	def prepare_fencing_watcher(self, name):
	# If we don't have quorum now but get it as a result of starting this node,
	# then a bunch of nodes might get fenced
	upnode = None
	if self.HasQuorum(None):
	self.debug("Have quorum")
	return None

	if not self.templates["Pat:Fencing_start"]:
	print("No start pattern")
	return None

	if not self.templates["Pat:Fencing_ok"]:
	print("No ok pattern")
	return None

	stonith = None
	stonithPats = []
	for peer in self.Env["nodes"]:
	if self.ShouldBeStatus[peer] != "up":
	stonithPats.append(self.templates["Pat:Fencing_ok"] % peer)
	stonithPats.append(self.templates["Pat:Fencing_start"] % peer)
	elif self.Env["Stack"] == "corosync (cman)":
	# There is a delay between gaining quorum and CMAN starting fencing
	# This can mean that even nodes that are fully up get fenced
	# There is no use fighting it, just look for everyone so that CTS doesn't get confused
	stonithPats.append(self.templates["Pat:Fencing_ok"] % peer)
	stonithPats.append(self.templates["Pat:Fencing_start"] % peer)

	stonith = LogWatcher(self.Env["LogFileName"], stonithPats, "StartupFencing", 0, hosts=self.Env["nodes"], kind=self.Env["LogWatcher"])
	stonith.setwatch()
	return stonith

	def fencing_cleanup(self, node, stonith):
	peer_list = []
	peer_state = {}

	self.debug("Looking for nodes that were fenced as a result of %s starting" % node)

	# If we just started a node, we may now have quorum (and permission to fence)
	if not stonith:
	self.debug("Nothing to do")
	return peer_list

	q = self.HasQuorum(None)
	if not q and len(self.Env["nodes"]) > 2:
	# We didn't gain quorum - we shouldn't have shot anyone
	self.debug("Quorum: %d Len: %d" % (q, len(self.Env["nodes"])))
	return peer_list

	for n in self.Env["nodes"]:
	peer_state[n] = "unknown"

	# Now see if any states need to be updated
	self.debug("looking for: " + repr(stonith.regexes))
	shot = stonith.look(0)
	while shot:
	line = repr(shot)
	self.debug("Found: " + line)
	del stonith.regexes[stonith.whichmatch]

	# Extract node name
	for n in self.Env["nodes"]:
	if re.search(self.templates["Pat:Fencing_ok"] % n, shot):
	peer = n
	peer_state[peer] = "complete"
	self.__instance_errorstoignore.append(self.templates["Pat:Fencing_ok"] % peer)

	elif peer_state[n] != "complete" and re.search(self.templates["Pat:Fencing_start"] % n, shot):
	# TODO: Correctly detect multiple fencing operations for the same host
	peer = n
	peer_state[peer] = "in-progress"
	self.__instance_errorstoignore.append(self.templates["Pat:Fencing_start"] % peer)

	if not peer:
	self.logger.log("ERROR: Unknown stonith match: %s" % line)

	elif not peer in peer_list:
	self.debug("Found peer: " + peer)
	peer_list.append(peer)

	# Get the next one
	shot = stonith.look(60)

	for peer in peer_list:

	self.debug(" Peer %s was fenced as a result of %s starting: %s" % (peer, node, peer_state[peer]))
	if self.Env["at-boot"]:
	self.ShouldBeStatus[peer] = "up"
	else:
	self.ShouldBeStatus[peer] = "down"

	if peer_state[peer] == "in-progress":
	# Wait for any in-progress operations to complete
	shot = stonith.look(60)
	while len(stonith.regexes) and shot:
	line = repr(shot)
	self.debug("Found: " + line)
	del stonith.regexes[stonith.whichmatch]
	shot = stonith.look(60)

	# Now make sure the node is alive too
	self.ns.WaitForNodeToComeUp(peer, self.Env["DeadTime"])

	# Poll until it comes up
	if self.Env["at-boot"]:
	if not self.StataCM(peer):
	time.sleep(self.Env["StartTime"])

	if not self.StataCM(peer):
	self.logger.log("ERROR: Peer %s failed to restart after being fenced" % peer)
	return None

	return peer_list

	def StartaCM(self, node, verbose=False):

	'''Start up the cluster manager on a given node'''
	if verbose: self.logger.log("Starting %s on node %s" % (self.templates["Name"], node))
	else: self.debug("Starting %s on node %s" % (self.templates["Name"], node))
	ret = 1

	if not node in self.ShouldBeStatus:
	self.ShouldBeStatus[node] = "down"

	if self.ShouldBeStatus[node] != "down":
	return 1

	patterns = []
	# Technically we should always be able to notice ourselves starting
	patterns.append(self.templates["Pat:Local_started"] % node)
	if self.upcount() == 0:
	patterns.append(self.templates["Pat:Master_started"] % node)
	else:
	patterns.append(self.templates["Pat:Slave_started"] % node)

	watch = LogWatcher(
	self.Env["LogFileName"], patterns, "StartaCM", self.Env["StartTime"]+10, hosts=self.Env["nodes"], kind=self.Env["LogWatcher"])

	self.install_config(node)

	self.ShouldBeStatus[node] = "any"
	if self.StataCM(node) and self.cluster_stable(self.Env["DeadTime"]):
	self.logger.log ("%s was already started" % (node))
	return 1

	# Clear out the host cache so autojoin can be exercised
	if self.clear_cache:
	self.debug("Removing cache file on: "+node)
	self.rsh(node, "rm -f "+CTSvars.HA_VARLIBHBDIR+"/hostcache")

	if not(self.Env["valgrind-tests"]):
	startCmd = self.templates["StartCmd"]
	else:
	if self.Env["valgrind-prefix"]:
	prefix = self.Env["valgrind-prefix"]
	else:
	prefix = "cts"

	startCmd = """G_SLICE=always-malloc HA_VALGRIND_ENABLED='%s' VALGRIND_OPTS='%s --log-file=/tmp/%s-%s.valgrind' %s""" % (
	self.Env["valgrind-procs"], self.Env["valgrind-opts"], prefix, """%p""", self.templates["StartCmd"])

	stonith = self.prepare_fencing_watcher(node)

	watch.setwatch()

	if self.rsh(node, startCmd) != 0:
	self.logger.log ("Warn: Start command failed on node %s" % (node))
	self.fencing_cleanup(node, stonith)
	return None

	self.ShouldBeStatus[node] = "up"
	watch_result = watch.lookforall()

	if watch.unmatched:
	for regex in watch.unmatched:
	self.logger.log ("Warn: Startup pattern not found: %s" % (regex))

	if watch_result and self.cluster_stable(self.Env["DeadTime"]):
	#self.debug("Found match: "+ repr(watch_result))
	self.fencing_cleanup(node, stonith)
	return 1

	elif self.StataCM(node) and self.cluster_stable(self.Env["DeadTime"]):
	self.fencing_cleanup(node, stonith)
	return 1

	self.logger.log ("Warn: Start failed for node %s" % (node))
	return None

	def StartaCMnoBlock(self, node, verbose=False):

	'''Start up the cluster manager on a given node with none-block mode'''

	if verbose: self.logger.log("Starting %s on node %s" % (self["Name"], node))
	else: self.debug("Starting %s on node %s" % (self["Name"], node))

	# Clear out the host cache so autojoin can be exercised
	if self.clear_cache:
	self.debug("Removing cache file on: "+node)
	self.rsh(node, "rm -f "+CTSvars.HA_VARLIBHBDIR+"/hostcache")

	self.install_config(node)
	if not(self.Env["valgrind-tests"]):
	startCmd = self.templates["StartCmd"]
	else:
	if self.Env["valgrind-prefix"]:
	prefix = self.Env["valgrind-prefix"]
	else:
	prefix = "cts"

	startCmd = """G_SLICE=always-malloc HA_VALGRIND_ENABLED='%s' VALGRIND_OPTS='%s --log-file=/tmp/%s-%s.valgrind' %s""" % (
	self.Env["valgrind-procs"], self.Env["valgrind-opts"], prefix, """%p""", self.templates["StartCmd"])

	self.rsh(node, startCmd, synchronous=0)
	self.ShouldBeStatus[node] = "up"
	return 1

	def StopaCM(self, node, verbose=False, force=False):

	'''Stop the cluster manager on a given node'''

	if verbose: self.logger.log("Stopping %s on node %s" % (self["Name"], node))
	else: self.debug("Stopping %s on node %s" % (self["Name"], node))

	if self.ShouldBeStatus[node] != "up" and force == False:
	return 1

	if self.rsh(node, self.templates["StopCmd"]) == 0:
	# Make sure we can continue even if corosync leaks
	# fdata-* is the old name
	#self.rsh(node, "rm -f /dev/shm/qb-* /dev/shm/fdata-*")
	self.ShouldBeStatus[node] = "down"
	self.cluster_stable(self.Env["DeadTime"])
	return 1
	else:
	self.logger.log ("ERROR: Could not stop %s on node %s" % (self["Name"], node))

	return None

	def StopaCMnoBlock(self, node):

	'''Stop the cluster manager on a given node with none-block mode'''

	self.debug("Stopping %s on node %s" % (self["Name"], node))

	self.rsh(node, self.templates["StopCmd"], synchronous=0)
	self.ShouldBeStatus[node] = "down"
	return 1

	def cluster_stable(self, timeout = None):
	time.sleep(self.Env["StableTime"])
	return 1

	def node_stable(self, node):
	return 1

	def RereadCM(self, node):

	'''Force the cluster manager on a given node to reread its config
	This may be a no-op on certain cluster managers.
	'''
	rc=self.rsh(node, self.templates["RereadCmd"])
	if rc == 0:
	return 1
	else:
	self.logger.log ("Could not force %s on node %s to reread its config"
	% (self["Name"], node))
	return None

	def StataCM(self, node):

	'''Report the status of the cluster manager on a given node'''

	out=self.rsh(node, self.templates["StatusCmd"] % node, 1)
	ret= (string.find(out, 'stopped') == -1)

	try:
	if ret:
	if self.ShouldBeStatus[node] == "down":
	self.logger.log(
	"Node status for %s is %s but we think it should be %s"
	% (node, "up", self.ShouldBeStatus[node]))
	else:
	if self.ShouldBeStatus[node] == "up":
	self.logger.log(
	"Node status for %s is %s but we think it should be %s"
	% (node, "down", self.ShouldBeStatus[node]))
	except KeyError: pass

	if ret:
	self.ShouldBeStatus[node] = "up"
	else:
	self.ShouldBeStatus[node] = "down"
	return ret

	def startall(self, nodelist=None, verbose=False, quick=False):

	'''Start the cluster manager on every node in the cluster.
	We can do it on a subset of the cluster if nodelist is not None.
	'''
	map = {}
	if not nodelist:
	nodelist = self.Env["nodes"]

	for node in nodelist:
	if self.ShouldBeStatus[node] == "down":
	self.ns.WaitForAllNodesToComeUp(nodelist, 300)

	if not quick:
	# This is used for "basic sanity checks", so only start one node ...
	if not self.StartaCM(node, verbose=verbose):
	return 0
	return 1

	# Approximation of SimulStartList for --boot
	watchpats = [ ]
	watchpats.append(self.templates["Pat:DC_IDLE"])
	for node in nodelist:
	watchpats.append(self.templates["Pat:Local_started"] % node)
	watchpats.append(self.templates["Pat:InfraUp"] % node)
	watchpats.append(self.templates["Pat:PacemakerUp"] % node)

	# Start all the nodes - at about the same time...
	watch = LogWatcher(self.Env["LogFileName"], watchpats, "fast-start", self.Env["DeadTime"]+10, hosts=self.Env["nodes"], kind=self.Env["LogWatcher"])
	watch.setwatch()

	if not self.StartaCM(nodelist[0], verbose=verbose):
	return 0
	for node in nodelist:
	self.StartaCMnoBlock(node, verbose=verbose)

	watch.lookforall()
	if watch.unmatched:
	for regex in watch.unmatched:
	self.logger.log ("Warn: Startup pattern not found: %s" % (regex))

	if not self.cluster_stable():
	self.logger.log("Cluster did not stabilize")
	return 0

	return 1

	def stopall(self, nodelist=None, verbose=False, force=False):

	'''Stop the cluster managers on every node in the cluster.
	We can do it on a subset of the cluster if nodelist is not None.
	'''

	ret = 1
	map = {}
	if not nodelist:
	nodelist = self.Env["nodes"]
	for node in self.Env["nodes"]:
	if self.ShouldBeStatus[node] == "up" or force == True:
	if not self.StopaCM(node, verbose=verbose, force=force):
	ret = 0
	return ret

	def rereadall(self, nodelist=None):

	'''Force the cluster managers on every node in the cluster
	to reread their config files. We can do it on a subset of the
	cluster if nodelist is not None.
	'''

	map = {}
	if not nodelist:
	nodelist = self.Env["nodes"]
	for node in self.Env["nodes"]:
	if self.ShouldBeStatus[node] == "up":
	self.RereadCM(node)

	def statall(self, nodelist=None):

	'''Return the status of the cluster managers in the cluster.
	We can do it on a subset of the cluster if nodelist is not None.
	'''

	result = {}
	if not nodelist:
	nodelist = self.Env["nodes"]
	for node in nodelist:
	if self.StataCM(node):
	result[node] = "up"
	else:
	result[node] = "down"
	return result

	def isolate_node(self, target, nodes=None):
	'''isolate the communication between the nodes'''
	if not nodes:
	nodes = self.Env["nodes"]

	for node in nodes:
	if node != target:
	rc = self.rsh(target, self.templates["BreakCommCmd"] % self.key_for_node(node))
	if rc != 0:
	self.logger.log("Could not break the communication between %s and %s: %d" % (target, node, rc))
	return None
	else:
	self.debug("Communication cut between %s and %s" % (target, node))
	return 1

	def unisolate_node(self, target, nodes=None):
	'''fix the communication between the nodes'''
	if not nodes:
	nodes = self.Env["nodes"]

	for node in nodes:
	if node != target:
	restored = 0

	# Limit the amount of time we have asynchronous connectivity for
	# Restore both sides as simultaneously as possible
	self.rsh(target, self.templates["FixCommCmd"] % self.key_for_node(node), synchronous=0)
	self.rsh(node, self.templates["FixCommCmd"] % self.key_for_node(target), synchronous=0)
	self.debug("Communication restored between %s and %s" % (target, node))

	def reducecomm_node(self,node):
	'''reduce the communication between the nodes'''
	rc = self.rsh(node, self.templates["ReduceCommCmd"]%(self.Env["XmitLoss"],self.Env["RecvLoss"]))
	if rc == 0:
	return 1
	else:
	self.logger.log("Could not reduce the communication between the nodes from node: %s" % node)
	return None

	def restorecomm_node(self,node):
	'''restore the saved communication between the nodes'''
	rc = 0
	if float(self.Env["XmitLoss"]) != 0 or float(self.Env["RecvLoss"]) != 0 :
	rc = self.rsh(node, self.templates["RestoreCommCmd"]);
	if rc == 0:
	return 1
	else:
	self.logger.log("Could not restore the communication between the nodes from node: %s" % node)
	return None

	def HasQuorum(self, node_list):
	"Return TRUE if the cluster currently has quorum"
	# If we are auditing a partition, then one side will
	# have quorum and the other not.
	# So the caller needs to tell us which we are checking
	# If no value for node_list is specified... assume all nodes
	raise ValueError("Abstract Class member (HasQuorum)")

	def Components(self):
	raise ValueError("Abstract Class member (Components)")

	def oprofileStart(self, node=None):
	if not node:
	for n in self.Env["oprofile"]:
	self.oprofileStart(n)

	elif node in self.Env["oprofile"]:
	self.debug("Enabling oprofile on %s" % node)
	self.rsh(node, "opcontrol --init")
	self.rsh(node, "opcontrol --setup --no-vmlinux --separate=lib --callgraph=20 --image=all")
	self.rsh(node, "opcontrol --start")
	self.rsh(node, "opcontrol --reset")

	def oprofileSave(self, test, node=None):
	if not node:
	for n in self.Env["oprofile"]:
	self.oprofileSave(test, n)

	elif node in self.Env["oprofile"]:
	self.rsh(node, "opcontrol --dump")
	self.rsh(node, "opcontrol --save=cts.%d" % test)
	# Read back with: opreport -l session:cts.0 image:/usr/lib/heartbeat/c*
	if None:
	self.rsh(node, "opcontrol --reset")
	else:
	self.oprofileStop(node)
	self.oprofileStart(node)

	def oprofileStop(self, node=None):
	if not node:
	for n in self.Env["oprofile"]:
	self.oprofileStop(n)

	elif node in self.Env["oprofile"]:
	self.debug("Stopping oprofile on %s" % node)
	self.rsh(node, "opcontrol --reset")
	self.rsh(node, "opcontrol --shutdown 2>&1 > /dev/null")


	def StatsExtract(self):
	if not self.Env["stats"]:
	return

	for host in self.Env["nodes"]:
	log_stats_file = "%s/cts-stats.csv" % CTSvars.CRM_DAEMON_DIR
	if host in has_log_stats:
	self.rsh(host, '''bash %s %s stop''' % (log_stats_bin, log_stats_file))
	(rc, lines) = self.rsh(host, '''cat %s''' % log_stats_file, stdout=2)
	self.rsh(host, '''bash %s %s delete''' % (log_stats_bin, log_stats_file))

	fname = "cts-stats-%d-nodes-%s.csv" % (len(self.Env["nodes"]), host)
	print("Extracted stats: %s" % fname)
	fd = open(fname, "a")
	fd.writelines(lines)
	fd.close()

	def StatsMark(self, testnum):
	'''Mark the test number in the stats log'''

	global has_log_stats
	if not self.Env["stats"]:
	return

	for host in self.Env["nodes"]:
	log_stats_file = "%s/cts-stats.csv" % CTSvars.CRM_DAEMON_DIR
	if not host in has_log_stats:

	global log_stats
	global log_stats_bin
	script=log_stats
	#script = re.sub("\\\\", "\\\\", script)
	script = re.sub('\"', '\\\"', script)
	script = re.sub("'", "\'", script)
	script = re.sub("`", "\`", script)
	script = re.sub("\$", "\\\$", script)

	self.debug("Installing %s on %s" % (log_stats_bin, host))
	self.rsh(host, '''echo "%s" > %s''' % (script, log_stats_bin), silent=True)
	self.rsh(host, '''bash %s %s delete''' % (log_stats_bin, log_stats_file))
	has_log_stats[host] = 1

	# Now mark it
	self.rsh(host, '''bash %s %s mark %s''' % (log_stats_bin, log_stats_file, testnum), synchronous=0)


	class Resource:
	'''
	This is an HA resource (not a resource group).
	A resource group is just an ordered list of Resource objects.
	'''

	def __init__(self, cm, rsctype=None, instance=None):
	self.CM = cm
	self.ResourceType = rsctype
	self.Instance = instance
	self.needs_quorum = 1

	def Type(self):
	return self.ResourceType

	def Instance(self, nodename):
	return self.Instance

	def IsRunningOn(self, nodename):
	'''
	This member function returns true if our resource is running
	on the given node in the cluster.
	It is analagous to the "status" operation on SystemV init scripts and
	heartbeat scripts. FailSafe calls it the "exclusive" operation.
	'''
	raise ValueError("Abstract Class member (IsRunningOn)")
	return None

	def IsWorkingCorrectly(self, nodename):
	'''
	This member function returns true if our resource is operating
	correctly on the given node in the cluster.
	Heartbeat does not require this operation, but it might be called
	the Monitor operation, which is what FailSafe calls it.
	For remotely monitorable resources (like IP addresses), they should
	be monitored remotely for testing.
	'''
	raise ValueError("Abstract Class member (IsWorkingCorrectly)")
	return None

	def Start(self, nodename):
	'''
	This member function starts or activates the resource.
	'''
	raise ValueError("Abstract Class member (Start)")
	return None

	def Stop(self, nodename):
	'''
	This member function stops or deactivates the resource.
	'''
	raise ValueError("Abstract Class member (Stop)")
	return None

	def __repr__(self):
	if (self.Instance and len(self.Instance) > 1):
	return "{" + self.ResourceType + "::" + self.Instance + "}"
	else:
	return "{" + self.ResourceType + "}"


	class Component:
	def kill(self, node):
	None


	class Process(Component):
	def __init__(self, cm, name, process=None, dc_only=0, pats=[], dc_pats=[], badnews_ignore=[], common_ignore=[], triggersreboot=0):
	self.name = str(name)
	self.dc_only = dc_only
	self.pats = pats
	self.dc_pats = dc_pats
	self.CM = cm
	self.badnews_ignore = badnews_ignore
	self.badnews_ignore.extend(common_ignore)
	self.triggersreboot = triggersreboot

	if process:
	self.proc = str(process)
	else:
	self.proc = str(name)
	self.KillCmd = "killall -9 " + self.proc

	def kill(self, node):
	if self.CM.rsh(node, self.KillCmd) != 0:
	self.CM.log ("ERROR: Kill %s failed on node %s" % (self.name,node))
	return None
	return 1
	diff --git a/cts/CTSaudits.py b/cts/CTSaudits.py
	index e8663f2475..f7a6648cfd 100755
	--- a/cts/CTSaudits.py
	+++ b/cts/CTSaudits.py
	@@ -1,863 +1,870 @@
	'''CTS: Cluster Testing System: Audit module
	'''

	__copyright__ = '''
	Copyright (C) 2000, 2001,2005 Alan Robertson <alanr@unix.sh>
	Licensed under the GNU GPL.
	'''

	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public License
	# as published by the Free Software Foundation; either version 2
	# of the License, or (at your option) any later version.
	#
	# This program is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	# GNU General Public License for more details.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write to the Free Software
	# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

	import time, re, uuid
	from watcher import LogWatcher


	class ClusterAudit:

	def __init__(self, cm):
	self.CM = cm

	def __call__(self):
	raise ValueError("Abstract Class member (__call__)")

	def is_applicable(self):
	'''Return TRUE if we are applicable in the current test configuration'''
	raise ValueError("Abstract Class member (is_applicable)")
	return 1

	def log(self, args):
	self.CM.log("audit: %s" % args)

	def debug(self, args):
	self.CM.debug("audit: %s" % args)

	def name(self):
	raise ValueError("Abstract Class member (name)")

	AllAuditClasses = [ ]


	class LogAudit(ClusterAudit):

	def name(self):
	return "LogAudit"

	def __init__(self, cm):
	self.CM = cm
	self.kinds = [ "combined syslog", "journal", "remote" ]

	def RestartClusterLogging(self, nodes=None):
	if not nodes:
	nodes = self.CM.Env["nodes"]

	self.CM.debug("Restarting logging on: %s" % repr(nodes))

	for node in nodes:
	if self.CM.Env["have_systemd"]:
	if self.CM.rsh(node, "systemctl stop systemd-journald.socket") != 0:
	self.CM.log ("ERROR: Cannot stop 'systemd-journald' on %s" % node)
	if self.CM.rsh(node, "systemctl start systemd-journald.service") != 0:
	self.CM.log ("ERROR: Cannot start 'systemd-journald' on %s" % node)

	if self.CM.rsh(node, "service %s restart" % self.CM.Env["syslogd"]) != 0:
	self.CM.log ("ERROR: Cannot restart '%s' on %s" % (self.CM.Env["syslogd"], node))

	def TestLogging(self):
	patterns = []
	prefix = "Test message from"
	suffix = str(uuid.uuid4())
	watch = {}

	for node in self.CM.Env["nodes"]:
	# Look for the node name in two places to make sure
	# that syslog is logging with the correct hostname
	m = re.search("^([^.]+).*", node)
	if m:
	simple = m.group(1)
	else:
	simple = node
	patterns.append("%s.*%s %s %s" % (simple, prefix, node, suffix))

	watch_pref = self.CM.Env["LogWatcher"]
	if watch_pref == "any":
	for k in self.kinds:
	watch[k] = LogWatcher(self.CM.Env["LogFileName"], patterns, "LogAudit", 5, silent=True, hosts=self.CM.Env["nodes"], kind=k)
	watch[k].setwatch()
	else:
	k = watch_pref
	watch[k] = LogWatcher(self.CM.Env["LogFileName"], patterns, "LogAudit", 5, silent=True, hosts=self.CM.Env["nodes"], kind=k)
	watch[k].setwatch()

	if watch_pref == "any": self.CM.log("Writing log with key: %s" % (suffix))
	for node in self.CM.Env["nodes"]:
	cmd = "logger -p %s.info %s %s %s" % (self.CM.Env["SyslogFacility"], prefix, node, suffix)
	if self.CM.rsh(node, cmd, synchronous=0, silent=True) != 0:
	self.CM.log ("ERROR: Cannot execute remote command [%s] on %s" % (cmd, node))

	for k in self.kinds:
	if k in watch:
	w = watch[k]
	if watch_pref == "any": self.CM.log("Testing for %s logs" % (k))
	w.lookforall(silent=True)
	if not w.unmatched:
	if watch_pref == "any":
	self.CM.log ("Continuing with %s-based log reader" % (w.kind))
	self.CM.Env["LogWatcher"] = w.kind
	return 1

	for k in list(watch.keys()):
	w = watch[k]
	if w.unmatched:
	for regex in w.unmatched:
	self.CM.log ("Test message [%s] not found in %s logs." % (regex, w.kind))

	return 0

	def __call__(self):
	max = 3
	attempt = 0

	self.CM.ns.WaitForAllNodesToComeUp(self.CM.Env["nodes"])
	while attempt <= max and self.TestLogging() == 0:
	attempt = attempt + 1
	self.RestartClusterLogging()
	time.sleep(60*attempt)

	if attempt > max:
	self.CM.log("ERROR: Cluster logging unrecoverable.")
	return 0

	return 1

	def is_applicable(self):
	if self.CM.Env["DoBSC"]:
	return 0
	if self.CM.Env["LogAuditDisabled"]:
	return 0
	return 1


	class DiskAudit(ClusterAudit):

	def name(self):
	return "DiskspaceAudit"

	def __init__(self, cm):
	self.CM = cm

	def __call__(self):
	result = 1
	dfcmd = "df -BM /var/log \| tail -1 \| awk '{print $(NF-1)\" \"$(NF-2)}' \| tr -d 'M%'"

	self.CM.ns.WaitForAllNodesToComeUp(self.CM.Env["nodes"])
	for node in self.CM.Env["nodes"]:
	dfout = self.CM.rsh(node, dfcmd, 1)
	if not dfout:
	self.CM.log ("ERROR: Cannot execute remote df command [%s] on %s" % (dfcmd, node))
	else:
	try:
	(used, remain) = dfout.split()
	used_percent = int(used)
	remaining_mb = int(remain)
	except (ValueError, TypeError):
	self.CM.log("Warning: df output '%s' from %s was invalid [%s, %s]"
	% (dfout, node, used, remain))
	else:
	if remaining_mb < 10 or used_percent > 95:
	self.CM.log("CRIT: Out of log disk space on %s (%d%% / %dMb)"
	% (node, used_percent, remaining_mb))
	result = None
	- answer = raw_input('Continue? [nY] ')
	+ if self.CM.Env["continue"] == 1:
	+ answer = "Y"
	+ else:
	+ try:
	+ answer = raw_input('Continue? [nY]')
	+ except EOFError, e:
	+ answer = "n"
	+
	if answer and answer == "n":
	raise ValueError("Disk full on %s" % (node))
	ret = 0

	elif remaining_mb < 100 or used_percent > 90:
	self.CM.log("WARN: Low on log disk space (%d Mbytes) on %s" % (remaining_mb, node))
	return result

	def is_applicable(self):
	if self.CM.Env["DoBSC"]:
	return 0
	return 1


	class FileAudit(ClusterAudit):

	def name(self):
	return "FileAudit"

	def __init__(self, cm):
	self.CM = cm
	self.known = []

	def __call__(self):
	result = 1

	self.CM.ns.WaitForAllNodesToComeUp(self.CM.Env["nodes"])
	for node in self.CM.Env["nodes"]:

	(rc, lsout) = self.CM.rsh(node, "ls -al /var/lib/heartbeat/cores/* \| grep core.[0-9]", None)
	for line in lsout:
	line = line.strip()
	if line not in self.known:
	result = 0
	self.known.append(line)
	self.CM.log("Warning: Pacemaker core file on %s: %s" % (node, line))

	(rc, lsout) = self.CM.rsh(node, "ls -al /var/lib/corosync \| grep core.[0-9]", None)
	for line in lsout:
	line = line.strip()
	if line not in self.known:
	result = 0
	self.known.append(line)
	self.CM.log("Warning: Corosync core file on %s: %s" % (node, line))

	if node in self.CM.ShouldBeStatus and self.CM.ShouldBeStatus[node] == "down":
	clean = 0
	(rc, lsout) = self.CM.rsh(node, "ls -al /dev/shm \| grep qb-", None)
	for line in lsout:
	result = 0
	clean = 1
	self.CM.log("Warning: Stale IPC file on %s: %s" % (node, line))

	if clean:
	(rc, lsout) = self.CM.rsh(node, "ps axf \| grep -e pacemaker -e corosync", None)
	for line in lsout:
	self.CM.debug("ps[%s]: %s" % (node, line))

	self.CM.rsh(node, "rm -f /dev/shm/qb-*")

	else:
	self.CM.debug("Skipping %s" % node)

	return result

	def is_applicable(self):
	return 1


	class AuditResource:
	def __init__(self, cm, line):
	fields = line.split()
	self.CM = cm
	self.line = line
	self.type = fields[1]
	self.id = fields[2]
	self.clone_id = fields[3]
	self.parent = fields[4]
	self.rprovider = fields[5]
	self.rclass = fields[6]
	self.rtype = fields[7]
	self.host = fields[8]
	self.needs_quorum = fields[9]
	self.flags = int(fields[10])
	self.flags_s = fields[11]

	if self.parent == "NA":
	self.parent = None

	def unique(self):
	if self.flags & int("0x00000020", 16):
	return 1
	return 0

	def orphan(self):
	if self.flags & int("0x00000001", 16):
	return 1
	return 0

	def managed(self):
	if self.flags & int("0x00000002", 16):
	return 1
	return 0


	class AuditConstraint:
	def __init__(self, cm, line):
	fields = line.split()
	self.CM = cm
	self.line = line
	self.type = fields[1]
	self.id = fields[2]
	self.rsc = fields[3]
	self.target = fields[4]
	self.score = fields[5]
	self.rsc_role = fields[6]
	self.target_role = fields[7]

	if self.rsc_role == "NA":
	self.rsc_role = None
	if self.target_role == "NA":
	self.target_role = None


	class PrimitiveAudit(ClusterAudit):
	def name(self):
	return "PrimitiveAudit"

	def __init__(self, cm):
	self.CM = cm

	def doResourceAudit(self, resource, quorum):
	rc = 1
	active = self.CM.ResourceLocation(resource.id)

	if len(active) == 1:
	if quorum:
	self.debug("Resource %s active on %s" % (resource.id, repr(active)))

	elif resource.needs_quorum == 1:
	self.CM.log("Resource %s active without quorum: %s"
	% (resource.id, repr(active)))
	rc = 0

	elif not resource.managed():
	self.CM.log("Resource %s not managed. Active on %s"
	% (resource.id, repr(active)))

	elif not resource.unique():
	# TODO: Figure out a clever way to actually audit these resource types
	if len(active) > 1:
	self.debug("Non-unique resource %s is active on: %s"
	% (resource.id, repr(active)))
	else:
	self.debug("Non-unique resource %s is not active" % resource.id)

	elif len(active) > 1:
	self.CM.log("Resource %s is active multiple times: %s"
	% (resource.id, repr(active)))
	rc = 0

	elif resource.orphan():
	self.debug("Resource %s is an inactive orphan" % resource.id)

	elif len(self.inactive_nodes) == 0:
	self.CM.log("WARN: Resource %s not served anywhere" % resource.id)
	rc = 0

	elif self.CM.Env["warn-inactive"] == 1:
	if quorum or not resource.needs_quorum:
	self.CM.log("WARN: Resource %s not served anywhere (Inactive nodes: %s)"
	% (resource.id, repr(self.inactive_nodes)))
	else:
	self.debug("Resource %s not served anywhere (Inactive nodes: %s)"
	% (resource.id, repr(self.inactive_nodes)))

	elif quorum or not resource.needs_quorum:
	self.debug("Resource %s not served anywhere (Inactive nodes: %s)"
	% (resource.id, repr(self.inactive_nodes)))

	return rc

	def setup(self):
	self.target = None
	self.resources = []
	self.constraints = []
	self.active_nodes = []
	self.inactive_nodes = []

	for node in self.CM.Env["nodes"]:
	if self.CM.ShouldBeStatus[node] == "up":
	self.active_nodes.append(node)
	else:
	self.inactive_nodes.append(node)

	for node in self.CM.Env["nodes"]:
	if self.target == None and self.CM.ShouldBeStatus[node] == "up":
	self.target = node

	if not self.target:
	# TODO: In Pacemaker 1.0 clusters we'll be able to run crm_resource
	# with CIB_file=/path/to/cib.xml even when the cluster isn't running
	self.debug("No nodes active - skipping %s" % self.name())
	return 0

	(rc, lines) = self.CM.rsh(self.target, "crm_resource -c", None)

	for line in lines:
	if re.search("^Resource", line):
	self.resources.append(AuditResource(self.CM, line))
	elif re.search("^Constraint", line):
	self.constraints.append(AuditConstraint(self.CM, line))
	else:
	self.CM.log("Unknown entry: %s" % line);

	return 1

	def __call__(self):
	rc = 1

	if not self.setup():
	return 1

	quorum = self.CM.HasQuorum(None)
	for resource in self.resources:
	if resource.type == "primitive":
	if self.doResourceAudit(resource, quorum) == 0:
	rc = 0
	return rc

	def is_applicable(self):
	if self.CM["Name"] == "crm-lha":
	return 1
	if self.CM["Name"] == "crm-ais":
	return 1
	return 0


	class GroupAudit(PrimitiveAudit):
	def name(self):
	return "GroupAudit"

	def __call__(self):
	rc = 1
	if not self.setup():
	return 1

	for group in self.resources:
	if group.type == "group":
	first_match = 1
	group_location = None
	for child in self.resources:
	if child.parent == group.id:
	nodes = self.CM.ResourceLocation(child.id)

	if first_match and len(nodes) > 0:
	group_location = nodes[0]

	first_match = 0

	if len(nodes) > 1:
	rc = 0
	self.CM.log("Child %s of %s is active more than once: %s"
	% (child.id, group.id, repr(nodes)))

	elif len(nodes) == 0:
	# Groups are allowed to be partially active
	# However we do need to make sure later children aren't running
	group_location = None
	self.debug("Child %s of %s is stopped" % (child.id, group.id))

	elif nodes[0] != group_location:
	rc = 0
	self.CM.log("Child %s of %s is active on the wrong node (%s) expected %s"
	% (child.id, group.id, nodes[0], group_location))
	else:
	self.debug("Child %s of %s is active on %s" % (child.id, group.id, nodes[0]))

	return rc


	class CloneAudit(PrimitiveAudit):
	def name(self):
	return "CloneAudit"

	def __call__(self):
	rc = 1
	if not self.setup():
	return 1

	for clone in self.resources:
	if clone.type == "clone":
	for child in self.resources:
	if child.parent == clone.id and child.type == "primitive":
	self.debug("Checking child %s of %s..." % (child.id, clone.id))
	# Check max and node_max
	# Obtain with:
	# crm_resource -g clone_max --meta -r child.id
	# crm_resource -g clone_node_max --meta -r child.id

	return rc


	class ColocationAudit(PrimitiveAudit):
	def name(self):
	return "ColocationAudit"

	def crm_location(self, resource):
	(rc, lines) = self.CM.rsh(self.target, "crm_resource -W -r %s -Q"%resource, None)
	hosts = []
	if rc == 0:
	for line in lines:
	fields = line.split()
	hosts.append(fields[0])

	return hosts

	def __call__(self):
	rc = 1
	if not self.setup():
	return 1

	for coloc in self.constraints:
	if coloc.type == "rsc_colocation":
	source = self.crm_location(coloc.rsc)
	target = self.crm_location(coloc.target)
	if len(source) == 0:
	self.debug("Colocation audit (%s): %s not running" % (coloc.id, coloc.rsc))
	else:
	for node in source:
	if not node in target:
	rc = 0
	self.CM.log("Colocation audit (%s): %s running on %s (not in %s)"
	% (coloc.id, coloc.rsc, node, repr(target)))
	else:
	self.debug("Colocation audit (%s): %s running on %s (in %s)"
	% (coloc.id, coloc.rsc, node, repr(target)))

	return rc


	class CrmdStateAudit(ClusterAudit):
	def __init__(self, cm):
	self.CM = cm
	self.Stats = {"calls":0
	, "success":0
	, "failure":0
	, "skipped":0
	, "auditfail":0}

	def has_key(self, key):
	return key in self.Stats

	def __setitem__(self, key, value):
	self.Stats[key] = value

	def __getitem__(self, key):
	return self.Stats[key]

	def incr(self, name):
	'''Increment (or initialize) the value associated with the given name'''
	if not name in self.Stats:
	self.Stats[name] = 0
	self.Stats[name] = self.Stats[name]+1

	def __call__(self):
	passed = 1
	up_are_down = 0
	down_are_up = 0
	unstable_list = []

	for node in self.CM.Env["nodes"]:
	should_be = self.CM.ShouldBeStatus[node]
	rc = self.CM.test_node_CM(node)
	if rc > 0:
	if should_be == "down":
	down_are_up = down_are_up + 1
	if rc == 1:
	unstable_list.append(node)
	elif should_be == "up":
	up_are_down = up_are_down + 1

	if len(unstable_list) > 0:
	passed = 0
	self.CM.log("Cluster is not stable: %d (of %d): %s"
	% (len(unstable_list), self.CM.upcount(), repr(unstable_list)))

	if up_are_down > 0:
	passed = 0
	self.CM.log("%d (of %d) nodes expected to be up were down."
	% (up_are_down, len(self.CM.Env["nodes"])))

	if down_are_up > 0:
	passed = 0
	self.CM.log("%d (of %d) nodes expected to be down were up."
	% (down_are_up, len(self.CM.Env["nodes"])))

	return passed

	def name(self):
	return "CrmdStateAudit"

	def is_applicable(self):
	if self.CM["Name"] == "crm-lha":
	return 1
	if self.CM["Name"] == "crm-ais":
	return 1
	return 0


	class CIBAudit(ClusterAudit):
	def __init__(self, cm):
	self.CM = cm
	self.Stats = {"calls":0
	, "success":0
	, "failure":0
	, "skipped":0
	, "auditfail":0}

	def has_key(self, key):
	return key in self.Stats

	def __setitem__(self, key, value):
	self.Stats[key] = value

	def __getitem__(self, key):
	return self.Stats[key]

	def incr(self, name):
	'''Increment (or initialize) the value associated with the given name'''
	if not name in self.Stats:
	self.Stats[name] = 0
	self.Stats[name] = self.Stats[name]+1

	def __call__(self):
	passed = 1
	ccm_partitions = self.CM.find_partitions()

	if len(ccm_partitions) == 0:
	self.debug("\tNo partitions to audit")
	return 1

	for partition in ccm_partitions:
	self.debug("\tAuditing CIB consistency for: %s" % partition)
	partition_passed = 0
	if self.audit_cib_contents(partition) == 0:
	passed = 0

	return passed

	def audit_cib_contents(self, hostlist):
	passed = 1
	node0 = None
	node0_xml = None

	partition_hosts = hostlist.split()
	for node in partition_hosts:
	node_xml = self.store_remote_cib(node, node0)

	if node_xml == None:
	self.CM.log("Could not perform audit: No configuration from %s" % node)
	passed = 0

	elif node0 == None:
	node0 = node
	node0_xml = node_xml

	elif node0_xml == None:
	self.CM.log("Could not perform audit: No configuration from %s" % node0)
	passed = 0

	else:
	(rc, result) = self.CM.rsh(
	node0, "crm_diff -VV -cf --new %s --original %s" % (node_xml, node0_xml), None)

	if rc != 0:
	self.CM.log("Diff between %s and %s failed: %d" % (node0_xml, node_xml, rc))
	passed = 0

	for line in result:
	if not re.search("<diff/>", line):
	passed = 0
	self.debug("CibDiff[%s-%s]: %s" % (node0, node, line))
	else:
	self.debug("CibDiff[%s-%s] Ignoring: %s" % (node0, node, line))

	# self.CM.rsh(node0, "rm -f %s" % node_xml)
	# self.CM.rsh(node0, "rm -f %s" % node0_xml)
	return passed

	def store_remote_cib(self, node, target):
	combined = ""
	filename = "/tmp/ctsaudit.%s.xml" % node

	if not target:
	target = node

	(rc, lines) = self.CM.rsh(node, self.CM["CibQuery"], None)
	if rc != 0:
	self.CM.log("Could not retrieve configuration")
	return None

	self.CM.rsh("localhost", "rm -f %s" % filename)
	for line in lines:
	self.CM.rsh("localhost", "echo \'%s\' >> %s" % (line[:-1], filename), silent=True)

	if self.CM.rsh.cp(filename, "root@%s:%s" % (target, filename), silent=True) != 0:
	self.CM.log("Could not store configuration")
	return None
	return filename

	def name(self):
	return "CibAudit"

	def is_applicable(self):
	if self.CM["Name"] == "crm-lha":
	return 1
	if self.CM["Name"] == "crm-ais":
	return 1
	return 0


	class PartitionAudit(ClusterAudit):
	def __init__(self, cm):
	self.CM = cm
	self.Stats = {"calls":0
	, "success":0
	, "failure":0
	, "skipped":0
	, "auditfail":0}
	self.NodeEpoch = {}
	self.NodeState = {}
	self.NodeQuorum = {}

	def has_key(self, key):
	return self.Stats.has_key(key)

	def __setitem__(self, key, value):
	self.Stats[key] = value

	def __getitem__(self, key):
	return self.Stats[key]

	def incr(self, name):
	'''Increment (or initialize) the value associated with the given name'''
	if not name in self.Stats:
	self.Stats[name] = 0
	self.Stats[name] = self.Stats[name]+1

	def __call__(self):
	passed = 1
	ccm_partitions = self.CM.find_partitions()

	if ccm_partitions == None or len(ccm_partitions) == 0:
	return 1

	self.CM.cluster_stable(double_check=True)

	if len(ccm_partitions) != self.CM.partitions_expected:
	self.CM.log("ERROR: %d cluster partitions detected:" % len(ccm_partitions))
	passed = 0
	for partition in ccm_partitions:
	self.CM.log("\t %s" % partition)

	for partition in ccm_partitions:
	partition_passed = 0
	if self.audit_partition(partition) == 0:
	passed = 0

	return passed

	def trim_string(self, avalue):
	if not avalue:
	return None
	if len(avalue) > 1:
	return avalue[:-1]

	def trim2int(self, avalue):
	if not avalue:
	return None
	if len(avalue) > 1:
	return int(avalue[:-1])

	def audit_partition(self, partition):
	passed = 1
	dc_found = []
	dc_allowed_list = []
	lowest_epoch = None
	node_list = partition.split()

	self.debug("Auditing partition: %s" % (partition))
	for node in node_list:
	if self.CM.ShouldBeStatus[node] != "up":
	self.CM.log("Warn: Node %s appeared out of nowhere" % (node))
	self.CM.ShouldBeStatus[node] = "up"
	# not in itself a reason to fail the audit (not what we're
	# checking for in this audit)

	self.NodeState[node] = self.CM.rsh(node, self.CM["StatusCmd"] % node, 1)
	self.NodeEpoch[node] = self.CM.rsh(node, self.CM["EpochCmd"], 1)
	self.NodeQuorum[node] = self.CM.rsh(node, self.CM["QuorumCmd"], 1)

	self.debug("Node %s: %s - %s - %s." % (node, self.NodeState[node], self.NodeEpoch[node], self.NodeQuorum[node]))
	self.NodeState[node] = self.trim_string(self.NodeState[node])
	self.NodeEpoch[node] = self.trim2int(self.NodeEpoch[node])
	self.NodeQuorum[node] = self.trim_string(self.NodeQuorum[node])

	if not self.NodeEpoch[node]:
	self.CM.log("Warn: Node %s dissappeared: cant determin epoch" % (node))
	self.CM.ShouldBeStatus[node] = "down"
	# not in itself a reason to fail the audit (not what we're
	# checking for in this audit)
	elif lowest_epoch == None or self.NodeEpoch[node] < lowest_epoch:
	lowest_epoch = self.NodeEpoch[node]

	if not lowest_epoch:
	self.CM.log("Lowest epoch not determined in %s" % (partition))
	passed = 0

	for node in node_list:
	if self.CM.ShouldBeStatus[node] == "up":
	if self.CM.is_node_dc(node, self.NodeState[node]):
	dc_found.append(node)
	if self.NodeEpoch[node] == lowest_epoch:
	self.debug("%s: OK" % node)
	elif not self.NodeEpoch[node]:
	self.debug("Check on %s ignored: no node epoch" % node)
	elif not lowest_epoch:
	self.debug("Check on %s ignored: no lowest epoch" % node)
	else:
	self.CM.log("DC %s is not the oldest node (%d vs. %d)"
	% (node, self.NodeEpoch[node], lowest_epoch))
	passed = 0

	if len(dc_found) == 0:
	self.CM.log("DC not found on any of the %d allowed nodes: %s (of %s)"
	% (len(dc_allowed_list), str(dc_allowed_list), str(node_list)))

	elif len(dc_found) > 1:
	self.CM.log("%d DCs (%s) found in cluster partition: %s"
	% (len(dc_found), str(dc_found), str(node_list)))
	passed = 0

	if passed == 0:
	for node in node_list:
	if self.CM.ShouldBeStatus[node] == "up":
	self.CM.log("epoch %s : %s"
	% (self.NodeEpoch[node], self.NodeState[node]))

	return passed

	def name(self):
	return "PartitionAudit"

	def is_applicable(self):
	if self.CM["Name"] == "crm-lha":
	return 1
	if self.CM["Name"] == "crm-ais":
	return 1
	return 0

	AllAuditClasses.append(DiskAudit)
	AllAuditClasses.append(FileAudit)
	AllAuditClasses.append(LogAudit)
	AllAuditClasses.append(CrmdStateAudit)
	AllAuditClasses.append(PartitionAudit)
	AllAuditClasses.append(PrimitiveAudit)
	AllAuditClasses.append(GroupAudit)
	AllAuditClasses.append(CloneAudit)
	AllAuditClasses.append(ColocationAudit)
	AllAuditClasses.append(CIBAudit)


	def AuditList(cm):
	result = []
	for auditclass in AllAuditClasses:
	a = auditclass(cm)
	if a.is_applicable():
	result.append(a)
	return result
	diff --git a/cts/CTSscenarios.py b/cts/CTSscenarios.py
	index 7e52e7558c..b8ee01a323 100644
	--- a/cts/CTSscenarios.py
	+++ b/cts/CTSscenarios.py
	@@ -1,579 +1,591 @@
	from CTS import *
	from CTStests import CTSTest
	from CTSaudits import ClusterAudit
	from cts.watcher import LogWatcher

	class ScenarioComponent:

	def __init__(self, Env):
	self.Env = Env

	def IsApplicable(self):
	'''Return TRUE if the current ScenarioComponent is applicable
	in the given LabEnvironment given to the constructor.
	'''

	raise ValueError("Abstract Class member (IsApplicable)")

	def SetUp(self, CM):
	'''Set up the given ScenarioComponent'''
	raise ValueError("Abstract Class member (Setup)")

	def TearDown(self, CM):
	'''Tear down (undo) the given ScenarioComponent'''
	raise ValueError("Abstract Class member (Setup)")


	class Scenario:
	(
	'''The basic idea of a scenario is that of an ordered list of
	ScenarioComponent objects. Each ScenarioComponent is SetUp() in turn,
	and then after the tests have been run, they are torn down using TearDown()
	(in reverse order).

	A Scenario is applicable to a particular cluster manager iff each
	ScenarioComponent is applicable.

	A partially set up scenario is torn down if it fails during setup.
	''')

	def __init__(self, ClusterManager, Components, Audits, Tests):

	"Initialize the Scenario from the list of ScenarioComponents"

	self.ClusterManager = ClusterManager
	self.Components = Components
	self.Audits = Audits
	self.Tests = Tests

	self.BadNews = None
	self.TestSets = []
	self.Stats = {"success":0, "failure":0, "BadNews":0, "skipped":0}
	self.Sets = []

	#self.ns=CTS.NodeStatus(self.Env)

	for comp in Components:
	if not issubclass(comp.__class__, ScenarioComponent):
	raise ValueError("Init value must be subclass of ScenarioComponent")

	for audit in Audits:
	if not issubclass(audit.__class__, ClusterAudit):
	raise ValueError("Init value must be subclass of ClusterAudit")

	for test in Tests:
	if not issubclass(test.__class__, CTSTest):
	raise ValueError("Init value must be a subclass of CTSTest")

	def IsApplicable(self):
	(
	'''A Scenario IsApplicable() iff each of its ScenarioComponents IsApplicable()
	'''
	)

	for comp in self.Components:
	if not comp.IsApplicable():
	return None
	return 1

	def SetUp(self):
	'''Set up the Scenario. Return TRUE on success.'''

	self.ClusterManager.prepare()
	self.audit() # Also detects remote/local log config
	self.ClusterManager.StatsMark(0)
	self.ClusterManager.ns.WaitForAllNodesToComeUp(self.ClusterManager.Env["nodes"])

	self.audit()
	if self.ClusterManager.Env["valgrind-tests"]:
	self.ClusterManager.install_helper("cts.supp")

	self.BadNews = LogWatcher(self.ClusterManager.Env["LogFileName"],
	self.ClusterManager.templates.get_patterns(
	self.ClusterManager.Env["Name"], "BadNews"), "BadNews", 0,
	kind=self.ClusterManager.Env["LogWatcher"],
	hosts=self.ClusterManager.Env["nodes"])
	self.BadNews.setwatch() # Call after we've figured out what type of log watching to do in LogAudit

	j = 0
	while j < len(self.Components):
	if not self.Components[j].SetUp(self.ClusterManager):
	# OOPS! We failed. Tear partial setups down.
	self.audit()
	self.ClusterManager.log("Tearing down partial setup")
	self.TearDown(j)
	return None
	j = j + 1

	self.audit()
	return 1

	def TearDown(self, max=None):

	'''Tear Down the Scenario - in reverse order.'''

	if max == None:
	max = len(self.Components)-1
	j = max
	while j >= 0:
	self.Components[j].TearDown(self.ClusterManager)
	j = j - 1

	self.audit()
	self.ClusterManager.StatsExtract()

	def incr(self, name):
	'''Increment (or initialize) the value associated with the given name'''
	if not name in self.Stats:
	self.Stats[name] = 0
	self.Stats[name] = self.Stats[name]+1

	def run(self, Iterations):
	self.ClusterManager.oprofileStart()
	try:
	self.run_loop(Iterations)
	self.ClusterManager.oprofileStop()
	except:
	self.ClusterManager.oprofileStop()
	raise

	def run_loop(self, Iterations):
	raise ValueError("Abstract Class member (run_loop)")

	def run_test(self, test, testcount):
	nodechoice = self.ClusterManager.Env.RandomNode()

	ret = 1
	where = ""
	did_run = 0

	self.ClusterManager.StatsMark(testcount)
	self.ClusterManager.instance_errorstoignore_clear()
	self.ClusterManager.log(("Running test %s" % test.name).ljust(35) + (" (%s) " % nodechoice).ljust(15) + "[" + ("%d" % testcount).rjust(3) + "]")

	starttime = test.set_timer()
	if not test.setup(nodechoice):
	self.ClusterManager.log("Setup failed")
	ret = 0

	elif not test.canrunnow(nodechoice):
	self.ClusterManager.log("Skipped")
	test.skipped()

	else:
	did_run = 1
	ret = test(nodechoice)

	if not test.teardown(nodechoice):
	self.ClusterManager.log("Teardown failed")
	- answer = raw_input('Continue? [nY] ')
	+ if self.ClusterManager.Env["continue"] == 1:
	+ answer = "Y"
	+ else:
	+ try:
	+ answer = raw_input('Continue? [nY]')
	+ except EOFError, e:
	+ answer = "n"
	if answer and answer == "n":
	raise ValueError("Teardown of %s on %s failed" % (test.name, nodechoice))
	ret = 0

	stoptime = time.time()
	self.ClusterManager.oprofileSave(testcount)

	elapsed_time = stoptime - starttime
	test_time = stoptime - test.get_timer()
	if not test["min_time"]:
	test["elapsed_time"] = elapsed_time
	test["min_time"] = test_time
	test["max_time"] = test_time
	else:
	test["elapsed_time"] = test["elapsed_time"] + elapsed_time
	if test_time < test["min_time"]:
	test["min_time"] = test_time
	if test_time > test["max_time"]:
	test["max_time"] = test_time

	if ret:
	self.incr("success")
	test.log_timer()
	else:
	self.incr("failure")
	self.ClusterManager.statall()
	did_run = 1 # Force the test count to be incremented anyway so test extraction works

	self.audit(test.errorstoignore())
	return did_run

	def summarize(self):
	self.ClusterManager.log("****************")
	self.ClusterManager.log("Overall Results:" + repr(self.Stats))
	self.ClusterManager.log("****************")

	stat_filter = {
	"calls":0,
	"failure":0,
	"skipped":0,
	"auditfail":0,
	}
	self.ClusterManager.log("Test Summary")
	for test in self.Tests:
	for key in list(stat_filter.keys()):
	stat_filter[key] = test.Stats[key]
	self.ClusterManager.log(("Test %s: "%test.name).ljust(25) + " %s"%repr(stat_filter))

	self.ClusterManager.debug("Detailed Results")
	for test in self.Tests:
	self.ClusterManager.debug(("Test %s: "%test.name).ljust(25) + " %s"%repr(test.Stats))

	self.ClusterManager.log("<<<<<<<<<<<<<<<< TESTS COMPLETED")

	def audit(self, LocalIgnore=[]):
	errcount = 0
	ignorelist = []
	ignorelist.append("CTS:")
	ignorelist.extend(LocalIgnore)
	ignorelist.extend(self.ClusterManager.errorstoignore())
	ignorelist.extend(self.ClusterManager.instance_errorstoignore())

	# This makes sure everything is stabilized before starting...
	failed = 0
	for audit in self.Audits:
	if not audit():
	self.ClusterManager.log("Audit " + audit.name() + " FAILED.")
	failed += 1
	else:
	self.ClusterManager.debug("Audit " + audit.name() + " passed.")

	while errcount < 1000:
	match = None
	if self.BadNews:
	match = self.BadNews.look(0)

	if match:
	add_err = 1
	for ignore in ignorelist:
	if add_err == 1 and re.search(ignore, match):
	add_err = 0
	if add_err == 1:
	self.ClusterManager.log("BadNews: " + match)
	self.incr("BadNews")
	errcount = errcount + 1
	else:
	break
	else:
	- answer = raw_input('Big problems. Continue? [nY]')
	+ if self.ClusterManager.Env["continue"] == 1:
	+ answer = "Y"
	+ else:
	+ try:
	+ answer = raw_input('Big problems. Continue? [nY]')
	+ except EOFError, e:
	+ answer = "n"
	if answer and answer == "n":
	self.ClusterManager.log("Shutting down.")
	self.summarize()
	self.TearDown()
	raise ValueError("Looks like we hit a BadNews jackpot!")

	if self.BadNews:
	self.BadNews.end()
	return failed


	class AllOnce(Scenario):
	'''Every Test Once''' # Accessable as __doc__
	def run_loop(self, Iterations):
	testcount = 1
	for test in self.Tests:
	self.run_test(test, testcount)
	testcount += 1


	class RandomTests(Scenario):
	'''Random Test Execution'''
	def run_loop(self, Iterations):
	testcount = 1
	while testcount <= Iterations:
	test = self.ClusterManager.Env.RandomGen.choice(self.Tests)
	self.run_test(test, testcount)
	testcount += 1


	class BasicSanity(Scenario):
	'''Basic Cluster Sanity'''
	def run_loop(self, Iterations):
	testcount = 1
	while testcount <= Iterations:
	test = self.Environment.RandomGen.choice(self.Tests)
	self.run_test(test, testcount)
	testcount += 1


	class Sequence(Scenario):
	'''Named Tests in Sequence'''
	def run_loop(self, Iterations):
	testcount = 1
	while testcount <= Iterations:
	for test in self.Tests:
	self.run_test(test, testcount)
	testcount += 1


	class Boot(Scenario):
	'''Start the Cluster'''
	def run_loop(self, Iterations):
	testcount = 0


	class BootCluster(ScenarioComponent):
	(
	'''BootCluster is the most basic of ScenarioComponents.
	This ScenarioComponent simply starts the cluster manager on all the nodes.
	It is fairly robust as it waits for all nodes to come up before starting
	as they might have been rebooted or crashed for some reason beforehand.
	''')
	def __init__(self, Env):
	pass

	def IsApplicable(self):
	'''BootCluster is so generic it is always Applicable'''
	return 1

	def SetUp(self, CM):
	'''Basic Cluster Manager startup. Start everything'''

	CM.prepare()

	# Clear out the cobwebs ;-)
	CM.stopall(verbose=True, force=True)

	# Now start the Cluster Manager on all the nodes.
	CM.log("Starting Cluster Manager on all nodes.")
	return CM.startall(verbose=True, quick=True)

	def TearDown(self, CM, force=False):
	'''Set up the given ScenarioComponent'''

	# Stop the cluster manager everywhere

	CM.log("Stopping Cluster Manager on all nodes")
	return CM.stopall(verbose=True, force=force)


	class LeaveBooted(BootCluster):
	def TearDown(self, CM):
	'''Set up the given ScenarioComponent'''

	# Stop the cluster manager everywhere

	CM.log("Leaving Cluster running on all nodes")
	return 1


	class PingFest(ScenarioComponent):
	(
	'''PingFest does a flood ping to each node in the cluster from the test machine.

	If the LabEnvironment Parameter PingSize is set, it will be used as the size
	of ping packet requested (via the -s option). If it is not set, it defaults
	to 1024 bytes.

	According to the manual page for ping:
	Outputs packets as fast as they come back or one hundred times per
	second, whichever is more. For every ECHO_REQUEST sent a period ``.''
	is printed, while for every ECHO_REPLY received a backspace is printed.
	This provides a rapid display of how many packets are being dropped.
	Only the super-user may use this option. This can be very hard on a net-
	work and should be used with caution.
	''' )

	def __init__(self, Env):
	self.Env = Env

	def IsApplicable(self):
	'''PingFests are always applicable ;-)
	'''

	return 1

	def SetUp(self, CM):
	'''Start the PingFest!'''

	self.PingSize = 1024
	if "PingSize" in CM.Env.keys():
	self.PingSize = CM.Env["PingSize"]

	CM.log("Starting %d byte flood pings" % self.PingSize)

	self.PingPids = []
	for node in CM.Env["nodes"]:
	self.PingPids.append(self._pingchild(node))

	CM.log("Ping PIDs: " + repr(self.PingPids))
	return 1

	def TearDown(self, CM):
	'''Stop it right now! My ears are pinging!!'''

	for pid in self.PingPids:
	if pid != None:
	CM.log("Stopping ping process %d" % pid)
	os.kill(pid, signal.SIGKILL)

	def _pingchild(self, node):

	Args = ["ping", "-qfn", "-s", str(self.PingSize), node]

	sys.stdin.flush()
	sys.stdout.flush()
	sys.stderr.flush()
	pid = os.fork()

	if pid < 0:
	self.Env.log("Cannot fork ping child")
	return None
	if pid > 0:
	return pid

	# Otherwise, we're the child process.

	os.execvp("ping", Args)
	self.Env.log("Cannot execvp ping: " + repr(Args))
	sys.exit(1)


	class PacketLoss(ScenarioComponent):
	(
	'''
	It would be useful to do some testing of CTS with a modest amount of packet loss
	enabled - so we could see that everything runs like it should with a certain
	amount of packet loss present.
	''')

	def IsApplicable(self):
	'''always Applicable'''
	return 1

	def SetUp(self, CM):
	'''Reduce the reliability of communications'''
	if float(CM.Env["XmitLoss"]) == 0 and float(CM.Env["RecvLoss"]) == 0 :
	return 1

	for node in CM.Env["nodes"]:
	CM.reducecomm_node(node)

	CM.log("Reduce the reliability of communications")

	return 1

	def TearDown(self, CM):
	'''Fix the reliability of communications'''

	if float(CM.Env["XmitLoss"]) == 0 and float(CM.Env["RecvLoss"]) == 0 :
	return 1

	for node in CM.Env["nodes"]:
	CM.unisolate_node(node)

	CM.log("Fix the reliability of communications")


	class BasicSanityCheck(ScenarioComponent):
	(
	'''
	''')

	def IsApplicable(self):
	return self.Env["DoBSC"]

	def SetUp(self, CM):

	CM.prepare()

	# Clear out the cobwebs
	self.TearDown(CM)

	# Now start the Cluster Manager on all the nodes.
	CM.log("Starting Cluster Manager on BSC node(s).")
	return CM.startall()

	def TearDown(self, CM):
	CM.log("Stopping Cluster Manager on BSC node(s).")
	return CM.stopall()


	class Benchmark(ScenarioComponent):
	(
	'''
	''')

	def IsApplicable(self):
	return self.Env["benchmark"]

	def SetUp(self, CM):

	CM.prepare()

	# Clear out the cobwebs
	self.TearDown(CM, force=True)

	# Now start the Cluster Manager on all the nodes.
	CM.log("Starting Cluster Manager on all node(s).")
	return CM.startall()

	def TearDown(self, CM):
	CM.log("Stopping Cluster Manager on all node(s).")
	return CM.stopall()


	class RollingUpgrade(ScenarioComponent):
	(
	'''
	Test a rolling upgrade between two versions of the stack
	''')

	def __init__(self, Env):
	self.Env = Env

	def IsApplicable(self):
	if not self.Env["rpm-dir"]:
	return None
	if not self.Env["current-version"]:
	return None
	if not self.Env["previous-version"]:
	return None

	return 1

	def install(self, node, version):

	target_dir = "/tmp/rpm-%s" % version
	src_dir = "%s/%s" % (self.CM.Env["rpm-dir"], version)

	rc = self.CM.rsh(node, "mkdir -p %s" % target_dir)
	rc = self.CM.cp("%s/*.rpm %s:%s" % (src_dir, node, target_dir))
	rc = self.CM.rsh(node, "rpm -Uvh --force %s/*.rpm" % (target_dir))

	return self.success()

	def upgrade(self, node):
	return self.install(node, self.CM.Env["current-version"])

	def downgrade(self, node):
	return self.install(node, self.CM.Env["previous-version"])

	def SetUp(self, CM):
	print(repr(self)+"prepare")
	CM.prepare()

	# Clear out the cobwebs
	CM.stopall(force=True)

	CM.log("Downgrading all nodes to %s." % self.Env["previous-version"])

	for node in self.Env["nodes"]:
	if not self.downgrade(node):
	CM.log("Couldn't downgrade %s" % node)
	return None

	return 1

	def TearDown(self, CM):
	# Stop everything
	CM.log("Stopping Cluster Manager on Upgrade nodes.")
	CM.stopall()

	CM.log("Upgrading all nodes to %s." % self.Env["current-version"])
	for node in self.Env["nodes"]:
	if not self.upgrade(node):
	CM.log("Couldn't upgrade %s" % node)
	return None

	return 1
	diff --git a/cts/CTStests.py b/cts/CTStests.py
	index 814a42186a..7f28a121fa 100644
	--- a/cts/CTStests.py
	+++ b/cts/CTStests.py
	@@ -1,3150 +1,3156 @@
	'''CTS: Cluster Testing System: Tests module

	There are a few things we want to do here:

	'''

	__copyright__ = '''
	Copyright (C) 2000, 2001 Alan Robertson <alanr@unix.sh>
	Licensed under the GNU GPL.

	Add RecourceRecover testcase Zhao Kai <zhaokai@cn.ibm.com>
	'''

	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public License
	# as published by the Free Software Foundation; either version 2
	# of the License, or (at your option) any later version.
	#
	# This program is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	# GNU General Public License for more details.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write to the Free Software
	# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

	#
	# SPECIAL NOTE:
	#
	# Tests may NOT implement any cluster-manager-specific code in them.
	# EXTEND the ClusterManager object to provide the base capabilities
	# the test needs if you need to do something that the current CM classes
	# do not. Otherwise you screw up the whole point of the object structure
	# in CTS.
	#
	# Thank you.
	#

	import time, os, re, string, subprocess, tempfile
	from stat import *
	from cts import CTS
	from cts.CTSaudits import *
	from cts.CTSvars import *
	from cts.patterns import PatternSelector
	from cts.logging import LogFactory
	from cts.remote import RemoteFactory
	from cts.watcher import LogWatcher
	from cts.environment import EnvFactory

	AllTestClasses = [ ]


	class CTSTest:
	'''
	A Cluster test.
	We implement the basic set of properties and behaviors for a generic
	cluster test.

	Cluster tests track their own statistics.
	We keep each of the kinds of counts we track as separate {name,value}
	pairs.
	'''

	def __init__(self, cm):
	#self.name="the unnamed test"
	self.Stats = {"calls":0
	, "success":0
	, "failure":0
	, "skipped":0
	, "auditfail":0}

	# if not issubclass(cm.__class__, ClusterManager):
	# raise ValueError("Must be a ClusterManager object")
	self.CM = cm
	self.Env = EnvFactory().getInstance()
	self.rsh = RemoteFactory().getInstance()
	self.logger = LogFactory()
	self.templates = PatternSelector(cm["Name"])
	self.Audits = []
	self.timeout = 120
	self.passed = 1
	self.is_loop = 0
	self.is_unsafe = 0
	self.is_docker_unsafe = 0
	self.is_experimental = 0
	self.is_container = 0
	self.is_valgrind = 0
	self.benchmark = 0 # which tests to benchmark
	self.timer = {} # timers

	def log(self, args):
	self.logger.log(args)

	def debug(self, args):
	self.logger.debug(args)

	def has_key(self, key):
	return key in self.Stats

	def __setitem__(self, key, value):
	self.Stats[key] = value

	def __getitem__(self, key):
	if str(key) == "0":
	raise ValueError("Bad call to 'foo in X', should reference 'foo in X.Stats' instead")

	if key in self.Stats:
	return self.Stats[key]
	return None

	def log_mark(self, msg):
	self.debug("MARK: test %s %s %d" % (self.name,msg,time.time()))
	return

	def get_timer(self,key = "test"):
	try: return self.timer[key]
	except: return 0

	def set_timer(self,key = "test"):
	self.timer[key] = time.time()
	return self.timer[key]

	def log_timer(self,key = "test"):
	elapsed = 0
	if key in self.timer:
	elapsed = time.time() - self.timer[key]
	s = key == "test" and self.name or "%s:%s" % (self.name,key)
	self.debug("%s runtime: %.2f" % (s, elapsed))
	del self.timer[key]
	return elapsed

	def incr(self, name):
	'''Increment (or initialize) the value associated with the given name'''
	if not name in self.Stats:
	self.Stats[name] = 0
	self.Stats[name] = self.Stats[name]+1

	# Reset the test passed boolean
	if name == "calls":
	self.passed = 1

	def failure(self, reason="none"):
	'''Increment the failure count'''
	self.passed = 0
	self.incr("failure")
	self.logger.log(("Test %s" % self.name).ljust(35) + " FAILED: %s" % reason)
	return None

	def success(self):
	'''Increment the success count'''
	self.incr("success")
	return 1

	def skipped(self):
	'''Increment the skipped count'''
	self.incr("skipped")
	return 1

	def __call__(self, node):
	'''Perform the given test'''
	raise ValueError("Abstract Class member (__call__)")
	self.incr("calls")
	return self.failure()

	def audit(self):
	passed = 1
	if len(self.Audits) > 0:
	for audit in self.Audits:
	if not audit():
	self.logger.log("Internal %s Audit %s FAILED." % (self.name, audit.name()))
	self.incr("auditfail")
	passed = 0
	return passed

	def setup(self, node):
	'''Setup the given test'''
	return self.success()

	def teardown(self, node):
	'''Tear down the given test'''
	return self.success()

	def create_watch(self, patterns, timeout, name=None):
	if not name:
	name = self.name
	return LogWatcher(self.Env["LogFileName"], patterns, name, timeout, kind=self.Env["LogWatcher"], hosts=self.Env["nodes"])

	def local_badnews(self, prefix, watch, local_ignore=[]):
	errcount = 0
	if not prefix:
	prefix = "LocalBadNews:"

	ignorelist = []
	ignorelist.append(" CTS: ")
	ignorelist.append(prefix)
	ignorelist.extend(local_ignore)

	while errcount < 100:
	match = watch.look(0)
	if match:
	add_err = 1
	for ignore in ignorelist:
	if add_err == 1 and re.search(ignore, match):
	add_err = 0
	if add_err == 1:
	self.logger.log(prefix + " " + match)
	errcount = errcount + 1
	else:
	break
	else:
	self.logger.log("Too many errors!")

	watch.end()
	return errcount

	def is_applicable(self):
	return self.is_applicable_common()

	def is_applicable_common(self):
	'''Return TRUE if we are applicable in the current test configuration'''
	#raise ValueError("Abstract Class member (is_applicable)")

	if self.is_loop and not self.Env["loop-tests"]:
	return 0
	elif self.is_unsafe and not self.Env["unsafe-tests"]:
	return 0
	elif self.is_valgrind and not self.Env["valgrind-tests"]:
	return 0
	elif self.is_experimental and not self.Env["experimental-tests"]:
	return 0
	elif self.is_docker_unsafe and self.Env["docker"]:
	return 0
	elif self.is_container and not self.Env["container-tests"]:
	return 0
	elif self.Env["benchmark"] and self.benchmark == 0:
	return 0

	return 1

	def find_ocfs2_resources(self, node):
	self.r_o2cb = None
	self.r_ocfs2 = []

	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	r = AuditResource(self.CM, line)
	if r.rtype == "o2cb" and r.parent != "NA":
	self.debug("Found o2cb: %s" % self.r_o2cb)
	self.r_o2cb = r.parent
	if re.search("^Constraint", line):
	c = AuditConstraint(self.CM, line)
	if c.type == "rsc_colocation" and c.target == self.r_o2cb:
	self.r_ocfs2.append(c.rsc)

	self.debug("Found ocfs2 filesystems: %s" % repr(self.r_ocfs2))
	return len(self.r_ocfs2)

	def canrunnow(self, node):
	'''Return TRUE if we can meaningfully run right now'''
	return 1

	def errorstoignore(self):
	'''Return list of errors which are 'normal' and should be ignored'''
	return []


	class StopTest(CTSTest):
	'''Stop (deactivate) the cluster manager on a node'''
	def __init__(self, cm):
	CTSTest.__init__(self, cm)
	self.name = "Stop"

	def __call__(self, node):
	'''Perform the 'stop' test. '''
	self.incr("calls")
	if self.CM.ShouldBeStatus[node] != "up":
	return self.skipped()

	patterns = []
	# Technically we should always be able to notice ourselves stopping
	patterns.append(self.templates["Pat:We_stopped"] % node)

	#if self.Env["use_logd"]:
	# patterns.append(self.templates["Pat:Logd_stopped"] % node)

	# Any active node needs to notice this one left
	# NOTE: This wont work if we have multiple partitions
	for other in self.Env["nodes"]:
	if self.CM.ShouldBeStatus[other] == "up" and other != node:
	patterns.append(self.templates["Pat:They_stopped"] %(other, self.CM.key_for_node(node)))
	#self.debug("Checking %s will notice %s left"%(other, node))

	watch = self.create_watch(patterns, self.Env["DeadTime"])
	watch.setwatch()

	if node == self.CM.OurNode:
	self.incr("us")
	else:
	if self.CM.upcount() <= 1:
	self.incr("all")
	else:
	self.incr("them")

	self.CM.StopaCM(node)
	watch_result = watch.lookforall()

	failreason = None
	UnmatchedList = "\|\|"
	if watch.unmatched:
	(rc, output) = self.rsh(node, "/bin/ps axf", None)
	for line in output:
	self.debug(line)

	(rc, output) = self.rsh(node, "/usr/sbin/dlm_tool dump", None)
	for line in output:
	self.debug(line)

	for regex in watch.unmatched:
	self.logger.log ("ERROR: Shutdown pattern not found: %s" % (regex))
	UnmatchedList += regex + "\|\|";
	failreason = "Missing shutdown pattern"

	self.CM.cluster_stable(self.Env["DeadTime"])

	if not watch.unmatched or self.CM.upcount() == 0:
	return self.success()

	if len(watch.unmatched) >= self.CM.upcount():
	return self.failure("no match against (%s)" % UnmatchedList)

	if failreason == None:
	return self.success()
	else:
	return self.failure(failreason)
	#
	# We don't register StopTest because it's better when called by
	# another test...
	#


	class StartTest(CTSTest):
	'''Start (activate) the cluster manager on a node'''
	def __init__(self, cm, debug=None):
	CTSTest.__init__(self,cm)
	self.name = "start"
	self.debug = debug

	def __call__(self, node):
	'''Perform the 'start' test. '''
	self.incr("calls")

	if self.CM.upcount() == 0:
	self.incr("us")
	else:
	self.incr("them")

	if self.CM.ShouldBeStatus[node] != "down":
	return self.skipped()
	elif self.CM.StartaCM(node):
	return self.success()
	else:
	return self.failure("Startup %s on node %s failed"
	% (self.Env["Name"], node))

	#
	# We don't register StartTest because it's better when called by
	# another test...
	#


	class FlipTest(CTSTest):
	'''If it's running, stop it. If it's stopped start it.
	Overthrow the status quo...
	'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Flip"
	self.start = StartTest(cm)
	self.stop = StopTest(cm)

	def __call__(self, node):
	'''Perform the 'Flip' test. '''
	self.incr("calls")
	if self.CM.ShouldBeStatus[node] == "up":
	self.incr("stopped")
	ret = self.stop(node)
	type = "up->down"
	# Give the cluster time to recognize it's gone...
	time.sleep(self.Env["StableTime"])
	elif self.CM.ShouldBeStatus[node] == "down":
	self.incr("started")
	ret = self.start(node)
	type = "down->up"
	else:
	return self.skipped()

	self.incr(type)
	if ret:
	return self.success()
	else:
	return self.failure("%s failure" % type)

	# Register FlipTest as a good test to run
	AllTestClasses.append(FlipTest)


	class RestartTest(CTSTest):
	'''Stop and restart a node'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Restart"
	self.start = StartTest(cm)
	self.stop = StopTest(cm)
	self.benchmark = 1

	def __call__(self, node):
	'''Perform the 'restart' test. '''
	self.incr("calls")

	self.incr("node:" + node)

	ret1 = 1
	if self.CM.StataCM(node):
	self.incr("WasStopped")
	if not self.start(node):
	return self.failure("start (setup) failure: "+node)

	self.set_timer()
	if not self.stop(node):
	return self.failure("stop failure: "+node)
	if not self.start(node):
	return self.failure("start failure: "+node)
	return self.success()

	# Register RestartTest as a good test to run
	AllTestClasses.append(RestartTest)


	class StonithdTest(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self, cm)
	self.name = "Stonithd"
	self.startall = SimulStartLite(cm)
	self.benchmark = 1

	def __call__(self, node):
	self.incr("calls")
	if len(self.Env["nodes"]) < 2:
	return self.skipped()

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	is_dc = self.CM.is_node_dc(node)

	watchpats = []
	watchpats.append(self.templates["Pat:FenceOpOK"] % node)
	watchpats.append(self.templates["Pat:NodeFenced"] % node)

	if self.Env["at-boot"] == 0:
	self.debug("Expecting %s to stay down" % node)
	self.CM.ShouldBeStatus[node] = "down"
	else:
	self.debug("Expecting %s to come up again %d" % (node, self.Env["at-boot"]))
	watchpats.append("%s.* S_STARTING -> S_PENDING" % node)
	watchpats.append("%s.* S_PENDING -> S_NOT_DC" % node)

	watch = self.create_watch(watchpats, 30 + self.Env["DeadTime"] + self.Env["StableTime"] + self.Env["StartTime"])
	watch.setwatch()

	origin = self.Env.RandomGen.choice(self.Env["nodes"])

	rc = self.rsh(origin, "stonith_admin --reboot %s -VVVVVV" % node)

	if rc == 194:
	# 194 - 256 = -62 = Timer expired
	#
	# Look for the patterns, usually this means the required
	# device was running on the node to be fenced - or that
	# the required devices were in the process of being loaded
	# and/or moved
	#
	# Effectively the node committed suicide so there will be
	# no confirmation, but pacemaker should be watching and
	# fence the node again

	self.logger.log("Fencing command on %s to fence %s timed out" % (origin, node))

	elif origin != node and rc != 0:
	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	self.debug("Waiting STONITHd node to come back up")
	self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)

	self.logger.log("Fencing command on %s failed to fence %s (rc=%d)" % (origin, node, rc))

	elif origin == node and rc != 255:
	# 255 == broken pipe, ie. the node was fenced as expected
	self.logger.log("Locally originated fencing returned %d" % rc)

	self.set_timer("fence")
	matched = watch.lookforall()
	self.log_timer("fence")
	self.set_timer("reform")
	if watch.unmatched:
	self.logger.log("Patterns not found: " + repr(watch.unmatched))

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	self.debug("Waiting STONITHd node to come back up")
	self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)

	self.debug("Waiting for the cluster to re-stabilize with all nodes")
	is_stable = self.CM.cluster_stable(self.Env["StartTime"])

	if not matched:
	return self.failure("Didn't find all expected patterns")
	elif not is_stable:
	return self.failure("Cluster did not become stable")

	self.log_timer("reform")
	return self.success()

	def errorstoignore(self):
	return [
	self.templates["Pat:Fencing_start"] % ".*",
	self.templates["Pat:Fencing_ok"] % ".*",
	r"error.: Resource .stonith::.* is active on 2 nodes attempting recovery",
	r"error.: Operation reboot of .by .* for stonith_admin.*: Timer expired",
	]

	def is_applicable(self):
	if not self.is_applicable_common():
	return 0

	if "DoFencing" in self.Env.keys():
	return self.Env["DoFencing"]

	return 1

	AllTestClasses.append(StonithdTest)


	class StartOnebyOne(CTSTest):
	'''Start all the nodes ~ one by one'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "StartOnebyOne"
	self.stopall = SimulStopLite(cm)
	self.start = StartTest(cm)
	self.ns = CTS.NodeStatus(cm.Env)

	def __call__(self, dummy):
	'''Perform the 'StartOnebyOne' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Shut down all the nodes...
	ret = self.stopall(None)
	if not ret:
	return self.failure("Test setup failed")

	failed = []
	self.set_timer()
	for node in self.Env["nodes"]:
	if not self.start(node):
	failed.append(node)

	if len(failed) > 0:
	return self.failure("Some node failed to start: " + repr(failed))

	return self.success()

	# Register StartOnebyOne as a good test to run
	AllTestClasses.append(StartOnebyOne)


	class SimulStart(CTSTest):
	'''Start all the nodes ~ simultaneously'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SimulStart"
	self.stopall = SimulStopLite(cm)
	self.startall = SimulStartLite(cm)

	def __call__(self, dummy):
	'''Perform the 'SimulStart' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Shut down all the nodes...
	ret = self.stopall(None)
	if not ret:
	return self.failure("Setup failed")

	self.CM.clear_all_caches()

	if not self.startall(None):
	return self.failure("Startall failed")

	return self.success()

	# Register SimulStart as a good test to run
	AllTestClasses.append(SimulStart)


	class SimulStop(CTSTest):
	'''Stop all the nodes ~ simultaneously'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SimulStop"
	self.startall = SimulStartLite(cm)
	self.stopall = SimulStopLite(cm)

	def __call__(self, dummy):
	'''Perform the 'SimulStop' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Start up all the nodes...
	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	if not self.stopall(None):
	return self.failure("Stopall failed")

	return self.success()

	# Register SimulStop as a good test to run
	AllTestClasses.append(SimulStop)


	class StopOnebyOne(CTSTest):
	'''Stop all the nodes in order'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "StopOnebyOne"
	self.startall = SimulStartLite(cm)
	self.stop = StopTest(cm)

	def __call__(self, dummy):
	'''Perform the 'StopOnebyOne' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Start up all the nodes...
	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	failed = []
	self.set_timer()
	for node in self.Env["nodes"]:
	if not self.stop(node):
	failed.append(node)

	if len(failed) > 0:
	return self.failure("Some node failed to stop: " + repr(failed))

	self.CM.clear_all_caches()
	return self.success()

	# Register StopOnebyOne as a good test to run
	AllTestClasses.append(StopOnebyOne)


	class RestartOnebyOne(CTSTest):
	'''Restart all the nodes in order'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "RestartOnebyOne"
	self.startall = SimulStartLite(cm)

	def __call__(self, dummy):
	'''Perform the 'RestartOnebyOne' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Start up all the nodes...
	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	did_fail = []
	self.set_timer()
	self.restart = RestartTest(self.CM)
	for node in self.Env["nodes"]:
	if not self.restart(node):
	did_fail.append(node)

	if did_fail:
	return self.failure("Could not restart %d nodes: %s"
	% (len(did_fail), repr(did_fail)))
	return self.success()

	# Register StopOnebyOne as a good test to run
	AllTestClasses.append(RestartOnebyOne)


	class PartialStart(CTSTest):
	'''Start a node - but tell it to stop before it finishes starting up'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "PartialStart"
	self.startall = SimulStartLite(cm)
	self.stopall = SimulStopLite(cm)
	self.stop = StopTest(cm)
	#self.is_unsafe = 1

	def __call__(self, node):
	'''Perform the 'PartialStart' test. '''
	self.incr("calls")

	ret = self.stopall(None)
	if not ret:
	return self.failure("Setup failed")

	# FIXME! This should use the CM class to get the pattern
	# then it would be applicable in general
	watchpats = []
	watchpats.append("crmd.*Connecting to cluster infrastructure")
	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
	watch.setwatch()

	self.CM.StartaCMnoBlock(node)
	ret = watch.lookforall()
	if not ret:
	self.logger.log("Patterns not found: " + repr(watch.unmatched))
	return self.failure("Setup of %s failed" % node)

	ret = self.stop(node)
	if not ret:
	return self.failure("%s did not stop in time" % node)

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''

	# We might do some fencing in the 2-node case if we make it up far enough
	return [
	"""Executing reboot fencing operation""",
	]

	# Register StopOnebyOne as a good test to run
	AllTestClasses.append(PartialStart)


	class StandbyTest(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Standby"
	self.benchmark = 1

	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)

	# make sure the node is active
	# set the node to standby mode
	# check resources, none resource should be running on the node
	# set the node to active mode
	# check resouces, resources should have been migrated back (SHOULD THEY?)

	def __call__(self, node):

	self.incr("calls")
	ret = self.startall(None)
	if not ret:
	return self.failure("Start all nodes failed")

	self.debug("Make sure node %s is active" % node)
	if self.CM.StandbyStatus(node) != "off":
	if not self.CM.SetStandbyMode(node, "off"):
	return self.failure("can't set node %s to active mode" % node)

	self.CM.cluster_stable()

	status = self.CM.StandbyStatus(node)
	if status != "off":
	return self.failure("standby status of %s is [%s] but we expect [off]" % (node, status))

	self.debug("Getting resources running on node %s" % node)
	rsc_on_node = self.CM.active_resources(node)

	watchpats = []
	watchpats.append(r"State transition .* -> S_POLICY_ENGINE")
	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
	watch.setwatch()

	self.debug("Setting node %s to standby mode" % node)
	if not self.CM.SetStandbyMode(node, "on"):
	return self.failure("can't set node %s to standby mode" % node)

	self.set_timer("on")

	ret = watch.lookforall()
	if not ret:
	self.logger.log("Patterns not found: " + repr(watch.unmatched))
	self.CM.SetStandbyMode(node, "off")
	return self.failure("cluster didn't react to standby change on %s" % node)

	self.CM.cluster_stable()

	status = self.CM.StandbyStatus(node)
	if status != "on":
	return self.failure("standby status of %s is [%s] but we expect [on]" % (node, status))
	self.log_timer("on")

	self.debug("Checking resources")
	bad_run = self.CM.active_resources(node)
	if len(bad_run) > 0:
	rc = self.failure("%s set to standby, %s is still running on it" % (node, repr(bad_run)))
	self.debug("Setting node %s to active mode" % node)
	self.CM.SetStandbyMode(node, "off")
	return rc

	self.debug("Setting node %s to active mode" % node)
	if not self.CM.SetStandbyMode(node, "off"):
	return self.failure("can't set node %s to active mode" % node)

	self.set_timer("off")
	self.CM.cluster_stable()

	status = self.CM.StandbyStatus(node)
	if status != "off":
	return self.failure("standby status of %s is [%s] but we expect [off]" % (node, status))
	self.log_timer("off")

	return self.success()

	AllTestClasses.append(StandbyTest)


	class ValgrindTest(CTSTest):
	'''Check for memory leaks'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Valgrind"
	self.stopall = SimulStopLite(cm)
	self.startall = SimulStartLite(cm)
	self.is_valgrind = 1
	self.is_loop = 1

	def setup(self, node):
	self.incr("calls")

	ret = self.stopall(None)
	if not ret:
	return self.failure("Stop all nodes failed")

	# Enable valgrind
	self.logger.logPat = "/tmp/%s-*.valgrind" % self.name

	self.Env["valgrind-prefix"] = self.name

	self.rsh(node, "rm -f %s" % self.logger.logPat, None)

	ret = self.startall(None)
	if not ret:
	return self.failure("Start all nodes failed")

	for node in self.Env["nodes"]:
	(rc, output) = self.rsh(node, "ps u --ppid `pidofproc aisexec`", None)
	for line in output:
	self.debug(line)

	return self.success()

	def teardown(self, node):
	# Disable valgrind
	self.Env["valgrind-prefix"] = None

	# Return all nodes to normal
	ret = self.stopall(None)
	if not ret:
	return self.failure("Stop all nodes failed")

	return self.success()

	def find_leaks(self):
	# Check for leaks
	leaked = []
	self.stop = StopTest(self.CM)

	for node in self.Env["nodes"]:
	(rc, ps_out) = self.rsh(node, "ps u --ppid `pidofproc aisexec`", None)
	rc = self.stop(node)
	if not rc:
	self.failure("Couldn't shut down %s" % node)

	rc = self.rsh(node, "grep -e indirectly.lost:.[1-9] -e definitely.lost:.[1-9] -e (ERROR\|error).SUMMARY:.[1-9].*errors %s" % self.logger.logPat, 0)
	if rc != 1:
	leaked.append(node)
	self.failure("Valgrind errors detected on %s" % node)
	for line in ps_out:
	self.logger.log(line)
	(rc, output) = self.rsh(node, "grep -e lost: -e SUMMARY: %s" % self.logger.logPat, None)
	for line in output:
	self.logger.log(line)
	(rc, output) = self.rsh(node, "cat %s" % self.logger.logPat, None)
	for line in output:
	self.debug(line)

	self.rsh(node, "rm -f %s" % self.logger.logPat, None)
	return leaked

	def __call__(self, node):
	leaked = self.find_leaks()
	if len(leaked) > 0:
	return self.failure("Nodes %s leaked" % repr(leaked))

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"cib.: \\\\\\\\\\\\\",
	r"cib.: . avoid confusing Valgrind",
	r"HA_VALGRIND_ENABLED",
	]


	class StandbyLoopTest(ValgrindTest):
	'''Check for memory leaks by putting a node in and out of standby for an hour'''
	def __init__(self, cm):
	ValgrindTest.__init__(self,cm)
	self.name = "StandbyLoop"

	def __call__(self, node):

	lpc = 0
	delay = 2
	failed = 0
	done = time.time() + self.Env["loop-minutes"] * 60
	while time.time() <= done and not failed:
	lpc = lpc + 1

	time.sleep(delay)
	if not self.CM.SetStandbyMode(node, "on"):
	self.failure("can't set node %s to standby mode" % node)
	failed = lpc

	time.sleep(delay)
	if not self.CM.SetStandbyMode(node, "off"):
	self.failure("can't set node %s to active mode" % node)
	failed = lpc

	leaked = self.find_leaks()
	if failed:
	return self.failure("Iteration %d failed" % failed)
	elif len(leaked) > 0:
	return self.failure("Nodes %s leaked" % repr(leaked))

	return self.success()

	AllTestClasses.append(StandbyLoopTest)


	class BandwidthTest(CTSTest):
	# Tests should not be cluster-manager-specific
	# If you need to find out cluster manager configuration to do this, then
	# it should be added to the generic cluster manager API.
	'''Test the bandwidth which heartbeat uses'''
	def __init__(self, cm):
	CTSTest.__init__(self, cm)
	self.name = "Bandwidth"
	self.start = StartTest(cm)
	self.__setitem__("min",0)
	self.__setitem__("max",0)
	self.__setitem__("totalbandwidth",0)
	(handle, self.tempfile) = tempfile.mkstemp(".cts")
	os.close(handle)
	self.startall = SimulStartLite(cm)

	def __call__(self, node):
	'''Perform the Bandwidth test'''
	self.incr("calls")

	if self.CM.upcount() < 1:
	return self.skipped()

	Path = self.CM.InternalCommConfig()
	if "ip" not in Path["mediatype"]:
	return self.skipped()

	port = Path["port"][0]
	port = int(port)

	ret = self.startall(None)
	if not ret:
	return self.failure("Test setup failed")
	time.sleep(5) # We get extra messages right after startup.

	fstmpfile = "/var/run/band_estimate"
	dumpcmd = "tcpdump -p -n -c 102 -i any udp port %d > %s 2>&1" \
	% (port, fstmpfile)

	rc = self.rsh(node, dumpcmd)
	if rc == 0:
	farfile = "root@%s:%s" % (node, fstmpfile)
	self.rsh.cp(farfile, self.tempfile)
	Bandwidth = self.countbandwidth(self.tempfile)
	if not Bandwidth:
	self.logger.log("Could not compute bandwidth.")
	return self.success()
	intband = int(Bandwidth + 0.5)
	self.logger.log("...bandwidth: %d bits/sec" % intband)
	self.Stats["totalbandwidth"] = self.Stats["totalbandwidth"] + Bandwidth
	if self.Stats["min"] == 0:
	self.Stats["min"] = Bandwidth
	if Bandwidth > self.Stats["max"]:
	self.Stats["max"] = Bandwidth
	if Bandwidth < self.Stats["min"]:
	self.Stats["min"] = Bandwidth
	self.rsh(node, "rm -f %s" % fstmpfile)
	os.unlink(self.tempfile)
	return self.success()
	else:
	return self.failure("no response from tcpdump command [%d]!" % rc)

	def countbandwidth(self, file):
	fp = open(file, "r")
	fp.seek(0)
	count = 0
	sum = 0
	while 1:
	line = fp.readline()
	if not line:
	return None
	if re.search("udp",line) or re.search("UDP,", line):
	count = count + 1
	linesplit = string.split(line," ")
	for j in range(len(linesplit)-1):
	if linesplit[j] == "udp": break
	if linesplit[j] == "length:": break

	try:
	sum = sum + int(linesplit[j+1])
	except ValueError:
	self.logger.log("Invalid tcpdump line: %s" % line)
	return None
	T1 = linesplit[0]
	timesplit = string.split(T1,":")
	time2split = string.split(timesplit[2],".")
	time1 = (int(timesplit[0])60+int(timesplit[1]))60+int(time2split[0])+int(time2split[1])*0.000001
	break

	while count < 100:
	line = fp.readline()
	if not line:
	return None
	if re.search("udp",line) or re.search("UDP,", line):
	count = count+1
	linessplit = string.split(line," ")
	for j in range(len(linessplit)-1):
	if linessplit[j] == "udp": break
	if linesplit[j] == "length:": break
	try:
	sum = int(linessplit[j+1]) + sum
	except ValueError:
	self.logger.log("Invalid tcpdump line: %s" % line)
	return None

	T2 = linessplit[0]
	timesplit = string.split(T2,":")
	time2split = string.split(timesplit[2],".")
	time2 = (int(timesplit[0])60+int(timesplit[1]))60+int(time2split[0])+int(time2split[1])*0.000001
	time = time2-time1
	if (time <= 0):
	return 0
	return (sum*8)/time

	def is_applicable(self):
	'''BandwidthTest never applicable'''
	return 0

	AllTestClasses.append(BandwidthTest)


	###################################################################
	class MaintenanceMode(CTSTest):
	###################################################################
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "MaintenanceMode"
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.max = 30
	#self.is_unsafe = 1
	self.benchmark = 1
	self.action = "asyncmon"
	self.interval = 0
	self.rid = "maintenanceDummy"

	def toggleMaintenanceMode(self, node, action):
	pats = []
	pats.append(self.templates["Pat:DC_IDLE"])

	# fail the resource right after turning Maintenance mode on
	# verify it is not recovered until maintenance mode is turned off
	if action == "On":
	pats.append(r"pengine.:\s+warning:.Processing failed op %s for %s on" % (self.action, self.rid))
	else:
	pats.append(self.templates["Pat:RscOpOK"] % (self.rid, "stop_0"))
	pats.append(self.templates["Pat:RscOpOK"] % (self.rid, "start_0"))

	watch = self.create_watch(pats, 60)
	watch.setwatch()

	self.debug("Turning maintenance mode %s" % action)
	self.rsh(node, self.templates["MaintenanceMode%s" % (action)])
	if (action == "On"):
	self.rsh(node, "crm_resource -V -F -r %s -H %s &>/dev/null" % (self.rid, node))

	self.set_timer("recover%s" % (action))
	watch.lookforall()
	self.log_timer("recover%s" % (action))
	if watch.unmatched:
	self.debug("Failed to find patterns when turning maintenance mode %s" % action)
	return repr(watch.unmatched)

	return ""

	def insertMaintenanceDummy(self, node):
	pats = []
	pats.append(("%s.*" % node) + (self.templates["Pat:RscOpOK"] % (self.rid, "start_0")))

	watch = self.create_watch(pats, 60)
	watch.setwatch()

	self.CM.AddDummyRsc(node, self.rid)

	self.set_timer("addDummy")
	watch.lookforall()
	self.log_timer("addDummy")

	if watch.unmatched:
	self.debug("Failed to find patterns when adding maintenance dummy resource")
	return repr(watch.unmatched)
	return ""

	def removeMaintenanceDummy(self, node):
	pats = []
	pats.append(self.templates["Pat:RscOpOK"] % (self.rid, "stop_0"))

	watch = self.create_watch(pats, 60)
	watch.setwatch()
	self.CM.RemoveDummyRsc(node, self.rid)

	self.set_timer("removeDummy")
	watch.lookforall()
	self.log_timer("removeDummy")

	if watch.unmatched:
	self.debug("Failed to find patterns when removing maintenance dummy resource")
	return repr(watch.unmatched)
	return ""

	def managedRscList(self, node):
	rscList = []
	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	tmp = AuditResource(self.CM, line)
	if tmp.managed():
	rscList.append(tmp.id)

	return rscList

	def verifyResources(self, node, rscList, managed):
	managedList = list(rscList)
	managed_str = "managed"
	if not managed:
	managed_str = "unmanaged"

	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	tmp = AuditResource(self.CM, line)
	if managed and not tmp.managed():
	continue
	elif not managed and tmp.managed():
	continue
	elif managedList.count(tmp.id):
	managedList.remove(tmp.id)

	if len(managedList) == 0:
	self.debug("Found all %s resources on %s" % (managed_str, node))
	return True

	self.logger.log("Could not find all %s resources on %s. %s" % (managed_str, node, managedList))
	return False

	def __call__(self, node):
	'''Perform the 'MaintenanceMode' test. '''
	self.incr("calls")
	verify_managed = False
	verify_unmanaged = False
	failPat = ""

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	# get a list of all the managed resources. We use this list
	# after enabling maintenance mode to verify all managed resources
	# become un-managed. After maintenance mode is turned off, we use
	# this list to verify all the resources become managed again.
	managedResources = self.managedRscList(node)
	if len(managedResources) == 0:
	self.logger.log("No managed resources on %s" % node)
	return self.skipped()

	# insert a fake resource we can fail during maintenance mode
	# so we can verify recovery does not take place until after maintenance
	# mode is disabled.
	failPat = failPat + self.insertMaintenanceDummy(node)

	# toggle maintenance mode ON, then fail dummy resource.
	failPat = failPat + self.toggleMaintenanceMode(node, "On")

	# verify all the resources are now unmanaged
	if self.verifyResources(node, managedResources, False):
	verify_unmanaged = True

	# Toggle maintenance mode OFF, verify dummy is recovered.
	failPat = failPat + self.toggleMaintenanceMode(node, "Off")

	# verify all the resources are now managed again
	if self.verifyResources(node, managedResources, True):
	verify_managed = True

	# Remove our maintenance dummy resource.
	failPat = failPat + self.removeMaintenanceDummy(node)

	self.CM.cluster_stable()

	if failPat != "":
	return self.failure("Unmatched patterns: %s" % (failPat))
	elif verify_unmanaged is False:
	return self.failure("Failed to verify resources became unmanaged during maintenance mode")
	elif verify_managed is False:
	return self.failure("Failed to verify resources switched back to managed after disabling maintenance mode")

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"Updating failcount for %s" % self.rid,
	r"pengine.: Recover %s\s$.*$" % self.rid,
	r"Unknown operation: fail",
	r"(ERROR\|error): sending stonithRA op to stonithd failed.",
	self.templates["Pat:RscOpOK"] % (self.rid, ("%s_%d" % (self.action, self.interval))),
	r"(ERROR\|error).: Action %s_%s_%d . initiated outside of a transition" % (self.rid, self.action, self.interval),
	]

	AllTestClasses.append(MaintenanceMode)


	class ResourceRecover(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "ResourceRecover"
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.max = 30
	self.rid = None
	self.rid_alt = None
	#self.is_unsafe = 1
	self.benchmark = 1

	# these are the values used for the new LRM API call
	self.action = "asyncmon"
	self.interval = 0

	def __call__(self, node):
	'''Perform the 'ResourceRecover' test. '''
	self.incr("calls")

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	resourcelist = self.CM.active_resources(node)
	# if there are no resourcelist, return directly
	if len(resourcelist) == 0:
	self.logger.log("No active resources on %s" % node)
	return self.skipped()

	self.rid = self.Env.RandomGen.choice(resourcelist)
	self.rid_alt = self.rid

	rsc = None
	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	tmp = AuditResource(self.CM, line)
	if tmp.id == self.rid:
	rsc = tmp
	# Handle anonymous clones that get renamed
	self.rid = rsc.clone_id
	break

	if not rsc:
	return self.failure("Could not find %s in the resource list" % self.rid)

	self.debug("Shooting %s aka. %s" % (rsc.clone_id, rsc.id))

	pats = []
	pats.append(r"pengine.:\s+warning:.Processing failed op %s for (%s\|%s) on" % (self.action,
	rsc.id, rsc.clone_id))

	if rsc.managed():
	pats.append(self.templates["Pat:RscOpOK"] % (self.rid, "stop_0"))
	if rsc.unique():
	pats.append(self.templates["Pat:RscOpOK"] % (self.rid, "start_0"))
	else:
	# Anonymous clones may get restarted with a different clone number
	pats.append(self.templates["Pat:RscOpOK"] % (".*", "start_0"))

	watch = self.create_watch(pats, 60)
	watch.setwatch()

	self.rsh(node, "crm_resource -V -F -r %s -H %s &>/dev/null" % (self.rid, node))

	self.set_timer("recover")
	watch.lookforall()
	self.log_timer("recover")

	self.CM.cluster_stable()
	recovered = self.CM.ResourceLocation(self.rid)

	if watch.unmatched:
	return self.failure("Patterns not found: %s" % repr(watch.unmatched))

	elif rsc.unique() and len(recovered) > 1:
	return self.failure("%s is now active on more than one node: %s"%(self.rid, repr(recovered)))

	elif len(recovered) > 0:
	self.debug("%s is running on: %s" % (self.rid, repr(recovered)))

	elif rsc.managed():
	return self.failure("%s was not recovered and is inactive" % self.rid)

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"Updating failcount for %s" % self.rid,
	r"pengine.: Recover (%s\|%s)\s$.*$" % (self.rid, self.rid_alt),
	r"Unknown operation: fail",
	r"(ERROR\|error): sending stonithRA op to stonithd failed.",
	self.templates["Pat:RscOpOK"] % (self.rid, ("%s_%d" % (self.action, self.interval))),
	r"(ERROR\|error).: Action %s_%s_%d . initiated outside of a transition" % (self.rid, self.action, self.interval),
	]

	AllTestClasses.append(ResourceRecover)


	class ComponentFail(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "ComponentFail"
	# TODO make this work correctly in docker.
	self.is_docker_unsafe = 1
	self.startall = SimulStartLite(cm)
	self.complist = cm.Components()
	self.patterns = []
	self.okerrpatterns = []
	self.is_unsafe = 1

	def __call__(self, node):
	'''Perform the 'ComponentFail' test. '''
	self.incr("calls")
	self.patterns = []
	self.okerrpatterns = []

	# start all nodes
	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	if not self.CM.cluster_stable(self.Env["StableTime"]):
	return self.failure("Setup failed - unstable")

	node_is_dc = self.CM.is_node_dc(node, None)

	# select a component to kill
	chosen = self.Env.RandomGen.choice(self.complist)
	while chosen.dc_only == 1 and node_is_dc == 0:
	chosen = self.Env.RandomGen.choice(self.complist)

	self.debug("...component %s (dc=%d,boot=%d)" % (chosen.name, node_is_dc,chosen.triggersreboot))
	self.incr(chosen.name)

	if chosen.name != "aisexec" and chosen.name != "corosync":
	if self.Env["Name"] != "crm-lha" or chosen.name != "pengine":
	self.patterns.append(self.templates["Pat:ChildKilled"] %(node, chosen.name))
	self.patterns.append(self.templates["Pat:ChildRespawn"] %(node, chosen.name))

	self.patterns.extend(chosen.pats)
	if node_is_dc:
	self.patterns.extend(chosen.dc_pats)

	# In an ideal world, this next stuff should be in the "chosen" object as a member function
	if self.Env["Name"] == "crm-lha" and chosen.triggersreboot:
	# Make sure the node goes down and then comes back up if it should reboot...
	for other in self.Env["nodes"]:
	if other != node:
	self.patterns.append(self.templates["Pat:They_stopped"] %(other, self.CM.key_for_node(node)))
	self.patterns.append(self.templates["Pat:Slave_started"] % node)
	self.patterns.append(self.templates["Pat:Local_started"] % node)

	if chosen.dc_only:
	# Sometimes these will be in the log, and sometimes they won't...
	self.okerrpatterns.append("%s .Process %s:. exited" % (node, chosen.name))
	self.okerrpatterns.append("%s .I_ERROR.crmdManagedChildDied" % node)
	self.okerrpatterns.append("%s .*The %s subsystem terminated unexpectedly" % (node, chosen.name))
	self.okerrpatterns.append("(ERROR\|error): Client .* exited with return code")
	else:
	# Sometimes this won't be in the log...
	self.okerrpatterns.append(self.templates["Pat:ChildKilled"] %(node, chosen.name))
	self.okerrpatterns.append(self.templates["Pat:ChildRespawn"] %(node, chosen.name))
	self.okerrpatterns.append(self.templates["Pat:ChildExit"])

	if chosen.name == "stonith":
	# Ignore actions for STONITH resources
	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	r = AuditResource(self.CM, line)
	if r.rclass == "stonith":
	self.okerrpatterns.append(self.templates["Pat:Fencing_recover"] % r.id)

	# supply a copy so self.patterns doesnt end up empty
	tmpPats = []
	tmpPats.extend(self.patterns)
	self.patterns.extend(chosen.badnews_ignore)

	# Look for STONITH ops, depending on Env["at-boot"] we might need to change the nodes status
	stonithPats = []
	stonithPats.append(self.templates["Pat:Fencing_ok"] % node)
	stonith = self.create_watch(stonithPats, 0)
	stonith.setwatch()

	# set the watch for stable
	watch = self.create_watch(
	tmpPats, self.Env["DeadTime"] + self.Env["StableTime"] + self.Env["StartTime"])
	watch.setwatch()

	# kill the component
	chosen.kill(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	self.debug("Waiting for any STONITHd node to come back up")
	self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)

	self.debug("Waiting for the cluster to re-stabilize with all nodes")
	self.CM.cluster_stable(self.Env["StartTime"])

	self.debug("Checking if %s was shot" % node)
	shot = stonith.look(60)
	if shot:
	self.debug("Found: " + repr(shot))
	self.okerrpatterns.append(self.templates["Pat:Fencing_start"] % node)

	if self.Env["at-boot"] == 0:
	self.CM.ShouldBeStatus[node] = "down"

	# If fencing occurred, chances are many (if not all) the expected logs
	# will not be sent - or will be lost when the node reboots
	return self.success()

	# check for logs indicating a graceful recovery
	matched = watch.lookforall(allow_multiple_matches=1)
	if watch.unmatched:
	self.logger.log("Patterns not found: " + repr(watch.unmatched))

	self.debug("Waiting for the cluster to re-stabilize with all nodes")
	is_stable = self.CM.cluster_stable(self.Env["StartTime"])

	if not matched:
	return self.failure("Didn't find all expected %s patterns" % chosen.name)
	elif not is_stable:
	return self.failure("Cluster did not become stable after killing %s" % chosen.name)

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	# Note that okerrpatterns refers to the last time we ran this test
	# The good news is that this works fine for us...
	self.okerrpatterns.extend(self.patterns)
	return self.okerrpatterns

	AllTestClasses.append(ComponentFail)


	class SplitBrainTest(CTSTest):
	'''It is used to test split-brain. when the path between the two nodes break
	check the two nodes both take over the resource'''
	def __init__(self,cm):
	CTSTest.__init__(self,cm)
	self.name = "SplitBrain"
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.is_experimental = 1

	def isolate_partition(self, partition):
	other_nodes = []
	other_nodes.extend(self.Env["nodes"])

	for node in partition:
	try:
	other_nodes.remove(node)
	except ValueError:
	self.logger.log("Node "+node+" not in " + repr(self.Env["nodes"]) + " from " +repr(partition))

	if len(other_nodes) == 0:
	return 1

	self.debug("Creating partition: " + repr(partition))
	self.debug("Everyone else: " + repr(other_nodes))

	for node in partition:
	if not self.CM.isolate_node(node, other_nodes):
	self.logger.log("Could not isolate %s" % node)
	return 0

	return 1

	def heal_partition(self, partition):
	other_nodes = []
	other_nodes.extend(self.Env["nodes"])

	for node in partition:
	try:
	other_nodes.remove(node)
	except ValueError:
	self.logger.log("Node "+node+" not in " + repr(self.Env["nodes"]))

	if len(other_nodes) == 0:
	return 1

	self.debug("Healing partition: " + repr(partition))
	self.debug("Everyone else: " + repr(other_nodes))

	for node in partition:
	self.CM.unisolate_node(node, other_nodes)

	def __call__(self, node):
	'''Perform split-brain test'''
	self.incr("calls")
	self.passed = 1
	partitions = {}

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	while 1:
	# Retry until we get multiple partitions
	partitions = {}
	p_max = len(self.Env["nodes"])
	for node in self.Env["nodes"]:
	p = self.Env.RandomGen.randint(1, p_max)
	if not p in partitions:
	partitions[p] = []
	partitions[p].append(node)
	p_max = len(partitions.keys())
	if p_max > 1:
	break
	# else, try again

	self.debug("Created %d partitions" % p_max)
	for key in list(partitions.keys()):
	self.debug("Partition["+str(key)+"]:\t"+repr(partitions[key]))

	# Disabling STONITH to reduce test complexity for now
	self.rsh(node, "crm_attribute -V -n stonith-enabled -v false")

	for key in list(partitions.keys()):
	self.isolate_partition(partitions[key])

	count = 30
	while count > 0:
	if len(self.CM.find_partitions()) != p_max:
	time.sleep(10)
	else:
	break
	else:
	self.failure("Expected partitions were not created")

	# Target number of partitions formed - wait for stability
	if not self.CM.cluster_stable():
	self.failure("Partitioned cluster not stable")

	# Now audit the cluster state
	self.CM.partitions_expected = p_max
	if not self.audit():
	self.failure("Audits failed")
	self.CM.partitions_expected = 1

	# And heal them again
	for key in list(partitions.keys()):
	self.heal_partition(partitions[key])

	# Wait for a single partition to form
	count = 30
	while count > 0:
	if len(self.CM.find_partitions()) != 1:
	time.sleep(10)
	count -= 1
	else:
	break
	else:
	self.failure("Cluster did not reform")

	# Wait for it to have the right number of members
	count = 30
	while count > 0:
	members = []

	partitions = self.CM.find_partitions()
	if len(partitions) > 0:
	members = partitions[0].split()

	if len(members) != len(self.Env["nodes"]):
	time.sleep(10)
	count -= 1
	else:
	break
	else:
	self.failure("Cluster did not completely reform")

	# Wait up to 20 minutes - the delay is more preferable than
	# trying to continue with in a messed up state
	if not self.CM.cluster_stable(1200):
	self.failure("Reformed cluster not stable")
	- answer = raw_input('Continue? [nY]')
	+ if self.Env["continue"] == 1:
	+ answer = "Y"
	+ else:
	+ try:
	+ answer = raw_input('Continue? [nY]')
	+ except EOFError, e:
	+ answer = "n"
	if answer and answer == "n":
	raise ValueError("Reformed cluster not stable")

	# Turn fencing back on
	if self.Env["DoFencing"]:
	self.rsh(node, "crm_attribute -V -D -n stonith-enabled")

	self.CM.cluster_stable()

	if self.passed:
	return self.success()
	return self.failure("See previous errors")

	def errorstoignore(self):
	'''Return list of errors which are 'normal' and should be ignored'''
	return [
	r"Another DC detected:",
	r"(ERROR\|error).: .Application of an update diff failed",
	r"crmd.:.not in our membership list",
	r"CRIT:.node.returning after partition",
	]

	def is_applicable(self):
	if not self.is_applicable_common():
	return 0
	return len(self.Env["nodes"]) > 2

	AllTestClasses.append(SplitBrainTest)


	class Reattach(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Reattach"
	self.startall = SimulStartLite(cm)
	self.restart1 = RestartTest(cm)
	self.stopall = SimulStopLite(cm)
	self.is_unsafe = 0 # Handled by canrunnow()

	def _is_managed(self, node):
	is_managed = self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -Q -G -d true", 1)
	is_managed = is_managed[:-1] # Strip off the newline
	return is_managed == "true"

	def _set_unmanaged(self, node):
	self.debug("Disable resource management")
	self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -v false")

	def _set_managed(self, node):
	self.debug("Re-enable resource management")
	self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -D")

	def setup(self, node):
	attempt = 0
	if not self.startall(None):
	return None

	# Make sure we are really _really_ stable and that all
	# resources, including those that depend on transient node
	# attributes, are started
	while not self.CM.cluster_stable(double_check=True):
	if attempt < 5:
	attempt += 1
	self.debug("Not stable yet, re-testing")
	else:
	self.logger.log("Cluster is not stable")
	return None

	return 1

	def teardown(self, node):

	# Make sure 'node' is up
	start = StartTest(self.CM)
	start(node)

	if not self._is_managed(node):
	self.logger.log("Attempting to re-enable resource management on %s" % node)
	self._set_managed(node)
	self.CM.cluster_stable()
	if not self._is_managed(node):
	self.logger.log("Could not re-enable resource management")
	return 0

	return 1

	def canrunnow(self, node):
	'''Return TRUE if we can meaningfully run right now'''
	if self.find_ocfs2_resources(node):
	self.logger.log("Detach/Reattach scenarios are not possible with OCFS2 services present")
	return 0
	return 1

	def __call__(self, node):
	self.incr("calls")

	pats = []
	# Conveniently, pengine will display this message when disabling management,
	# even if fencing is not enabled, so we can rely on it.
	managed = self.create_watch(["Delaying fencing operations"], 60)
	managed.setwatch()

	self._set_unmanaged(node)

	if not managed.lookforall():
	self.logger.log("Patterns not found: " + repr(managed.unmatched))
	return self.failure("Resource management not disabled")

	pats = []
	pats.append(self.templates["Pat:RscOpOK"] % (".*", "start"))
	pats.append(self.templates["Pat:RscOpOK"] % (".*", "stop"))
	pats.append(self.templates["Pat:RscOpOK"] % (".*", "promote"))
	pats.append(self.templates["Pat:RscOpOK"] % (".*", "demote"))
	pats.append(self.templates["Pat:RscOpOK"] % (".*", "migrate"))

	watch = self.create_watch(pats, 60, "ShutdownActivity")
	watch.setwatch()

	self.debug("Shutting down the cluster")
	ret = self.stopall(None)
	if not ret:
	self._set_managed(node)
	return self.failure("Couldn't shut down the cluster")

	self.debug("Bringing the cluster back up")
	ret = self.startall(None)
	time.sleep(5) # allow ping to update the CIB
	if not ret:
	self._set_managed(node)
	return self.failure("Couldn't restart the cluster")

	if self.local_badnews("ResourceActivity:", watch):
	self._set_managed(node)
	return self.failure("Resources stopped or started during cluster restart")

	watch = self.create_watch(pats, 60, "StartupActivity")
	watch.setwatch()

	# Re-enable resource management (and verify it happened).
	self._set_managed(node)
	self.CM.cluster_stable()
	if not self._is_managed(node):
	return self.failure("Could not re-enable resource management")

	# Ignore actions for STONITH resources
	ignore = []
	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	r = AuditResource(self.CM, line)
	if r.rclass == "stonith":

	self.debug("Ignoring start actions for %s" % r.id)
	ignore.append(self.templates["Pat:RscOpOK"] % (r.id, "start_0"))

	if self.local_badnews("ResourceActivity:", watch, ignore):
	return self.failure("Resources stopped or started after resource management was re-enabled")

	return ret

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"resources were active at shutdown",
	]

	def is_applicable(self):
	if self.Env["Name"] == "crm-lha":
	return None
	return 1

	AllTestClasses.append(Reattach)


	class SpecialTest1(CTSTest):
	'''Set up a custom test to cause quorum failure issues for Andrew'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SpecialTest1"
	self.startall = SimulStartLite(cm)
	self.restart1 = RestartTest(cm)
	self.stopall = SimulStopLite(cm)

	def __call__(self, node):
	'''Perform the 'SpecialTest1' test for Andrew. '''
	self.incr("calls")

	# Shut down all the nodes...
	ret = self.stopall(None)
	if not ret:
	return self.failure("Could not stop all nodes")

	# Test config recovery when the other nodes come up
	self.rsh(node, "rm -f "+CTSvars.CRM_CONFIG_DIR+"/cib*")

	# Start the selected node
	ret = self.restart1(node)
	if not ret:
	return self.failure("Could not start "+node)

	# Start all remaining nodes
	ret = self.startall(None)
	if not ret:
	return self.failure("Could not start the remaining nodes")

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	# Errors that occur as a result of the CIB being wiped
	return [
	r"error.*: v1 patchset error, patch failed to apply: Application of an update diff failed",
	r"error.*: Resource start-up disabled since no STONITH resources have been defined",
	r"error.*: Either configure some or disable STONITH with the stonith-enabled option",
	r"error.*: NOTE: Clusters with shared data need STONITH to ensure data integrity",
	]

	AllTestClasses.append(SpecialTest1)


	class HAETest(CTSTest):
	'''Set up a custom test to cause quorum failure issues for Andrew'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "HAETest"
	self.stopall = SimulStopLite(cm)
	self.startall = SimulStartLite(cm)
	self.is_loop = 1

	def setup(self, node):
	# Start all remaining nodes
	ret = self.startall(None)
	if not ret:
	return self.failure("Couldn't start all nodes")
	return self.success()

	def teardown(self, node):
	# Stop everything
	ret = self.stopall(None)
	if not ret:
	return self.failure("Couldn't stop all nodes")
	return self.success()

	def wait_on_state(self, node, resource, expected_clones, attempts=240):
	while attempts > 0:
	active = 0
	(rc, lines) = self.rsh(node, "crm_resource -r %s -W -Q" % resource, stdout=None)

	# Hack until crm_resource does the right thing
	if rc == 0 and lines:
	active = len(lines)

	if len(lines) == expected_clones:
	return 1

	elif rc == 1:
	self.debug("Resource %s is still inactive" % resource)

	elif rc == 234:
	self.logger.log("Unknown resource %s" % resource)
	return 0

	elif rc == 246:
	self.logger.log("Cluster is inactive")
	return 0

	elif rc != 0:
	self.logger.log("Call to crm_resource failed, rc=%d" % rc)
	return 0

	else:
	self.debug("Resource %s is active on %d times instead of %d" % (resource, active, expected_clones))

	attempts -= 1
	time.sleep(1)

	return 0

	def find_dlm(self, node):
	self.r_dlm = None

	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	r = AuditResource(self.CM, line)
	if r.rtype == "controld" and r.parent != "NA":
	self.debug("Found dlm: %s" % self.r_dlm)
	self.r_dlm = r.parent
	return 1
	return 0

	def find_hae_resources(self, node):
	self.r_dlm = None
	self.r_o2cb = None
	self.r_ocfs2 = []

	if self.find_dlm(node):
	self.find_ocfs2_resources(node)

	def is_applicable(self):
	if not self.is_applicable_common():
	return 0
	if self.Env["Schema"] == "hae":
	return 1
	return None


	class HAERoleTest(HAETest):
	def __init__(self, cm):
	'''Lars' mount/unmount test for the HA extension. '''
	HAETest.__init__(self,cm)
	self.name = "HAERoleTest"

	def change_state(self, node, resource, target):
	rc = self.rsh(node, "crm_resource -V -r %s -p target-role -v %s --meta" % (resource, target))
	return rc

	def __call__(self, node):
	self.incr("calls")
	lpc = 0
	failed = 0
	delay = 2
	done = time.time() + self.Env["loop-minutes"]*60
	self.find_hae_resources(node)

	clone_max = len(self.Env["nodes"])
	while time.time() <= done and not failed:
	lpc = lpc + 1

	self.change_state(node, self.r_dlm, "Stopped")
	if not self.wait_on_state(node, self.r_dlm, 0):
	self.failure("%s did not go down correctly" % self.r_dlm)
	failed = lpc

	self.change_state(node, self.r_dlm, "Started")
	if not self.wait_on_state(node, self.r_dlm, clone_max):
	self.failure("%s did not come up correctly" % self.r_dlm)
	failed = lpc

	if not self.wait_on_state(node, self.r_o2cb, clone_max):
	self.failure("%s did not come up correctly" % self.r_o2cb)
	failed = lpc

	for fs in self.r_ocfs2:
	if not self.wait_on_state(node, fs, clone_max):
	self.failure("%s did not come up correctly" % fs)
	failed = lpc

	if failed:
	return self.failure("iteration %d failed" % failed)
	return self.success()

	AllTestClasses.append(HAERoleTest)


	class HAEStandbyTest(HAETest):
	'''Set up a custom test to cause quorum failure issues for Andrew'''
	def __init__(self, cm):
	HAETest.__init__(self,cm)
	self.name = "HAEStandbyTest"

	def change_state(self, node, resource, target):
	rc = self.rsh(node, "crm_standby -V -l reboot -v %s" % (target))
	return rc

	def __call__(self, node):
	self.incr("calls")

	lpc = 0
	failed = 0
	done = time.time() + self.Env["loop-minutes"]*60
	self.find_hae_resources(node)

	clone_max = len(self.Env["nodes"])
	while time.time() <= done and not failed:
	lpc = lpc + 1

	self.change_state(node, self.r_dlm, "true")
	if not self.wait_on_state(node, self.r_dlm, clone_max-1):
	self.failure("%s did not go down correctly" % self.r_dlm)
	failed = lpc

	self.change_state(node, self.r_dlm, "false")
	if not self.wait_on_state(node, self.r_dlm, clone_max):
	self.failure("%s did not come up correctly" % self.r_dlm)
	failed = lpc

	if not self.wait_on_state(node, self.r_o2cb, clone_max):
	self.failure("%s did not come up correctly" % self.r_o2cb)
	failed = lpc

	for fs in self.r_ocfs2:
	if not self.wait_on_state(node, fs, clone_max):
	self.failure("%s did not come up correctly" % fs)
	failed = lpc

	if failed:
	return self.failure("iteration %d failed" % failed)
	return self.success()

	AllTestClasses.append(HAEStandbyTest)


	class NearQuorumPointTest(CTSTest):
	'''
	This test brings larger clusters near the quorum point (50%).
	In addition, it will test doing starts and stops at the same time.

	Here is how I think it should work:
	- loop over the nodes and decide randomly which will be up and which
	will be down Use a 50% probability for each of up/down.
	- figure out what to do to get into that state from the current state
	- in parallel, bring up those going up and bring those going down.
	'''

	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "NearQuorumPoint"

	def __call__(self, dummy):
	'''Perform the 'NearQuorumPoint' test. '''
	self.incr("calls")
	startset = []
	stopset = []

	stonith = self.CM.prepare_fencing_watcher("NearQuorumPoint")
	#decide what to do with each node
	for node in self.Env["nodes"]:
	action = self.Env.RandomGen.choice(["start","stop"])
	#action = self.Env.RandomGen.choice(["start","stop","no change"])
	if action == "start" :
	startset.append(node)
	elif action == "stop" :
	stopset.append(node)

	self.debug("start nodes:" + repr(startset))
	self.debug("stop nodes:" + repr(stopset))

	#add search patterns
	watchpats = [ ]
	for node in stopset:
	if self.CM.ShouldBeStatus[node] == "up":
	watchpats.append(self.templates["Pat:We_stopped"] % node)

	for node in startset:
	if self.CM.ShouldBeStatus[node] == "down":
	#watchpats.append(self.templates["Pat:Slave_started"] % node)
	watchpats.append(self.templates["Pat:Local_started"] % node)
	else:
	for stopping in stopset:
	if self.CM.ShouldBeStatus[stopping] == "up":
	watchpats.append(self.templates["Pat:They_stopped"] % (node, self.CM.key_for_node(stopping)))

	if len(watchpats) == 0:
	return self.skipped()

	if len(startset) != 0:
	watchpats.append(self.templates["Pat:DC_IDLE"])

	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)

	watch.setwatch()

	#begin actions
	for node in stopset:
	if self.CM.ShouldBeStatus[node] == "up":
	self.CM.StopaCMnoBlock(node)

	for node in startset:
	if self.CM.ShouldBeStatus[node] == "down":
	self.CM.StartaCMnoBlock(node)

	#get the result
	if watch.lookforall():
	self.CM.cluster_stable()
	self.CM.fencing_cleanup("NearQuorumPoint", stonith)
	return self.success()

	self.logger.log("Warn: Patterns not found: " + repr(watch.unmatched))

	#get the "bad" nodes
	upnodes = []
	for node in stopset:
	if self.CM.StataCM(node) == 1:
	upnodes.append(node)

	downnodes = []
	for node in startset:
	if self.CM.StataCM(node) == 0:
	downnodes.append(node)

	self.CM.fencing_cleanup("NearQuorumPoint", stonith)
	if upnodes == [] and downnodes == []:
	self.CM.cluster_stable()

	# Make sure they're completely down with no residule
	for node in stopset:
	self.rsh(node, self.templates["StopCmd"])

	return self.success()

	if len(upnodes) > 0:
	self.logger.log("Warn: Unstoppable nodes: " + repr(upnodes))

	if len(downnodes) > 0:
	self.logger.log("Warn: Unstartable nodes: " + repr(downnodes))

	return self.failure()

	def is_applicable(self):
	if self.Env["Name"] == "crm-cman":
	return None
	return 1

	AllTestClasses.append(NearQuorumPointTest)


	class RollingUpgradeTest(CTSTest):
	'''Perform a rolling upgrade of the cluster'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "RollingUpgrade"
	self.start = StartTest(cm)
	self.stop = StopTest(cm)
	self.stopall = SimulStopLite(cm)
	self.startall = SimulStartLite(cm)

	def setup(self, node):
	# Start all remaining nodes
	ret = self.stopall(None)
	if not ret:
	return self.failure("Couldn't stop all nodes")

	for node in self.Env["nodes"]:
	if not self.downgrade(node, None):
	return self.failure("Couldn't downgrade %s" % node)

	ret = self.startall(None)
	if not ret:
	return self.failure("Couldn't start all nodes")
	return self.success()

	def teardown(self, node):
	# Stop everything
	ret = self.stopall(None)
	if not ret:
	return self.failure("Couldn't stop all nodes")

	for node in self.Env["nodes"]:
	if not self.upgrade(node, None):
	return self.failure("Couldn't upgrade %s" % node)

	return self.success()

	def install(self, node, version, start=1, flags="--force"):

	target_dir = "/tmp/rpm-%s" % version
	src_dir = "%s/%s" % (self.Env["rpm-dir"], version)

	self.logger.log("Installing %s on %s with %s" % (version, node, flags))
	if not self.stop(node):
	return self.failure("stop failure: "+node)

	rc = self.rsh(node, "mkdir -p %s" % target_dir)
	rc = self.rsh(node, "rm -f %s/*.rpm" % target_dir)
	(rc, lines) = self.rsh(node, "ls -1 %s/*.rpm" % src_dir, None)
	for line in lines:
	line = line[:-1]
	rc = self.rsh.cp("%s" % (line), "%s:%s/" % (node, target_dir))
	rc = self.rsh(node, "rpm -Uvh %s %s/*.rpm" % (flags, target_dir))

	if start and not self.start(node):
	return self.failure("start failure: "+node)

	return self.success()

	def upgrade(self, node, start=1):
	return self.install(node, self.Env["current-version"], start)

	def downgrade(self, node, start=1):
	return self.install(node, self.Env["previous-version"], start, "--force --nodeps")

	def __call__(self, node):
	'''Perform the 'Rolling Upgrade' test. '''
	self.incr("calls")

	for node in self.Env["nodes"]:
	if self.upgrade(node):
	return self.failure("Couldn't upgrade %s" % node)

	self.CM.cluster_stable()

	return self.success()

	def is_applicable(self):
	if not self.is_applicable_common():
	return None

	if not "rpm-dir" in self.Env.keys():
	return None
	if not "current-version" in self.Env.keys():
	return None
	if not "previous-version" in self.Env.keys():
	return None

	return 1

	# Register RestartTest as a good test to run
	AllTestClasses.append(RollingUpgradeTest)


	class BSC_AddResource(CTSTest):
	'''Add a resource to the cluster'''
	def __init__(self, cm):
	CTSTest.__init__(self, cm)
	self.name = "AddResource"
	self.resource_offset = 0
	self.cib_cmd = """cibadmin -C -o %s -X '%s' """

	def __call__(self, node):
	self.incr("calls")
	self.resource_offset = self.resource_offset + 1

	r_id = "bsc-rsc-%s-%d" % (node, self.resource_offset)
	start_pat = "crmd.%s_start_0.confirmed.*ok"

	patterns = []
	patterns.append(start_pat % r_id)

	watch = self.create_watch(patterns, self.Env["DeadTime"])
	watch.setwatch()

	ip = self.NextIP()
	if not self.make_ip_resource(node, r_id, "ocf", "IPaddr", ip):
	return self.failure("Make resource %s failed" % r_id)

	failed = 0
	watch_result = watch.lookforall()
	if watch.unmatched:
	for regex in watch.unmatched:
	self.logger.log ("Warn: Pattern not found: %s" % (regex))
	failed = 1

	if failed:
	return self.failure("Resource pattern(s) not found")

	if not self.CM.cluster_stable(self.Env["DeadTime"]):
	return self.failure("Unstable cluster")

	return self.success()

	def NextIP(self):
	ip = self.Env["IPBase"]
	if ":" in ip:
	fields = ip.rpartition(":")
	fields[2] = str(hex(int(fields[2], 16)+1))
	print(str(hex(int(f[2], 16)+1)))
	else:
	fields = ip.rpartition('.')
	fields[2] = str(int(fields[2])+1)

	ip = fields[0] + fields[1] + fields[3];
	self.Env["IPBase"] = ip
	return ip.strip()

	def make_ip_resource(self, node, id, rclass, type, ip):
	self.logger.log("Creating %s::%s:%s (%s) on %s" % (rclass,type,id,ip,node))
	rsc_xml="""
	<primitive id="%s" class="%s" type="%s" provider="heartbeat">
	<instance_attributes id="%s"><attributes>
	<nvpair id="%s" name="ip" value="%s"/>
	</attributes></instance_attributes>
	</primitive>""" % (id, rclass, type, id, id, ip)

	node_constraint = """
	<rsc_location id="run_%s" rsc="%s">
	<rule id="pref_run_%s" score="100">
	<expression id="%s_loc_expr" attribute="#uname" operation="eq" value="%s"/>
	</rule>
	</rsc_location>""" % (id, id, id, id, node)

	rc = 0
	(rc, lines) = self.rsh(node, self.cib_cmd % ("constraints", node_constraint), None)
	if rc != 0:
	self.logger.log("Constraint creation failed: %d" % rc)
	return None

	(rc, lines) = self.rsh(node, self.cib_cmd % ("resources", rsc_xml), None)
	if rc != 0:
	self.logger.log("Resource creation failed: %d" % rc)
	return None

	return 1

	def is_applicable(self):
	if self.Env["DoBSC"]:
	return 1
	return None

	AllTestClasses.append(BSC_AddResource)


	class SimulStopLite(CTSTest):
	'''Stop any active nodes ~ simultaneously'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SimulStopLite"

	def __call__(self, dummy):
	'''Perform the 'SimulStopLite' setup work. '''
	self.incr("calls")

	self.debug("Setup: " + self.name)

	# We ignore the "node" parameter...
	watchpats = [ ]

	for node in self.Env["nodes"]:
	if self.CM.ShouldBeStatus[node] == "up":
	self.incr("WasStarted")
	watchpats.append(self.templates["Pat:We_stopped"] % node)
	#if self.Env["use_logd"]:
	# watchpats.append(self.templates["Pat:Logd_stopped"] % node)

	if len(watchpats) == 0:
	self.CM.clear_all_caches()
	return self.success()

	# Stop all the nodes - at about the same time...
	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)

	watch.setwatch()
	self.set_timer()
	for node in self.Env["nodes"]:
	if self.CM.ShouldBeStatus[node] == "up":
	self.CM.StopaCMnoBlock(node)
	if watch.lookforall():
	self.CM.clear_all_caches()

	# Make sure they're completely down with no residule
	for node in self.Env["nodes"]:
	self.rsh(node, self.templates["StopCmd"])

	return self.success()

	did_fail = 0
	up_nodes = []
	for node in self.Env["nodes"]:
	if self.CM.StataCM(node) == 1:
	did_fail = 1
	up_nodes.append(node)

	if did_fail:
	return self.failure("Active nodes exist: " + repr(up_nodes))

	self.logger.log("Warn: All nodes stopped but CTS didnt detect: "
	+ repr(watch.unmatched))

	self.CM.clear_all_caches()
	return self.failure("Missing log message: "+repr(watch.unmatched))

	def is_applicable(self):
	'''SimulStopLite is a setup test and never applicable'''
	return 0


	class SimulStartLite(CTSTest):
	'''Start any stopped nodes ~ simultaneously'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SimulStartLite"

	def __call__(self, dummy):
	'''Perform the 'SimulStartList' setup work. '''
	self.incr("calls")
	self.debug("Setup: " + self.name)

	# We ignore the "node" parameter...
	node_list = []
	for node in self.Env["nodes"]:
	if self.CM.ShouldBeStatus[node] == "down":
	self.incr("WasStopped")
	node_list.append(node)

	self.set_timer()
	while len(node_list) > 0:
	# Repeat until all nodes come up
	watchpats = [ ]

	uppat = self.templates["Pat:Slave_started"]
	if self.CM.upcount() == 0:
	uppat = self.templates["Pat:Local_started"]

	watchpats.append(self.templates["Pat:DC_IDLE"])
	for node in node_list:
	watchpats.append(uppat % node)
	watchpats.append(self.templates["Pat:InfraUp"] % node)
	watchpats.append(self.templates["Pat:PacemakerUp"] % node)

	# Start all the nodes - at about the same time...
	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
	watch.setwatch()

	stonith = self.CM.prepare_fencing_watcher(self.name)

	for node in node_list:
	self.CM.StartaCMnoBlock(node)

	watch.lookforall()

	node_list = self.CM.fencing_cleanup(self.name, stonith)

	# Remove node_list messages from watch.unmatched
	for node in node_list:
	self.logger.debug("Dealing with stonith operations for %s" % repr(node_list))
	if watch.unmatched:
	try:
	watch.unmatched.remove(uppat % node)
	except:
	self.debug("Already matched: %s" % (uppat % node))
	try:
	watch.unmatched.remove(self.templates["Pat:InfraUp"] % node)
	except:
	self.debug("Already matched: %s" % (self.templates["Pat:InfraUp"] % node))
	try:
	watch.unmatched.remove(self.templates["Pat:PacemakerUp"] % node)
	except:
	self.debug("Already matched: %s" % (self.templates["Pat:PacemakerUp"] % node))

	if watch.unmatched:
	for regex in watch.unmatched:
	self.logger.log ("Warn: Startup pattern not found: %s" %(regex))

	if not self.CM.cluster_stable():
	return self.failure("Cluster did not stabilize")

	did_fail = 0
	unstable = []
	for node in self.Env["nodes"]:
	if self.CM.StataCM(node) == 0:
	did_fail = 1
	unstable.append(node)

	if did_fail:
	return self.failure("Unstarted nodes exist: " + repr(unstable))

	unstable = []
	for node in self.Env["nodes"]:
	if not self.CM.node_stable(node):
	did_fail = 1
	unstable.append(node)

	if did_fail:
	return self.failure("Unstable cluster nodes exist: " + repr(unstable))

	return self.success()

	def is_applicable(self):
	'''SimulStartLite is a setup test and never applicable'''
	return 0


	def TestList(cm, audits):
	result = []
	for testclass in AllTestClasses:
	bound_test = testclass(cm)
	if bound_test.is_applicable():
	bound_test.Audits = audits
	result.append(bound_test)
	return result


	class RemoteLXC(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "RemoteLXC"
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.num_containers = 2
	self.is_container = 1
	self.is_docker_unsafe = 1
	self.failed = 0
	self.fail_string = ""

	def start_lxc_simple(self, node):

	# restore any artifacts laying around from a previous test.
	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -R &>/dev/null")

	# generate the containers, put them in the config, add some resources to them
	pats = [ ]
	watch = self.create_watch(pats, 120)
	watch.setwatch()
	pats.append(self.templates["Pat:RscOpOK"] % ("lxc1", "start_0"))
	pats.append(self.templates["Pat:RscOpOK"] % ("lxc2", "start_0"))
	pats.append(self.templates["Pat:RscOpOK"] % ("lxc-ms", "start_0"))
	pats.append(self.templates["Pat:RscOpOK"] % ("lxc-ms", "promote_0"))

	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -g -a -m -s -c %d &>/dev/null" % self.num_containers)
	self.set_timer("remoteSimpleInit")
	watch.lookforall()
	self.log_timer("remoteSimpleInit")
	if watch.unmatched:
	self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
	self.failed = 1

	def cleanup_lxc_simple(self, node):

	pats = [ ]
	# if the test failed, attempt to clean up the cib and libvirt environment
	# as best as possible
	if self.failed == 1:
	# restore libvirt and cib
	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -R &>/dev/null")
	self.rsh(node, "crm_resource -C -r container1 &>/dev/null")
	self.rsh(node, "crm_resource -C -r container2 &>/dev/null")
	self.rsh(node, "crm_resource -C -r lxc1 &>/dev/null")
	self.rsh(node, "crm_resource -C -r lxc2 &>/dev/null")
	self.rsh(node, "crm_resource -C -r lxc-ms &>/dev/null")
	time.sleep(20)
	return

	watch = self.create_watch(pats, 120)
	watch.setwatch()

	pats.append(self.templates["Pat:RscOpOK"] % ("container1", "stop_0"))
	pats.append(self.templates["Pat:RscOpOK"] % ("container2", "stop_0"))

	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -p &>/dev/null")
	self.set_timer("remoteSimpleCleanup")
	watch.lookforall()
	self.log_timer("remoteSimpleCleanup")

	if watch.unmatched:
	self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
	self.failed = 1

	# cleanup libvirt
	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -R &>/dev/null")

	def __call__(self, node):
	'''Perform the 'RemoteLXC' test. '''
	self.incr("calls")

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed, start all nodes failed.")

	rc = self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -v &>/dev/null")
	if rc == 1:
	self.log("Environment test for lxc support failed.")
	return self.skipped()

	self.start_lxc_simple(node)
	self.cleanup_lxc_simple(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	if self.failed == 1:
	return self.failure(self.fail_string)

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"Updating failcount for ping",
	r"pengine.: Recover (ping\|lxc-ms\|container)\s$.*$",
	# The orphaned lxc-ms resource causes an expected transition error
	# that is a result of the pengine not having knowledge that the
	# ms resource used to be a clone. As a result it looks like that
	# resource is running in multiple locations when it shouldn't... But in
	# this instance we know why this error is occurring and that it is expected.
	r"Calculated Transition .* /var/lib/pacemaker/pengine/pe-error",
	r"Resource lxc-ms .* is active on 2 nodes attempting recovery",
	r"Unknown operation: fail",
	r"(ERROR\|error): sending stonithRA op to stonithd failed.",
	]

	AllTestClasses.append(RemoteLXC)


	class RemoteDriver(CTSTest):

	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = self.__class__.__name__
	self.is_docker_unsafe = 1
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.stop = StopTest(cm)
	self.remote_rsc = "remote-rsc"
	self.cib_cmd = """cibadmin -C -o %s -X '%s' """
	self.reset()

	def reset(self):
	self.pcmk_started = 0
	self.failed = False
	self.fail_string = ""
	self.remote_node_added = 0
	self.remote_rsc_added = 0
	self.remote_use_reconnect_interval = self.Env.RandomGen.choice([True,False])

	def fail(self, msg):
	""" Mark test as failed. """

	self.failed = True

	# Always log the failure.
	self.logger.log(msg)

	# Use first failure as test status, as it's likely to be most useful.
	if not self.fail_string:
	self.fail_string = msg

	def get_othernode(self, node):
	for othernode in self.Env["nodes"]:
	if othernode == node:
	# we don't want to try and use the cib that we just shutdown.
	# find a cluster node that is not our soon to be remote-node.
	continue
	else:
	return othernode

	def del_rsc(self, node, rsc):
	othernode = self.get_othernode(node)
	rc = self.rsh(othernode, "crm_resource -D -r %s -t primitive" % (rsc))
	if rc != 0:
	self.fail("Removal of resource '%s' failed" % rsc)

	def add_rsc(self, node, rsc_xml):
	othernode = self.get_othernode(node)
	rc = self.rsh(othernode, self.cib_cmd % ("resources", rsc_xml))
	if rc != 0:
	self.fail("resource creation failed")

	def add_primitive_rsc(self, node):
	rsc_xml = """
	<primitive class="ocf" id="%s" provider="heartbeat" type="Dummy">
	<operations>
	<op id="remote-rsc-monitor-interval-10s" interval="10s" name="monitor"/>
	</operations>
	<meta_attributes id="remote-meta_attributes"/>
	</primitive>""" % (self.remote_rsc)
	self.add_rsc(node, rsc_xml)
	if not self.failed:
	self.remote_rsc_added = 1

	def add_connection_rsc(self, node):
	if self.remote_use_reconnect_interval:
	# use reconnect interval and make sure to set cluster-recheck-interval as well.
	rsc_xml = """
	<primitive class="ocf" id="%s" provider="pacemaker" type="remote">
	<instance_attributes id="remote-instance_attributes"/>
	<instance_attributes id="remote-instance_attributes">
	<nvpair id="remote-instance_attributes-server" name="server" value="%s"/>
	<nvpair id="remote-instance_attributes-reconnect_interval" name="reconnect_interval" value="60s"/>
	</instance_attributes>
	<operations>
	<op id="remote-monitor-interval-60s" interval="60s" name="monitor"/>
	<op id="remote-name-start-interval-0-timeout-120" interval="0" name="start" timeout="60"/>
	</operations>
	</primitive>""" % (self.remote_node, node)
	self.rsh(self.get_othernode(node), self.templates["SetCheckInterval"] % ("45s"))
	else:
	# not using reconnect interval
	rsc_xml = """
	<primitive class="ocf" id="%s" provider="pacemaker" type="remote">
	<instance_attributes id="remote-instance_attributes"/>
	<instance_attributes id="remote-instance_attributes">
	<nvpair id="remote-instance_attributes-server" name="server" value="%s"/>
	</instance_attributes>
	<operations>
	<op id="remote-monitor-interval-60s" interval="60s" name="monitor"/>
	<op id="remote-name-start-interval-0-timeout-120" interval="0" name="start" timeout="120"/>
	</operations>
	</primitive>""" % (self.remote_node, node)

	self.add_rsc(node, rsc_xml)
	if not self.failed:
	self.remote_node_added = 1

	def stop_pcmk_remote(self, node):
	# disable pcmk remote
	for i in range(10):
	rc = self.rsh(node, "service pacemaker_remote stop")
	if rc != 0:
	time.sleep(6)
	else:
	break

	def start_pcmk_remote(self, node):
	for i in range(10):
	rc = self.rsh(node, "service pacemaker_remote start")
	if rc != 0:
	time.sleep(6)
	else:
	self.pcmk_started = 1
	break

	def start_metal(self, node):
	pcmk_started = 0

	# make sure the resource doesn't already exist for some reason
	self.rsh(node, "crm_resource -D -r %s -t primitive" % (self.remote_rsc))
	self.rsh(node, "crm_resource -D -r %s -t primitive" % (self.remote_node))

	if not self.stop(node):
	self.fail("Failed to shutdown cluster node %s" % node)
	return

	self.start_pcmk_remote(node)

	if self.pcmk_started == 0:
	self.fail("Failed to start pacemaker_remote on node %s" % node)
	return

	# convert node to baremetal node now that it has shutdow the cluster stack
	pats = [ ]
	watch = self.create_watch(pats, 120)
	watch.setwatch()
	pats.append(self.templates["Pat:RscOpOK"] % (self.remote_node, "start"))
	pats.append(self.templates["Pat:DC_IDLE"])

	self.add_connection_rsc(node)

	self.set_timer("remoteMetalInit")
	watch.lookforall()
	self.log_timer("remoteMetalInit")
	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)

	def migrate_connection(self, node):
	if self.failed:
	return

	pats = [ ]
	pats.append(self.templates["Pat:RscOpOK"] % (self.remote_node, "migrate_to"))
	pats.append(self.templates["Pat:RscOpOK"] % (self.remote_node, "migrate_from"))
	pats.append(self.templates["Pat:DC_IDLE"])
	watch = self.create_watch(pats, 120)
	watch.setwatch()

	(rc, lines) = self.rsh(node, "crm_resource -M -r %s" % (self.remote_node), None)
	if rc != 0:
	self.fail("failed to move remote node connection resource")
	return

	self.set_timer("remoteMetalMigrate")
	watch.lookforall()
	self.log_timer("remoteMetalMigrate")

	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)
	return

	def fail_rsc(self, node):
	if self.failed:
	return

	watchpats = [ ]
	watchpats.append(self.templates["Pat:RscRemoteOpOK"] % (self.remote_rsc, "stop", self.remote_node))
	watchpats.append(self.templates["Pat:RscRemoteOpOK"] % (self.remote_rsc, "start", self.remote_node))
	watchpats.append(self.templates["Pat:DC_IDLE"])

	watch = self.create_watch(watchpats, 120)
	watch.setwatch()

	self.debug("causing dummy rsc to fail.")

	rc = self.rsh(node, "rm -f /var/run/resource-agents/Dummy*")

	self.set_timer("remoteRscFail")
	watch.lookforall()
	self.log_timer("remoteRscFail")
	if watch.unmatched:
	self.fail("Unmatched patterns during rsc fail: %s" % watch.unmatched)

	def fail_connection(self, node):
	if self.failed:
	return

	watchpats = [ ]
	watchpats.append(self.templates["Pat:FenceOpOK"] % self.remote_node)
	watchpats.append(self.templates["Pat:NodeFenced"] % self.remote_node)

	watch = self.create_watch(watchpats, 120)
	watch.setwatch()

	# force stop the pcmk remote daemon. this will result in fencing
	self.debug("Force stopped active remote node")
	self.stop_pcmk_remote(node)

	self.debug("Waiting for remote node to be fenced.")
	self.set_timer("remoteMetalFence")
	watch.lookforall()
	self.log_timer("remoteMetalFence")
	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)
	return

	self.debug("Waiting for the remote node to come back up")
	self.CM.ns.WaitForNodeToComeUp(node, 120);

	pats = [ ]
	watch = self.create_watch(pats, 240)
	watch.setwatch()
	pats.append(self.templates["Pat:RscOpOK"] % (self.remote_node, "start"))
	if self.remote_rsc_added == 1:
	pats.append(self.templates["Pat:RscRemoteOpOK"] % (self.remote_rsc, "start", self.remote_node))

	# start the remote node again watch it integrate back into cluster.
	self.start_pcmk_remote(node)
	if self.pcmk_started == 0:
	self.fail("Failed to start pacemaker_remote on node %s" % node)
	return

	self.debug("Waiting for remote node to rejoin cluster after being fenced.")
	self.set_timer("remoteMetalRestart")
	watch.lookforall()
	self.log_timer("remoteMetalRestart")
	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)
	return

	def add_dummy_rsc(self, node):
	if self.failed:
	return

	# verify we can put a resource on the remote node
	pats = [ ]
	watch = self.create_watch(pats, 120)
	watch.setwatch()
	pats.append(self.templates["Pat:RscRemoteOpOK"] % (self.remote_rsc, "start", self.remote_node))
	pats.append(self.templates["Pat:DC_IDLE"])

	# Add a resource that must live on remote-node
	self.add_primitive_rsc(node)

	# force that rsc to prefer the remote node.
	(rc, line) = self.CM.rsh(node, "crm_resource -M -r %s -N %s -f" % (self.remote_rsc, self.remote_node), None)
	if rc != 0:
	self.fail("Failed to place remote resource on remote node.")
	return

	self.set_timer("remoteMetalRsc")
	watch.lookforall()
	self.log_timer("remoteMetalRsc")
	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)

	def test_attributes(self, node):
	if self.failed:
	return

	# This verifies permanent attributes can be set on a remote-node. It also
	# verifies the remote-node can edit it's own cib node section remotely.
	(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -v testval -N %s" % (self.remote_node), None)
	if rc != 0:
	self.fail("Failed to set remote-node attribute. rc:%s output:%s" % (rc, line))
	return

	(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -Q -N %s" % (self.remote_node), None)
	if rc != 0:
	self.fail("Failed to get remote-node attribute")
	return

	(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -D -N %s" % (self.remote_node), None)
	if rc != 0:
	self.fail("Failed to delete remote-node attribute")
	return

	def cleanup_metal(self, node):
	if self.pcmk_started == 0:
	return

	pats = [ ]

	watch = self.create_watch(pats, 120)
	watch.setwatch()

	if self.remote_rsc_added == 1:
	pats.append(self.templates["Pat:RscOpOK"] % (self.remote_rsc, "stop"))
	if self.remote_node_added == 1:
	pats.append(self.templates["Pat:RscOpOK"] % (self.remote_node, "stop"))

	self.set_timer("remoteMetalCleanup")

	if self.remote_use_reconnect_interval:
	self.debug("Cleaning up re-check interval")
	self.rsh(self.get_othernode(node), self.templates["ClearCheckInterval"])

	if self.remote_rsc_added == 1:

	# Remove dummy resource added for remote node tests
	self.debug("Cleaning up dummy rsc put on remote node")
	self.rsh(node, "crm_resource -U -r %s" % self.remote_rsc)
	self.del_rsc(node, self.remote_rsc)

	if self.remote_node_added == 1:

	# Remove remote node's connection resource
	self.debug("Cleaning up remote node connection resource")
	self.rsh(node, "crm_resource -U -r %s" % (self.remote_node))
	self.del_rsc(node, self.remote_node)

	watch.lookforall()
	self.log_timer("remoteMetalCleanup")

	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)

	self.stop_pcmk_remote(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	if self.remote_node_added == 1:
	# Remove remote node itself
	self.debug("Cleaning up node entry for remote node")
	self.rsh(self.get_othernode(node), "crm_node --force --remove %s" % self.remote_node)

	def setup_env(self, node):

	self.remote_node = "remote_%s" % (node)

	# we are assuming if all nodes have a key, that it is
	# the right key... If any node doesn't have a remote
	# key, we regenerate it everywhere.
	if self.rsh.exists_on_all("/etc/pacemaker/authkey", self.Env["nodes"]):
	return

	# create key locally
	(handle, keyfile) = tempfile.mkstemp(".cts")
	os.close(handle)
	devnull = open(os.devnull, 'wb')
	subprocess.check_call(["dd", "if=/dev/urandom", "of=%s" % keyfile, "bs=4096", "count=1"],
	stdout=devnull, stderr=devnull)
	devnull.close()

	# sync key throughout the cluster
	for node in self.Env["nodes"]:
	self.rsh(node, "mkdir -p --mode=0750 /etc/pacemaker")
	self.rsh.cp(keyfile, "root@%s:/etc/pacemaker/authkey" % node)
	self.rsh(node, "chgrp haclient /etc/pacemaker /etc/pacemaker/authkey")
	self.rsh(node, "chmod 0640 /etc/pacemaker/authkey")
	os.unlink(keyfile)

	def is_applicable(self):
	if not self.is_applicable_common():
	return False

	for node in self.Env["nodes"]:
	rc = self.rsh(node, "type pacemaker_remoted >/dev/null 2>&1")
	if rc != 0:
	return False
	return True

	def start_new_test(self, node):
	self.incr("calls")
	self.reset()

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed, start all nodes failed.")

	self.setup_env(node)
	self.start_metal(node)
	self.add_dummy_rsc(node)

	def __call__(self, node):
	return self.failure("This base class is not meant to be called directly.")

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [ """is running on remote.*which isn't allowed""",
	"""Connection terminated""",
	"""Failed to send remote""",
	]

	# RemoteDriver is just a base class for other tests, so it is not added to AllTestClasses


	class RemoteBasic(RemoteDriver):

	def __call__(self, node):
	'''Perform the 'RemoteBaremetal' test. '''

	self.start_new_test(node)
	self.test_attributes(node)
	self.cleanup_metal(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()
	if self.failed:
	return self.failure(self.fail_string)

	return self.success()

	AllTestClasses.append(RemoteBasic)

	class RemoteStonithd(RemoteDriver):

	def __call__(self, node):
	'''Perform the 'RemoteStonithd' test. '''

	self.start_new_test(node)
	self.fail_connection(node)
	self.cleanup_metal(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()
	if self.failed:
	return self.failure(self.fail_string)

	return self.success()

	def is_applicable(self):
	if not RemoteDriver.is_applicable(self):
	return False

	if "DoFencing" in self.Env.keys():
	return self.Env["DoFencing"]

	return True

	def errorstoignore(self):
	ignore_pats = [
	r"Unexpected disconnect on remote-node",
	r"crmd.: error.: Operation remote_.*_monitor",
	r"pengine.: Recover remote_.\s$.$",
	r"Calculated Transition .* /var/lib/pacemaker/pengine/pe-error",
	r"error.: Resource .ocf::.* is active on 2 nodes attempting recovery",
	]

	ignore_pats.extend(RemoteDriver.errorstoignore(self))
	return ignore_pats

	AllTestClasses.append(RemoteStonithd)


	class RemoteMigrate(RemoteDriver):

	def __call__(self, node):
	'''Perform the 'RemoteMigrate' test. '''

	self.start_new_test(node)
	self.migrate_connection(node)
	self.cleanup_metal(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()
	if self.failed:
	return self.failure(self.fail_string)

	return self.success()

	AllTestClasses.append(RemoteMigrate)


	class RemoteRscFailure(RemoteDriver):

	def __call__(self, node):
	'''Perform the 'RemoteRscFailure' test. '''

	self.start_new_test(node)

	# This is an important step. We are migrating the connection
	# before failing the resource. This verifies that the migration
	# has properly maintained control over the remote-node.
	self.migrate_connection(node)

	self.fail_rsc(node)
	self.cleanup_metal(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()
	if self.failed:
	return self.failure(self.fail_string)

	return self.success()

	def errorstoignore(self):
	ignore_pats = [
	r"pengine.: Recover remote-rsc\s$.*$",
	]

	ignore_pats.extend(RemoteDriver.errorstoignore(self))
	return ignore_pats

	AllTestClasses.append(RemoteRscFailure)

	# vim:ts=4:sw=4:et:
	diff --git a/cts/environment.py b/cts/environment.py
	index 16ccbf89e5..c9db4bf371 100644
	--- a/cts/environment.py
	+++ b/cts/environment.py
	@@ -1,697 +1,700 @@
	'''
	Classes related to producing and searching logs
	'''

	__copyright__='''
	Copyright (C) 2014 Andrew Beekhof <andrew@beekhof.net>
	Licensed under the GNU GPL.
	'''

	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public License
	# as published by the Free Software Foundation; either version 2
	# of the License, or (at your option) any later version.
	#
	# This program is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	# GNU General Public License for more details.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write to the Free Software
	# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

	import sys, time, os, socket, random

	from cts.remote import *

	class Environment:

	def __init__(self, args):
	self.data = {}
	self.Nodes = []

	self["DeadTime"] = 300
	self["StartTime"] = 300
	self["StableTime"] = 30
	self["tests"] = []
	self["IPagent"] = "IPaddr2"
	self["DoStandby"] = 1
	self["DoFencing"] = 1
	self["XmitLoss"] = "0.0"
	self["RecvLoss"] = "0.0"
	self["ClobberCIB"] = 0
	self["CIBfilename"] = None
	self["CIBResource"] = 0
	self["DoBSC"] = 0
	self["use_logd"] = 0
	self["oprofile"] = []
	self["warn-inactive"] = 0
	self["ListTests"] = 0
	self["benchmark"] = 0
	self["LogWatcher"] = "any"
	self["SyslogFacility"] = "daemon"
	self["LogFileName"] = "/var/log/messages"
	self["Schema"] = "pacemaker-2.0"
	self["Stack"] = "corosync"
	self["stonith-type"] = "external/ssh"
	self["stonith-params"] = "hostlist=all,livedangerously=yes"
	self["notification-agent"] = "/var/lib/pacemaker/notify.sh"
	self["notification-recipient"] = "/var/lib/pacemaker/notify.log"
	self["loop-minutes"] = 60
	self["valgrind-prefix"] = None
	self["valgrind-procs"] = "attrd cib crmd lrmd pengine stonith-ng"
	self["valgrind-opts"] = """--leak-check=full --show-reachable=yes --trace-children=no --num-callers=25 --gen-suppressions=all --suppressions="""+CTSvars.CTS_home+"""/cts.supp"""

	self["experimental-tests"] = 0
	self["container-tests"] = 0
	self["valgrind-tests"] = 0
	self["unsafe-tests"] = 1
	self["loop-tests"] = 1
	self["scenario"] = "random"
	self["stats"] = 0
	self["docker"] = 0
	+ self["continue"] = 0

	self.RandomGen = random.Random()
	self.logger = LogFactory()

	self.SeedRandom()
	self.rsh = RemoteFactory().getInstance()

	self.target = "localhost"

	self.parse_args(args)
	self.discover()
	self.validate()

	def SeedRandom(self, seed=None):
	if not seed:
	seed = int(time.time())

	self["RandSeed"] = seed
	self.RandomGen.seed(str(seed))

	def dump(self):
	keys = []
	for key in list(self.data.keys()):
	keys.append(key)

	keys.sort()
	for key in keys:
	self.logger.debug("Environment["+key+"]:\t"+str(self[key]))

	def keys(self):
	return self.data.keys()

	def has_key(self, key):
	if key == "nodes":
	return True

	return key in self.data

	def __getitem__(self, key):
	if str(key) == "0":
	raise ValueError("Bad call to 'foo in X', should reference 'foo in X.keys()' instead")

	if key == "nodes":
	return self.Nodes

	elif key == "Name":
	return self.get_stack_short()

	elif key in self.data:
	return self.data[key]

	else:
	return None

	def __setitem__(self, key, value):
	if key == "Stack":
	self.set_stack(value)

	elif key == "node-limit":
	self.data[key] = value
	self.filter_nodes()

	elif key == "nodes":
	self.Nodes = []
	for node in value:
	# I don't think I need the IP address, etc. but this validates
	# the node name against /etc/hosts and/or DNS, so it's a
	# GoodThing(tm).
	try:
	n = node.strip()
	if self.data["docker"] == 0:
	socket.gethostbyname_ex(n)

	self.Nodes.append(n)
	except:
	self.logger.log(node+" not found in DNS... aborting")
	raise

	self.filter_nodes()

	else:
	self.data[key] = value

	def RandomNode(self):
	'''Choose a random node from the cluster'''
	return self.RandomGen.choice(self["nodes"])

	def set_stack(self, name):
	# Normalize stack names
	if name == "heartbeat" or name == "lha":
	self.data["Stack"] = "heartbeat"

	elif name == "openais" or name == "ais" or name == "whitetank":
	self.data["Stack"] = "corosync (plugin v0)"

	elif name == "corosync" or name == "cs" or name == "mcp":
	self.data["Stack"] = "corosync 2.x"

	elif name == "cman":
	self.data["Stack"] = "corosync (cman)"

	elif name == "v1":
	self.data["Stack"] = "corosync (plugin v1)"

	elif name == "v0":
	self.data["Stack"] = "corosync (plugin v0)"

	else:
	raise ValueError("Unknown stack: "+name)
	sys.exit(1)

	def get_stack_short(self):
	# Create the Cluster Manager object
	if not "Stack" in self.data:
	return "unknown"

	elif self.data["Stack"] == "heartbeat":
	return "crm-lha"

	elif self.data["Stack"] == "corosync 2.x":
	if self["docker"]:
	return "crm-mcp-docker"
	else:
	return "crm-mcp"

	elif self.data["Stack"] == "corosync (cman)":
	return "crm-cman"

	elif self.data["Stack"] == "corosync (plugin v1)":
	return "crm-plugin-v1"

	elif self.data["Stack"] == "corosync (plugin v0)":
	return "crm-plugin-v0"

	else:
	LogFactory().log("Unknown stack: "+self["stack"])
	raise ValueError("Unknown stack: "+self["stack"])

	def detect_syslog(self):
	# Detect syslog variant
	if not "syslogd" in self.data:
	if self["have_systemd"]:
	# Systemd
	self["syslogd"] = self.rsh(self.target, "systemctl list-units \| grep syslog.\.service.active.running \| sed 's:.service.::'", stdout=1).strip()
	else:
	# SYS-V
	self["syslogd"] = self.rsh(self.target, "chkconfig --list \| grep syslog.*on \| awk '{print $1}' \| head -n 1", stdout=1).strip()

	if not "syslogd" in self.data or not self["syslogd"]:
	# default
	self["syslogd"] = "rsyslog"

	def detect_at_boot(self):
	# Detect if the cluster starts at boot
	if not "at-boot" in self.data:
	atboot = 0

	if self["have_systemd"]:
	# Systemd
	atboot = atboot or not self.rsh(self.target, "systemctl is-enabled heartbeat.service")
	atboot = atboot or not self.rsh(self.target, "systemctl is-enabled corosync.service")
	atboot = atboot or not self.rsh(self.target, "systemctl is-enabled pacemaker.service")
	else:
	# SYS-V
	atboot = atboot or not self.rsh(self.target, "chkconfig --list \| grep -e corosync.on -e heartbeat.on -e pacemaker.*on")

	self["at-boot"] = atboot

	def detect_ip_offset(self):
	# Try to determin an offset for IPaddr resources
	if self["CIBResource"] and not "IPBase" in self.data:
	network=self.rsh(self.target, "ip addr \| grep inet \| grep -v -e link -e inet6 -e '/32' -e ' lo' \| awk '{print $2}'", stdout=1).strip()
	self["IPBase"] = self.rsh(self.target, "nmap -sn -n %s \| grep 'scan report' \| awk '{print $NF}' \| sed 's:(::' \| sed 's:)::' \| sort -V \| tail -n 1" % network, stdout=1).strip()
	if not self["IPBase"]:
	self["IPBase"] = " fe80::1234:56:7890:1000"
	self.logger.log("Could not determine an offset for IPaddr resources. Perhaps nmap is not installed on the nodes.")
	self.logger.log("Defaulting to '%s', use --test-ip-base to override" % self["IPBase"])

	elif int(self["IPBase"].split('.')[3]) >= 240:
	self.logger.log("Could not determine an offset for IPaddr resources. Upper bound is too high: %s %s"
	% (self["IPBase"], self["IPBase"].split('.')[3]))
	self["IPBase"] = " fe80::1234:56:7890:1000"
	self.logger.log("Defaulting to '%s', use --test-ip-base to override" % self["IPBase"])

	def filter_nodes(self):
	if self["node-limit"] > 0:
	if len(self["nodes"]) > self["node-limit"]:
	self.logger.log("Limiting the number of nodes configured=%d (max=%d)"
	%(len(self["nodes"]), self["node-limit"]))
	while len(self["nodes"]) > self["node-limit"]:
	self["nodes"].pop(len(self["nodes"])-1)

	def validate(self):
	if len(self["nodes"]) < 1:
	print("No nodes specified!")
	sys.exit(1)

	def discover(self):
	self.target = random.Random().choice(self["nodes"])

	master = socket.gethostname()

	# Use the IP where possible to avoid name lookup failures
	for ip in socket.gethostbyname_ex(master)[2]:
	if ip != "127.0.0.1":
	master = ip
	break;
	self["cts-master"] = master

	if not "have_systemd" in self.data:
	self["have_systemd"] = not self.rsh(self.target, "systemctl list-units")

	self.detect_syslog()
	self.detect_at_boot()
	self.detect_ip_offset()

	self.validate()

	def parse_args(self, args):
	skipthis=None

	if not args:
	args=sys.argv[1:]

	for i in range(0, len(args)):
	if skipthis:
	skipthis=None
	continue

	elif args[i] == "-l" or args[i] == "--limit-nodes":
	skipthis=1
	self["node-limit"] = int(args[i+1])

	elif args[i] == "-r" or args[i] == "--populate-resources":
	self["CIBResource"] = 1
	self["ClobberCIB"] = 1

	elif args[i] == "--outputfile":
	skipthis=1
	self["OutputFile"] = args[i+1]
	LogFactory().add_file(self["OutputFile"])

	elif args[i] == "-L" or args[i] == "--logfile":
	skipthis=1
	self["LogWatcher"] = "remote"
	self["LogAuditDisabled"] = 1
	self["LogFileName"] = args[i+1]

	elif args[i] == "--ip" or args[i] == "--test-ip-base":
	skipthis=1
	self["IPBase"] = args[i+1]
	self["CIBResource"] = 1
	self["ClobberCIB"] = 1

	elif args[i] == "--oprofile":
	skipthis=1
	self["oprofile"] = args[i+1].split(' ')

	elif args[i] == "--trunc":
	self["TruncateLog"]=1

	elif args[i] == "--list-tests" or args[i] == "--list" :
	self["ListTests"]=1

	elif args[i] == "--benchmark":
	self["benchmark"]=1

	elif args[i] == "--bsc":
	self["DoBSC"] = 1
	self["scenario"] = "basic-sanity"

	elif args[i] == "--qarsh":
	RemoteFactory().enable_qarsh()

	elif args[i] == "--docker":
	self["docker"] = 1
	RemoteFactory().enable_docker()
	-
	+ elif args[i] == "--yes" or args[i] == "-y":
	+ self["continue"] = 1
	elif args[i] == "--stonith" or args[i] == "--fencing":
	skipthis=1
	if args[i+1] == "1" or args[i+1] == "yes":
	self["DoFencing"]=1
	elif args[i+1] == "0" or args[i+1] == "no":
	self["DoFencing"]=0
	elif args[i+1] == "phd":
	self["DoStonith"]=1
	self["stonith-type"] = "fence_phd_kvm"
	self["stonith-params"] = "pcmk_arg_map=domain:uname,delay=0"
	elif args[i+1] == "rhcs" or args[i+1] == "xvm" or args[i+1] == "virt":
	self["DoStonith"]=1
	self["stonith-type"] = "fence_xvm"
	self["stonith-params"] = "pcmk_arg_map=domain:uname,delay=0"
	elif args[i+1] == "docker":
	self["DoStonith"]=1
	self["stonith-type"] = "fence_docker_cts"
	elif args[i+1] == "scsi":
	self["DoStonith"]=1
	self["stonith-type"] = "fence_scsi"
	self["stonith-params"] = "delay=0"
	elif args[i+1] == "ssh" or args[i+1] == "lha":
	self["DoStonith"]=1
	self["stonith-type"] = "external/ssh"
	self["stonith-params"] = "hostlist=all,livedangerously=yes"
	elif args[i+1] == "north":
	self["DoStonith"]=1
	self["stonith-type"] = "fence_apc"
	self["stonith-params"] = "ipaddr=north-apc,login=apc,passwd=apc,pcmk_host_map=north-01:2;north-02:3;north-03:4;north-04:5;north-05:6;north-06:7;north-07:9;north-08:10;north-09:11;north-10:12;north-11:13;north-12:14;north-13:15;north-14:18;north-15:17;north-16:19;"
	elif args[i+1] == "south":
	self["DoStonith"]=1
	self["stonith-type"] = "fence_apc"
	self["stonith-params"] = "ipaddr=south-apc,login=apc,passwd=apc,pcmk_host_map=south-01:2;south-02:3;south-03:4;south-04:5;south-05:6;south-06:7;south-07:9;south-08:10;south-09:11;south-10:12;south-11:13;south-12:14;south-13:15;south-14:18;south-15:17;south-16:19;"
	elif args[i+1] == "east":
	self["DoStonith"]=1
	self["stonith-type"] = "fence_apc"
	self["stonith-params"] = "ipaddr=east-apc,login=apc,passwd=apc,pcmk_host_map=east-01:2;east-02:3;east-03:4;east-04:5;east-05:6;east-06:7;east-07:9;east-08:10;east-09:11;east-10:12;east-11:13;east-12:14;east-13:15;east-14:18;east-15:17;east-16:19;"
	elif args[i+1] == "west":
	self["DoStonith"]=1
	self["stonith-type"] = "fence_apc"
	self["stonith-params"] = "ipaddr=west-apc,login=apc,passwd=apc,pcmk_host_map=west-01:2;west-02:3;west-03:4;west-04:5;west-05:6;west-06:7;west-07:9;west-08:10;west-09:11;west-10:12;west-11:13;west-12:14;west-13:15;west-14:18;west-15:17;west-16:19;"
	elif args[i+1] == "openstack":
	self["DoStonith"]=1
	self["stonith-type"] = "fence_openstack"

	print("Obtaining OpenStack credentials from the current environment")
	self["stonith-params"] = "region=%s,tenant=%s,auth=%s,user=%s,password=%s" % (
	os.environ['OS_REGION_NAME'],
	os.environ['OS_TENANT_NAME'],
	os.environ['OS_AUTH_URL'],
	os.environ['OS_USERNAME'],
	os.environ['OS_PASSWORD']
	)

	elif args[i+1] == "rhevm":
	self["DoStonith"]=1
	self["stonith-type"] = "fence_rhevm"

	print("Obtaining RHEV-M credentials from the current environment")
	self["stonith-params"] = "login=%s,passwd=%s,ipaddr=%s,ipport=%s,ssl=1,shell_timeout=10" % (
	os.environ['RHEVM_USERNAME'],
	os.environ['RHEVM_PASSWORD'],
	os.environ['RHEVM_SERVER'],
	os.environ['RHEVM_PORT'],
	)

	else:
	self.usage(args[i+1])

	elif args[i] == "--stonith-type":
	self["stonith-type"] = args[i+1]
	skipthis=1

	elif args[i] == "--stonith-args":
	self["stonith-params"] = args[i+1]
	skipthis=1

	elif args[i] == "--standby":
	skipthis=1
	if args[i+1] == "1" or args[i+1] == "yes":
	self["DoStandby"] = 1
	elif args[i+1] == "0" or args[i+1] == "no":
	self["DoStandby"] = 0
	else:
	self.usage(args[i+1])

	elif args[i] == "--clobber-cib" or args[i] == "-c":
	self["ClobberCIB"] = 1

	elif args[i] == "--cib-filename":
	skipthis=1
	self["CIBfilename"] = args[i+1]

	elif args[i] == "--xmit-loss":
	try:
	float(args[i+1])
	except ValueError:
	print("--xmit-loss parameter should be float")
	self.usage(args[i+1])
	skipthis=1
	self["XmitLoss"] = args[i+1]

	elif args[i] == "--recv-loss":
	try:
	float(args[i+1])
	except ValueError:
	print("--recv-loss parameter should be float")
	self.usage(args[i+1])
	skipthis=1
	self["RecvLoss"] = args[i+1]

	elif args[i] == "--choose":
	skipthis=1
	self["tests"].append(args[i+1])
	self["scenario"] = "sequence"

	elif args[i] == "--nodes":
	skipthis=1
	self["nodes"] = args[i+1].split(' ')

	elif args[i] == "-g" or args[i] == "--group" or args[i] == "--dsh-group":
	skipthis=1
	self["OutputFile"] = "%s/cluster-%s.log" % (os.environ['HOME'], args[i+1])
	LogFactory().add_file(self["OutputFile"], "CTS")

	dsh_file = "%s/.dsh/group/%s" % (os.environ['HOME'], args[i+1])

	# Hacks to make my life easier
	if args[i+1] == "r6":
	self["Stack"] = "cman"
	self["DoStonith"]=1
	self["stonith-type"] = "fence_xvm"
	self["stonith-params"] = "delay=0"
	self["IPBase"] = " fe80::1234:56:7890:4000"

	elif args[i+1] == "virt1":
	self["Stack"] = "corosync"
	self["DoStonith"]=1
	self["stonith-type"] = "fence_xvm"
	self["stonith-params"] = "delay=0"
	self["IPBase"] = " fe80::1234:56:7890:1000"

	elif args[i+1] == "east16" or args[i+1] == "nsew":
	self["Stack"] = "corosync"
	self["DoStonith"]=1
	self["stonith-type"] = "fence_apc"
	self["stonith-params"] = "ipaddr=east-apc,login=apc,passwd=apc,pcmk_host_map=east-01:2;east-02:3;east-03:4;east-04:5;east-05:6;east-06:7;east-07:9;east-08:10;east-09:11;east-10:12;east-11:13;east-12:14;east-13:15;east-14:18;east-15:17;east-16:19;"
	self["IPBase"] = " fe80::1234:56:7890:2000"

	if args[i+1] == "east16":
	# Requires newer python than available via nsew
	self["IPagent"] = "Dummy"

	elif args[i+1] == "corosync8":
	self["Stack"] = "corosync"
	self["DoStonith"]=1
	self["stonith-type"] = "fence_rhevm"

	print("Obtaining RHEV-M credentials from the current environment")
	self["stonith-params"] = "login=%s,passwd=%s,ipaddr=%s,ipport=%s,ssl=1,shell_timeout=10" % (
	os.environ['RHEVM_USERNAME'],
	os.environ['RHEVM_PASSWORD'],
	os.environ['RHEVM_SERVER'],
	os.environ['RHEVM_PORT'],
	)
	self["IPBase"] = " fe80::1234:56:7890:3000"

	if os.path.isfile(dsh_file):
	self["nodes"] = []
	f = open(dsh_file, 'r')
	for line in f:
	l = line.strip().rstrip()
	if not l.startswith('#'):
	self["nodes"].append(l)
	f.close()

	else:
	print("Unknown DSH group: %s" % args[i+1])

	elif args[i] == "--syslog-facility" or args[i] == "--facility":
	skipthis=1
	self["SyslogFacility"] = args[i+1]

	elif args[i] == "--seed":
	skipthis=1
	self.SeedRandom(args[i+1])

	elif args[i] == "--warn-inactive":
	self["warn-inactive"] = 1

	elif args[i] == "--schema":
	skipthis=1
	self["Schema"] = args[i+1]

	elif args[i] == "--ais":
	self["Stack"] = "openais"

	elif args[i] == "--at-boot" or args[i] == "--cluster-starts-at-boot":
	skipthis=1
	if args[i+1] == "1" or args[i+1] == "yes":
	self["at-boot"] = 1
	elif args[i+1] == "0" or args[i+1] == "no":
	self["at-boot"] = 0
	else:
	self.usage(args[i+1])

	elif args[i] == "--heartbeat" or args[i] == "--lha":
	self["Stack"] = "heartbeat"

	elif args[i] == "--hae":
	self["Stack"] = "openais"
	self["Schema"] = "hae"

	elif args[i] == "--stack":
	if args[i+1] == "fedora" or args[i+1] == "fedora-17" or args[i+1] == "fedora-18":
	self["Stack"] = "corosync"
	elif args[i+1] == "rhel-6":
	self["Stack"] = "cman"
	elif args[i+1] == "rhel-7":
	self["Stack"] = "corosync"
	else:
	self["Stack"] = args[i+1]
	skipthis=1

	elif args[i] == "--once":
	self["scenario"] = "all-once"

	elif args[i] == "--boot":
	self["scenario"] = "boot"

	elif args[i] == "--notification-agent":
	self["notification-agent"] = args[i+1]
	skipthis = 1

	elif args[i] == "--notification-recipient":
	self["notification-recipient"] = args[i+1]
	skipthis = 1

	elif args[i] == "--valgrind-tests":
	self["valgrind-tests"] = 1

	elif args[i] == "--valgrind-procs":
	self["valgrind-procs"] = args[i+1]
	skipthis = 1

	elif args[i] == "--no-loop-tests":
	self["loop-tests"] = 0

	elif args[i] == "--loop-minutes":
	skipthis=1
	try:
	self["loop-minutes"]=int(args[i+1])
	except ValueError:
	self.usage(args[i])

	elif args[i] == "--no-unsafe-tests":
	self["unsafe-tests"] = 0

	elif args[i] == "--experimental-tests":
	self["experimental-tests"] = 1

	elif args[i] == "--container-tests":
	self["container-tests"] = 1

	elif args[i] == "--set":
	skipthis=1
	(name, value) = args[i+1].split('=')
	self[name] = value
	print("Setting %s = %s" % (name, value))

	elif args[i] == "--help":
	self.usage(args[i], 0)

	elif args[i] == "--":
	break

	else:
	try:
	NumIter=int(args[i])
	self["iterations"] = NumIter
	except ValueError:
	self.usage(args[i])

	def usage(self, arg, status=1):
	if status:
	print("Illegal argument %s" % arg)
	print("usage: " + sys.argv[0] +" [options] number-of-iterations")
	print("\nCommon options: ")
	print("\t [--nodes 'node list'] list of cluster nodes separated by whitespace")
	print("\t [--group \| -g 'name'] use the nodes listed in the named DSH group (~/.dsh/groups/$name)")
	print("\t [--limit-nodes max] only use the first 'max' cluster nodes supplied with --nodes")
	print("\t [--stack (v0\|v1\|cman\|corosync\|heartbeat\|openais)] which cluster stack is installed")
	print("\t [--list-tests] list the valid tests")
	print("\t [--benchmark] add the timing information")
	print("\t ")
	print("Options that CTS will usually auto-detect correctly: ")
	print("\t [--logfile path] where should the test software look for logs from cluster nodes")
	print("\t [--syslog-facility name] which syslog facility should the test software log to")
	print("\t [--at-boot (1\|0)] does the cluster software start at boot time")
	print("\t [--test-ip-base ip] offset for generated IP address resources")
	print("\t ")
	print("Options for release testing: ")
	print("\t [--populate-resources \| -r] generate a sample configuration")
	print("\t [--choose name] run only the named test")
	print("\t [--stonith (1 \| 0 \| yes \| no \| rhcs \| ssh)]")
	print("\t [--once] run all valid tests once")
	print("\t ")
	print("Additional (less common) options: ")
	print("\t [--clobber-cib \| -c ] erase any existing configuration")
	print("\t [--outputfile path] optional location for the test software to write logs to")
	print("\t [--trunc] truncate logfile before starting")
	print("\t [--xmit-loss lost-rate(0.0-1.0)]")
	print("\t [--recv-loss lost-rate(0.0-1.0)]")
	print("\t [--standby (1 \| 0 \| yes \| no)]")
	print("\t [--fencing (1 \| 0 \| yes \| no \| rhcs \| lha \| openstack )]")
	print("\t [--stonith-type type]")
	print("\t [--stonith-args name=value]")
	print("\t [--bsc]")
	print("\t [--notification-agent path] script to configure for Pacemaker notifications")
	print("\t [--notification-recipient r] recipient to pass to notification agent")
	print("\t [--no-loop-tests] dont run looping/time-based tests")
	print("\t [--no-unsafe-tests] dont run tests that are unsafe for use with ocfs2/drbd")
	print("\t [--valgrind-tests] include tests using valgrind")
	print("\t [--experimental-tests] include experimental tests")
	print("\t [--container-tests] include pacemaker_remote tests that run in lxc container resources")
	print("\t [--oprofile 'node list'] list of cluster nodes to run oprofile on]")
	print("\t [--qarsh] use the QARSH backdoor to access nodes instead of SSH")
	print("\t [--docker] Indicates nodes are docker nodes.")
	print("\t [--seed random_seed]")
	print("\t [--set option=value]")
	+ print("\t [--yes \| -y] continue to run cts when there is an interaction whether to continue running pacemaker-cts")
	print("\t ")
	print("\t Example: ")
	print("\t python sys.argv[0] -g virt1 --stack cs -r --stonith ssh --schema pacemaker-1.0 500")

	sys.exit(status)

	class EnvFactory:
	instance = None
	def __init__(self):
	pass

	def getInstance(self, args=None):
	if not EnvFactory.instance:
	EnvFactory.instance = Environment(args)
	return EnvFactory.instance

File Metadata

Mime Type: text/x-diff
Expires: Thu, Oct 16, 3:28 PM (5 h, 38 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2533549
Default Alt Text: (221 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions