No OneTemporary
Actions

Size

39 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/cts/CTSaudits.py.in b/cts/CTSaudits.py.in
	index dcc791426c..c5cff2856f 100755
	--- a/cts/CTSaudits.py.in
	+++ b/cts/CTSaudits.py.in
	@@ -1,580 +1,587 @@
	#!@PYTHON@

	'''CTS: Cluster Testing System: Audit module
	'''

	__copyright__='''
	Copyright (C) 2000, 2001 Alan Robertson <alanr@unix.sh>
	Licensed under the GNU GPL.
	'''

	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public License
	# as published by the Free Software Foundation; either version 2
	# of the License, or (at your option) any later version.
	#
	# This program is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	# GNU General Public License for more details.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write to the Free Software
	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

	import time, os, popen2, string
	import CTS
	import os
	import popen2


	class ClusterAudit:

	def __init__(self, cm):
	self.CM = cm

	def __call__(self):
	raise ValueError("Abstract Class member (__call__)")

	def is_applicable(self):
	'''Return TRUE if we are applicable in the current test configuration'''
	raise ValueError("Abstract Class member (is_applicable)")
	return 1

	def name(self):
	raise ValueError("Abstract Class member (name)")

	AllAuditClasses = [ ]

	class ResourceAudit(ClusterAudit):

	def name(self):
	return "ResourceAudit"

	def _doauditRsc(self, resource):
	ResourceNodes = []
	for node in self.CM.Env["nodes"]:
	if self.CM.ShouldBeStatus[node] == self.CM["up"]:
	if resource.IsRunningOn(node):
	ResourceNodes.append(node)
	return ResourceNodes

	def _doaudit(self):
	'''Check to see if all resources are running in exactly one place
	in the cluster.
	We also verify that the members of a resource group are all
	running on the same node in the cluster,
	and we monitor that they are all running "properly".
	'''
	Fatal = 0
	result = []


	Groups = self.CM.ResourceGroups()
	for group in Groups:
	GrpServedBy = None
	lastResource = None

	for resource in group:

	#
	# _doauditRsc returns the set of nodes serving
	# the given resource. This is normally a single node.
	#

	ResourceNodes = self._doauditRsc(resource)


	# Is the resource served without quorum present?

	if not self.CM.HasQuorum() and len(ResourceNodes) != 0:

	result.append("Resource " + repr(resource)
	+ " active without Quorum: "
	+ repr(ResourceNodes))

	# Is the resource served at all?

	elif len(ResourceNodes) == 0 and self.CM.HasQuorum():
	result.append("Resource " + repr(resource)
	+ " not served anywhere.")

	# Is the resource served too many times?

	elif len(ResourceNodes) > 1:
	result.append("Resource " + repr(resource)
	+ " served too many times: "
	+ repr(ResourceNodes))
	self.CM.log("Resource " + repr(resource)
	+ " served too many times: "
	+ repr(ResourceNodes))
	Fatal = 1
	elif GrpServedBy == None:
	GrpServedBy = ResourceNodes

	# Are all the members of the Rsc Grp served by the same node?

	elif GrpServedBy != ResourceNodes:
	result.append("Resource group resources" + repr(resource)
	+ " running on different nodes: "
	+ repr(ResourceNodes)+" vs "+repr(GrpServedBy)
	+ "(otherRsc = " + repr(lastResource) + ")")
	self.CM.log("Resource group resources" + repr(resource)
	+ " running on different nodes: "
	+ repr(ResourceNodes)+" vs "+repr(GrpServedBy)
	+ "(otherRsc = " + repr(lastResource) + ")")
	Fatal = 1

	if self.CM.Env.has_key("SuppressMonitoring") and \
	self.CM.Env["SuppressMonitoring"]:
	continue

	# Is the resource working correctly ?

	if not Fatal and len(ResourceNodes) == 1:
	beforearpchild = popen2.Popen3("date;/sbin/arp -n\|cut -c1-15,26-50,75-"
	, None)
	beforearpchild.tochild.close() # /dev/null
	if not resource.IsWorkingCorrectly(ResourceNodes[0]):
	afterarpchild = popen2.Popen3("/sbin/arp -n\|cut -c1-15,26-50,75-"
	, None)
	afterarpchild.tochild.close() # /dev/null
	result.append("Resource " + repr(resource)
	+ " not operating properly."
	+ " Resource is running on " + ResourceNodes[0]);
	Fatal = 1
	self.CM.log("ARP table before failure ========");
	for line in beforearpchild.fromchild.readlines():
	self.CM.log(line)
	self.CM.log("ARP table after failure ========");
	for line in afterarpchild.fromchild.readlines():
	self.CM.log(line)
	self.CM.log("End of ARP tables ========");
	try:
	beforearpchild.wait()
	afterarpchild.wait()
	except OSError: pass
	afterarpchild.fromchild.close()
	beforearpchild.fromchild.close()

	lastResource = resource

	if (Fatal):
	result.insert(0, "FATAL") # Kludgy.

	return result


	def __call__(self):
	#
	# Audit the resources. Since heartbeat doesn't really
	# know when resource acquisition is complete, we will
	# poll until things get stable.
	#
	# Having a resource duplicately implemented is a Fatal Error
	# with no tolerance granted.
	#
	audresult = self._doaudit()
	#
	# Probably the constant below should be a CM parameter.
	# Then it could be 0 for FailSafe.
	# Of course, it really depends on what resources
	# you have in the test suite, and how long it takes
	# for them to settle.
	# Recently, we've changed heartbeat so we know better when
	# resource acquisition is done.
	#
	audcount=5;

	while(audcount > 0):
	audresult = self._doaudit()
	if (len(audresult) <= 0 or audresult[0] == "FATAL"):
	audcount=0
	else:
	audcount = audcount - 1
	if (audcount > 0):
	time.sleep(1)
	if (len(audresult) > 0):
	self.CM.log("Fatal Audit error: " + repr(audresult))

	return (len(audresult) == 0)

	def is_applicable(self):
	if self.CM["Name"] == "heartbeat":
	return 1
	return 0

	AllAuditClasses.append(ResourceAudit)

	class HAResourceAudit(ClusterAudit):
	def __init__(self, cm):
	self.CM = cm
	-
	+
	def _RscRunningNodes(self, resource):
	ResourceNodes = []
	for node in self.CM.Env["nodes"]:
	if self.CM.ShouldBeStatus[node] == self.CM["up"]:
	if resource.IsRunningOn(node):
	ResourceNodes.append(node)
	return ResourceNodes

	def __call__(self):
	passed = 1
	NodeofRsc = {}
	-
	+ self.CM.debug("Do Audit HAResourceAudit")
	+
	+ for node in self.CM.Env["nodes"]:
	+ if self.CM.ShouldBeStatus[node] == self.CM["up"]:
	+ break
	+ else:
	+ return passed
	+
	#Make sure the resouces are running on one and only one node
	Resources = self.CM.Resources()
	for resource in Resources :
	RunningNodes = self._RscRunningNodes(resource)
	NodeofRsc[resource.rid]=RunningNodes
	if len(RunningNodes) == 0 :
	self.CM.log("%s isn't running anywhere" %resource)
	passed = 0
	if len(RunningNodes) > 1:
	self.CM.log("%s is running more than once %s"
	- %(resource, str(RunningNodes)))
	+ %(resource, str(RunningNodes)))
	passed = 0

	#Make sure the resouces with "must","placement" constraint are running on the same node
	Dependancies = self.CM.Dependancies()
	for dependancy in Dependancies:
	if dependancy["type"] == "placement" and dependancy["strength"] == "must":
	if NodeofRsc[dependancy["from"]] != NodeofRsc[dependancy["to"]]:
	print dependancy["from"] + " and " + dependancy["to"] + " should be run on same node"
	passed = 0

	return passed

	def is_applicable(self):
	- if self.CM["Name"] == "linux-ha-v2":
	+ if self.CM["Name"] == "linux-ha-v2" and self.CM.Env["ResCanStop"] == 0:
	return 1
	return 0

	def name(self):
	return "HAResourceAudit"

	-#AllAuditClasses.append(HAResourceAudit)
	+AllAuditClasses.append(HAResourceAudit)

	class CrmdStateAudit(ClusterAudit):
	def __init__(self, cm):
	self.CM = cm
	self.Stats = {"calls":0
	, "success":0
	, "failure":0
	, "skipped":0
	, "auditfail":0}

	def has_key(self, key):
	return self.Stats.has_key(key)

	def __setitem__(self, key, value):
	self.Stats[key] = value

	def __getitem__(self, key):
	return self.Stats[key]

	def incr(self, name):
	'''Increment (or initialize) the value associated with the given name'''
	if not self.Stats.has_key(name):
	self.Stats[name]=0
	self.Stats[name] = self.Stats[name]+1

	def __call__(self):
	self.CM.debug("Do Audit %s"%self.name())
	passed = 1
	dc_list = []
	up_count = 0
	node_count = 0
	up_are_down = 0
	down_are_up = 0
	slave_count = 0
	unstable_list = []

	for node in self.CM.Env["nodes"]:
	out=self.CM.rsh.readaline(node, self.CM["StatusCmd"]%node)
	ret = (string.find(out, 'ok') != -1)
	node_count = node_count + 1
	if ret:
	up_count = up_count + 1
	if self.CM.ShouldBeStatus[node] == self.CM["down"]:
	self.CM.log(
	"Node %s %s when it should be %s"
	% (node, self.CM["up"], self.CM.ShouldBeStatus[node]))
	self.CM.ShouldBeStatus[node] = self.CM["up"]
	down_are_up = down_are_up + 1

	ret= (string.find(out, 'S_NOT_DC') != -1)
	if ret:
	slave_count = slave_count + 1
	else:
	ret= (string.find(out, 'S_IDLE') != -1)
	if ret:
	dc_list.append(node)
	else:
	unstable_list.append(out)
	else:
	if self.CM.ShouldBeStatus[node] == self.CM["up"]:
	self.CM.log(
	"Node %s %s when it should be %s"
	% (node, self.CM["down"], self.CM.ShouldBeStatus[node]))
	self.CM.ShouldBeStatus[node] = self.CM["down"]
	up_are_down = up_are_down + 1

	# if up_count > 0 and len(dc_list) != 1:
	# passed = 0
	# self.CM.log("Exactly 1 node should be DC. We found %d (of %d): %s"
	# %(len(dc_list), up_count, str(dc_list)))

	if len(unstable_list) > 0:
	passed = 0
	self.CM.log("Cluster is not stable: %d (of %d)."
	%(len(unstable_list), up_count))
	for status in unstable_list:
	self.CM.log("%s" %(status))

	if up_are_down > 0:
	passed = 0
	self.CM.log("%d (of %d) nodes expected to be up were down."
	%(up_are_down, node_count))

	if down_are_up > 0:
	passed = 0
	self.CM.log("%d (of %d) nodes expected to be down were up."
	%(down_are_up, node_count))

	return passed

	def name(self):
	return "CrmdStateAudit"

	def is_applicable(self):
	if self.CM["Name"] == "linux-ha-v2":
	return 1
	return 0

	AllAuditClasses.append(CrmdStateAudit)

	class PartitionAudit(ClusterAudit):
	def __init__(self, cm):
	self.CM = cm
	self.Stats = {"calls":0
	, "success":0
	, "failure":0
	, "skipped":0
	, "auditfail":0}
	self.NodeEpoche={}
	self.NodeState={}
	self.NodeQuorum={}
	self.NodeCCM={}

	def has_key(self, key):
	return self.Stats.has_key(key)

	def __setitem__(self, key, value):
	self.Stats[key] = value

	def __getitem__(self, key):
	return self.Stats[key]

	def incr(self, name):
	'''Increment (or initialize) the value associated with the given name'''
	if not self.Stats.has_key(name):
	self.Stats[name]=0
	self.Stats[name] = self.Stats[name]+1

	def __call__(self):
	self.CM.debug("Do Audit %s"%self.name())
	passed = 1
	nodes_up = 0
	ccm_partitions = []

	for node in self.CM.Env["nodes"]:
	if self.CM.ShouldBeStatus[node] != self.CM["up"]:
	self.NodeCCM[node] = None
	self.NodeState[node] = None
	self.NodeEpoche[node] = None
	self.NodeQuorum[node] = None
	else:
	nodes_up = nodes_up + 1
	# self.PS_State[node] = os.system("@SSH@ root@%s ps -C crmd" %(node))
	self.NodeQuorum[node] = self.CM.rsh.readaline(
	node, self.CM["QuorumCmd"])
	self.NodeCCM[node] = self.CM.rsh.readaline(
	node, self.CM["ParitionCmd"])
	self.NodeEpoche[node] = self.CM.rsh.readaline(
	node, self.CM["EpocheCmd"])
	self.NodeState[node] = self.CM.rsh.readaline(
	node, self.CM["StatusCmd"]%node)

	self.NodeState[node] = self.trim_string(self.NodeState[node])
	self.NodeEpoche[node] = self.trim2int(self.NodeEpoche[node])
	self.NodeQuorum[node] = self.trim_string(self.NodeQuorum[node])
	self.NodeCCM[node] = self.trim_string(self.NodeCCM[node])

	if len(self.NodeCCM[node]) > 1:
	found = 0
	for partition in ccm_partitions:
	if partition == self.NodeCCM[node]:
	found = 1
	if found == 0:
	ccm_partitions.append(self.NodeCCM[node])

	if nodes_up == 0:
	return 1

	if len(ccm_partitions) > 1:
	self.CM.log("Warn: %d cluster partitions detected:" %len(ccm_partitions))
	for partition in ccm_partitions:
	self.CM.log("\t %s" %partition)

	for partition in ccm_partitions:
	partition_passed = 0
	if self.audit_partition(partition) == 0:
	passed = 0

	return passed

	def trim_string(self, avalue):
	if not avalue:
	return None
	if len(avalue) > 1:
	return avalue[:-1]

	def trim2int(self, avalue):
	if not avalue:
	return None
	if len(avalue) > 1:
	return int(avalue[:-1])


	def audit_partition(self, partition):
	passed = 0
	dc_found = []
	dc_allowed_list = []
	lowest_epoche = None
	node_list = partition.split()

	self.CM.debug("Auditing partition: %s" %(partition))
	for node in node_list:
	if not self.NodeEpoche[node]:
	self.CM.log("Warn: Node %s appeared out of nowhere" %(node))
	# not in itself a reason to fail the audit (not what we're checking for in this one)
	# passed = 0
	self.CM.ShouldBeStatus[node] = self.CM["up"]

	self.NodeState[node] = self.CM.rsh.readaline(
	node, self.CM["StatusCmd"]%node)
	self.NodeEpoche[node] = self.CM.rsh.readaline(
	node, self.CM["EpocheCmd"])
	self.NodeQuorum[node] = self.CM.rsh.readaline(
	node, self.CM["QuorumCmd"])

	self.NodeState[node] = self.trim_string(self.NodeState[node])
	self.NodeEpoche[node] = self.trim2int(self.NodeEpoche[node])
	self.NodeQuorum[node] = self.trim_string(self.NodeQuorum[node])

	elif lowest_epoche == None or self.NodeEpoche[node] < lowest_epoche:
	lowest_epoche = self.NodeEpoche[node]

	for node in node_list:
	if self.CM.ShouldBeStatus[node] == self.CM["up"]:
	if self.is_node_dc(self.NodeState[node]):
	dc_found.append(node)
	if self.NodeEpoche[node] == lowest_epoche:
	passed = 1
	elif not self.NodeEpoche[node]:
	self.CM.log("Cant determin epoche for DC %s" %(node))
	passed = 0

	else:
	self.CM.log("DC %s is not the oldest node (%d vs. %d)"
	%(node, self.NodeEpoche[node], lowest_epoche))
	passed = 0

	if len(dc_found) == 0:
	self.CM.log("DC not found on any of the %d allowed nodes: %s (of %s)"
	%(len(dc_allowed_list), str(dc_allowed_list), str(node_list)))
	elif len(dc_found) > 1:
	self.CM.log("%d DCs (%s) found in cluster partition: %s"
	%(len(dc_found), str(dc_found), str(node_list)))
	passed = 0

	if passed == 0:
	for node in node_list:
	if self.CM.ShouldBeStatus[node] == self.CM["up"]:
	self.CM.log("epoche %s : %s"
	%(self.NodeEpoche[node], self.NodeState[node]))

	if self.CM.Env["CIBResource"] == 1 and len(dc_found) > 0 and self.NodeQuorum[dc_found[0]]:
	if self.audit_dc_resources(node_list, dc_found) == 0:
	passed = 0

	Resources = self.CM.Resources()
	for node in node_list:
	for resource in Resources:
	if resource.rid == "rsc_"+node:
	if resource.IsRunningOn(node) == 0:
	self.CM.log("Node %s is not running its own resource" %(node))
	passed = 0
	elif self.CM.Env["CIBResource"] == 1:
	# no quorum means no resource management
	self.CM.debug("Not auditing resources - no quorum")

	return passed

	def audit_dc_resources(self, node_list, dc_list):
	passed = 1
	Resources = self.CM.Resources()
	for resource in Resources:
	if resource.rid == "DcIPaddr":
	self.CM.debug("Auditing resource: %s" %(resource))
	# All DCs are running the resource
	for dc in dc_list:
	if self.NodeQuorum[dc]:
	if resource.IsRunningOn(dc) == 0:
	self.CM.log("Resource %s not running on DC: %s"
	%(resource, dc))
	passed = 0

	# All nodes running the resource are DCs
	for node in node_list:
	if resource.IsRunningOn(node):
	if self.is_node_dc(self.NodeState[node]) == 0:
	self.CM.log("Resource %s is running on non-DC node %s"
	%("DcIPaddr", node))
	passed = 0

	return passed

	def is_node_dc(self, status_line):
	rc = 0
	if not status_line:
	rc = 0
	elif string.find(status_line, 'S_IDLE') != -1:
	rc = 1
	elif string.find(status_line, 'S_INTEGRATION') != -1:
	rc = 1
	elif string.find(status_line, 'S_FINALIZE_JOIN') != -1:
	rc = 1
	elif string.find(status_line, 'S_POLICY_ENGINE') != -1:
	rc = 1
	elif string.find(status_line, 'S_TRANSITION_ENGINE') != -1:
	rc = 1
	return rc

	def name(self):
	return "PartitionAudit"

	def is_applicable(self):
	if self.CM["Name"] == "linux-ha-v2":
	return 1
	return 0

	AllAuditClasses.append(PartitionAudit)

	def AuditList(cm):
	result = []
	for auditclass in AllAuditClasses:
	result.append(auditclass(cm))
	return result
	diff --git a/cts/CTSlab.py.in b/cts/CTSlab.py.in
	index ca3f8fbb90..b5f4017dd5 100755
	--- a/cts/CTSlab.py.in
	+++ b/cts/CTSlab.py.in
	@@ -1,519 +1,523 @@
	#!@PYTHON@

	'''CTS: Cluster Testing System: Lab environment module


	'''

	__copyright__='''
	Copyright (C) 2001 Alan Robertson <alanr@unix.sh>
	Licensed under the GNU GPL.
	'''

	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public License
	# as published by the Free Software Foundation; either version 2
	# of the License, or (at your option) any later version.
	#
	# This program is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	# GNU General Public License for more details.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write to the Free Software
	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

	from UserDict import UserDict
	import sys, time, types, syslog, whrandom, os, struct, string
	from CTS import ClusterManager
	from CM_hb import HeartbeatCM
	from socket import gethostbyname_ex

	class ResetMechanism:
	def reset(self, node):
	raise ValueError("Abstract class member (reset)")

	class Stonith(ResetMechanism):
	def __init__(self, sttype="ssh", parm="foobar"
	, path="@sbindir@/stonith"):
	self.pathname=path
	self.configstring=parm
	self.stonithtype=sttype

	def reset(self, node):
	cmdstring = "%s -t '%s' -p '%s' '%s' 2>/dev/null" % (self.pathname
	, self.stonithtype, self.configstring, node)
	return (os.system(cmdstring) == 0)

	class Logger:
	TimeFormat = "%Y/%m/%d_%H:%M:%S\t"

	def __call__(self, lines):
	raise ValueError("Abstract class member (__call__)")

	class SysLog(Logger):
	# http://docs.python.org/lib/module-syslog.html
	defaultsource="CTS"
	defaultfacility= syslog.LOG_LOCAL7
	map = {
	"kernel": syslog.LOG_KERN,
	"user": syslog.LOG_USER,
	"mail": syslog.LOG_MAIL,
	"daemon": syslog.LOG_MAIL,
	"auth": syslog.LOG_AUTH,
	"lpr": syslog.LOG_LPR,
	"news": syslog.LOG_NEWS,
	"uucp": syslog.LOG_UUCP,
	"cron": syslog.LOG_CRON,
	"local0": syslog.LOG_LOCAL0,
	"local1": syslog.LOG_LOCAL1,
	"local2": syslog.LOG_LOCAL2,
	"local3": syslog.LOG_LOCAL3,
	"local4": syslog.LOG_LOCAL4,
	"local5": syslog.LOG_LOCAL5,
	"local6": syslog.LOG_LOCAL6,
	"local7": syslog.LOG_LOCAL7,
	}
	def __init__(self, labinfo):

	if labinfo.has_key("syslogsource"):
	self.source=labinfo["syslogsource"]
	else:
	self.source=SysLog.defaultsource

	if labinfo.has_key("SyslogFacility"):
	self.facility=labinfo["SyslogFacility"]
	if SysLog.map.has_key(self.facility):
	self.facility=SysLog.map[self.facility]
	else:
	self.facility=SysLog.defaultfacility

	syslog.openlog(self.source, 0, self.facility)

	def setfacility(self, facility):
	self.facility = facility
	if SysLog.map.has_key(self.facility):
	self.facility=SysLog.map[self.facility]
	syslog.closelog()
	syslog.openlog(self.source, 0, self.facility)


	def __call__(self, lines):
	if isinstance(lines, types.StringType):
	syslog.syslog(lines)
	else:
	for line in lines:
	syslog.syslog(line)

	def name(self):
	return "Syslog"

	class StdErrLog(Logger):

	def __init__(self, labinfo):
	pass

	def __call__(self, lines):
	t = time.strftime(Logger.TimeFormat, time.localtime(time.time()))
	if isinstance(lines, types.StringType):
	sys.__stderr__.writelines([t, lines, "\n"])
	else:
	for line in lines:
	sys.__stderr__.writelines([t, line, "\n"])
	sys.__stderr__.flush()

	def name(self):
	return "StdErrLog"

	class FileLog(Logger):
	def __init__(self, labinfo, filename=None):

	if filename == None:
	filename=labinfo["logfile"]

	self.logfile=filename

	def __call__(self, lines):

	fd = open(self.logfile, "a")
	t = time.strftime(Logger.TimeFormat, time.localtime(time.time()))

	if isinstance(lines, types.StringType):
	fd.writelines([t, lines, "\n"])
	else:
	for line in lines:
	fd.writelines([t, line, "\n"])
	fd.close()

	def name(self):
	return "FileLog"

	class CtsLab(UserDict):
	'''This class defines the Lab Environment for the Cluster Test System.
	It defines those things which are expected to change from test
	environment to test environment for the same cluster manager.

	It is where you define the set of nodes that are in your test lab
	what kind of reset mechanism you use, etc.

	This class is derived from a UserDict because we hold many
	different parameters of different kinds, and this provides
	provide a uniform and extensible interface useful for any kind of
	communication between the user/administrator/tester and CTS.

	At this point in time, it is the intent of this class to model static
	configuration and/or environmental data about the environment which
	doesn't change as the tests proceed.

	Well-known names (keys) are an important concept in this class.
	The HasMinimalKeys member function knows the minimal set of
	well-known names for the class.

	The following names are standard (well-known) at this time:

	nodes An array of the nodes in the cluster
	reset A ResetMechanism object
	logger An array of objects that log strings...
	CMclass The type of ClusterManager we are running
	(This is a class object, not a class instance)
	RandSeed Random seed. It is a triple of bytes. (optional)
	HAdir Base directory for HA installation

	The CTS code ignores names it doesn't know about/need.
	The individual tests have access to this information, and it is
	perfectly acceptable to provide hints, tweaks, fine-tuning
	directions or other information to the tests through this mechanism.
	'''

	def __init__(self, nodes):
	self.data = {}
	self["nodes"] = nodes
	self.MinimalKeys=["nodes", "reset", "logger", "CMclass", "HAdir"]

	def HasMinimalKeys(self):
	'Return TRUE if our object has the minimal set of keys/values in it'
	result = 1
	for key in self.MinimalKeys:
	if not self.has_key(key):
	result = None
	return result

	def SupplyDefaults(self):
	if not self.has_key("logger"):
	self["logger"] = (SysLog(self), StdErrLog(self))
	if not self.has_key("reset"):
	self["reset"] = Stonith()
	if not self.has_key("CMclass"):
	self["CMclass"] = HeartbeatCM
	if not self.has_key("HAdir"):
	self["HAdir"] = "@sysconfdir@/ha.d"
	if not self.has_key("LogFileName"):
	self["LogFileName"] = "/var/log/ha-log"

	#
	# Now set up our random number generator...
	#
	self.RandomGen = whrandom.whrandom()

	# Get a random seed for the random number generator.

	if self.has_key("RandSeed"):
	randseed = self["RandSeed"]
	else:
	f=open("/dev/urandom", "r")
	string=f.read(3)
	f.close()
	randseed=struct.unpack("BBB", string)

	self.log("Random seed is: " + str(randseed))
	self.randseed=randseed

	self.RandomGen.seed(randseed[0], randseed[1], randseed[2])

	def log(self, args):
	"Log using each of the supplied logging methods"
	for logfcn in self._logfunctions:
	logfcn(string.strip(args))

	def debug(self, args):
	"Log using each of the supplied logging methods"
	for logfcn in self._logfunctions:
	if logfcn.name() != "StdErrLog":
	logfcn(string.strip(args))

	def __setitem__(self, key, value):
	'''Since this function gets called whenever we modify the
	dictionary (object), we can (and do) validate those keys that we
	know how to validate. For the most part, we know how to validate
	the "MinimalKeys" elements.
	'''

	#
	# List of nodes in the system
	#
	if key == "nodes":
	self.Nodes = {}
	for node in value:
	# I don't think I need the IP address, etc. but this validates
	# the node name against /etc/hosts and/or DNS, so it's a
	# GoodThing(tm).
	self.Nodes[node] = gethostbyname_ex(node)
	if len(value) < 2:
	raise ValueError("Must have at least two nodes in system")

	#
	# Reset Mechanism
	#
	elif key == "reset":
	if not issubclass(value.__class__, ResetMechanism):
	raise ValueError("'reset' Value must be a subclass"
	" of ResetMechanism")
	#
	# List of Logging Mechanism(s)
	#
	elif key == "logger":
	if len(value) < 1:
	raise ValueError("Must have at least one logging mechanism")
	for logger in value:
	if not callable(logger):
	raise ValueError("'logger' elements must be callable")
	self._logfunctions = value
	#
	# Cluster Manager Class
	#
	elif key == "CMclass":
	if not issubclass(value, ClusterManager):
	raise ValueError("'CMclass' must be a subclass of"
	" ClusterManager")
	#
	# Initial Random seed...
	#
	elif key == "RandSeed":
	if len(value) != 3:
	raise ValueError("'Randseed' must be a 3-element list/tuple")
	for elem in value:
	if not isinstance(elem, types.IntType):
	raise ValueError("'Randseed' list must all be ints")

	self.data[key] = value

	def IsValidNode(self, node):
	'Return TRUE if the given node is valid'
	return self.Nodes.has_key(node)

	def __CheckNode(self, node):
	"Raise a ValueError if the given node isn't valid"

	if not self.IsValidNode(node):
	raise ValueError("Invalid node [%s] in CheckNode" % node)

	def RandomNode(self):
	'''Choose a random node from the cluster'''
	return self.RandomGen.choice(self["nodes"])

	def ResetNode(self, node):
	"Reset a node, (normally) using a hardware mechanism"
	self.__CheckNode(node)
	return self["reset"].reset(node)

	def usage(arg):
	print "Illegal argument " + arg
	print "usage: " + sys.argv[0] \
	+ " --directory config-directory" \
	+ " -D config-directory" \
	+ " --logfile system-logfile-name" \
	+ " -L system-logfile-name" \
	+ " --choose testcase" \
	+ + " --resource-can-stop" \
	+ " -v2"\
	+ " --stonith (1 \| 0 \| yes \| no)" \
	+ " --standby (1 \| 0 \| yes \| no)" \
	+ " --fencing (1 \| 0 \| yes \| no)" \
	+ " [number-of-iterations]"
	sys.exit(1)


	#
	# A little test code...
	#
	if __name__ == '__main__':

	from CTSaudits import AuditList
	from CTStests import TestList,RandomTests
	from CTS import Scenario, InitClusterManager, PingFest
	import CM_hb

	HAdir = "/etc/ha.d"
	LogFile = "/var/log/ha-log-local7"
	DoStonith = 1
	DoStandby = 1
	DoFencing = 0
	NumIter = 500
	SuppressMonitoring = None
	Version = 1
	CIBfilename = None
	CIBResource = 0
	ClobberCIB = 0
	LimitNodes = 0
	TestCase = None
	LogFacility = None
	-
	+ ResCanStop = 0
	#
	# The values of the rest of the parameters are now properly derived from
	# the configuration files.
	#
	# Stonith is configurable because it's slow, I have a few machines which
	# don't reboot very reliably, and it can mild damage to your machine if
	# you're using a real power switch.
	#
	# Standby is configurable because the test is very heartbeat specific
	# and I haven't written the code to set it properly yet. Patches are
	# being accepted...



	# Process arguments...

	skipthis=None
	args=sys.argv[1:]
	for i in range(0, len(args)):
	if skipthis:
	skipthis=None
	continue
	elif args[i] == "-D" or args[i] == "--directory":
	skipthis=1
	HAdir = args[i+1]
	elif args[i] == "-l" or args[i] == "--limit-nodes":
	skipthis=1
	LimitNodes = int(args[i+1])
	elif args[i] == "-r":
	CIBResource = 1
	elif args[i] == "-L" or args[i] == "--logfile":
	skipthis=1
	LogFile = args[i+1]
	elif args[i] == "-v2":
	Version=2
	elif args[i] == "--stonith":
	skipthis=1
	if args[i+1] == "1" or args[i+1] == "yes":
	DoStonith=1
	elif args[i+1] == "0" or args[i+1] == "no":
	DoStonith=0
	else:
	usage(args[i+1])
	elif args[i] == "--standby":
	skipthis=1
	if args[i+1] == "1" or args[i+1] == "yes":
	DoStandby=1
	elif args[i+1] == "0" or args[i+1] == "no":
	DoStandby=0
	else:
	usage(args[i+1])
	elif args[i] == "--fencing":
	skipthis=1
	if args[i+1] == "1" or args[i+1] == "yes":
	DoFencing=1
	elif args[i+1] == "0" or args[i+1] == "no":
	DoFencing=0
	else:
	usage(args[i+1])
	elif args[i] == "--suppressmonitoring":
	SuppressMonitoring = 1
	+ elif args[i] == "--resource-can-stop":
	+ ResCanStop = 1
	elif args[i] == "-2" or args[i] == "--crm":
	Version = 2
	elif args[i] == "-1" or args[i] == "--classic":
	Version = 1
	elif args[i] == "--clobber-cib" or args[i] == "-c":
	ClobberCIB = 1
	elif args[i] == "--cib-filename":
	skipthis=1
	CIBfilename = args[i+1]
	elif args[i] == "--choose":
	skipthis=1
	TestCase = args[i+1]
	elif args[i] == "--syslog-facility" or args[i] == "--facility":
	skipthis=1
	LogFacility = args[i+1]
	else:
	NumIter=int(args[i])

	#
	# This reading of HBconfig here is ugly, and I suppose ought to
	# be done by the Cluster manager. This would probably mean moving the
	# list of cluster nodes into the ClusterManager class. A good thought
	# for our Copious Spare Time in the future...
	#
	config = CM_hb.HBConfig(HAdir)
	node_list = config.Parameters["node"]
	if LogFacility == None:
	if config.Parameters.has_key("logfacility"):
	LogFacility = config.Parameters["logfacility"][0]
	else:
	LogFacility = "local7"

	if LimitNodes > 0:
	if len(node_list) > LimitNodes:
	print("Limiting the number of nodes configured=%d (max=%d)"
	%(len(node_list), LimitNodes))
	while len(node_list) > LimitNodes:
	node_list.pop(len(node_list)-1)

	Environment = CtsLab(node_list)
	Environment["HAdir"] = HAdir
	Environment["ClobberCIB"] = ClobberCIB
	Environment["CIBfilename"] = CIBfilename
	Environment["CIBResource"] = CIBResource
	Environment["LogFileName"] = LogFile
	Environment["DoStonith"] = DoStonith
	Environment["SyslogFacility"] = LogFacility
	Environment["DoStandby"] = DoStandby
	Environment["DoFencing"] = DoFencing
	+ Environment["ResCanStop"] = ResCanStop

	Environment["SuppressMonitoring"] = SuppressMonitoring
	if Version == 2:
	from CM_LinuxHAv2 import LinuxHAv2
	Environment['CMclass']=LinuxHAv2
	#Environment["RandSeed"] = (156, 104, 218)

	Environment.SupplyDefaults()

	# Your basic start up the world type of test scenario...

	#scenario = Scenario(
	#[ InitClusterManager(Environment)
	#, PingFest(Environment)])
	scenario = Scenario(
	[ InitClusterManager(Environment)])

	# Create the Cluster Manager object

	cm = Environment['CMclass'](Environment)


	cm.log(">>>>>>>>>>>>>>>> BEGINNING " + repr(NumIter) + " TESTS ")
	cm.log("HA configuration directory: " + Environment["HAdir"])
	cm.log("System log files: " + Environment["LogFileName"])
	cm.log("Enable Stonith: " + ("%d" % Environment["DoStonith"]))
	cm.log("Enable Standby: " + ("%d" % Environment["DoStandby"]))
	if Environment.has_key("SuppressMonitoring") \
	and Environment["SuppressMonitoring"]:
	cm.log("Resource Monitoring is disabled")
	cm.log("Cluster nodes: " + repr(config.Parameters["node"]))

	Audits = AuditList(cm)
	Tests = []

	if TestCase != None:
	for test in TestList(cm):
	if test.name == TestCase:
	Tests.append(test)
	if Tests == []:
	usage("--choose: No applicable/valid tests chosen")
	else:
	Tests = TestList(cm)

	tests = RandomTests(scenario, cm, Tests, Audits)
	Environment.RandomTests = tests
	overall, detailed = tests.run(NumIter)
	tests.summarize()

File Metadata

Mime Type: text/x-diff
Expires: Thu, Jul 10, 1:50 AM (1 d, 9 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2009589
Default Alt Text: (39 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions