diff --git a/cts/CIB.py b/cts/CIB.py index b12f17f9cb..ad7b2ba6a3 100644 --- a/cts/CIB.py +++ b/cts/CIB.py @@ -1,282 +1,741 @@ '''CTS: Cluster Testing System: CIB generator ''' __copyright__=''' Author: Andrew Beekhof Copyright (C) 2008 Andrew Beekhof ''' from UserDict import UserDict import sys, time, types, syslog, os, struct, string, signal, traceback, warnings, socket from cts.CTSvars import * -from cts.CTS import ClusterManager +from cts.CTS import ClusterManager, RemoteExec class CibBase: cts_cib = None cib_tmpfile = None version = "unknown" feature_set = "unknown" - target = None + Factory = None - def __init__(self, CM, tmpfile=None): + def __init__(self, CM, factory, tmpfile=None): self.CM = CM - #self.target = self.CM.Env["nodes"][0] + self.Factory = factory if not tmpfile: warnings.filterwarnings("ignore") self.cib_tmpfile=os.tmpnam() warnings.resetwarnings() else: self.cib_tmpfile = tmpfile + self.Factory.tmpfile = self.cib_tmpfile + def version(self): return self.version def NextIP(self): fields = string.split(self.CM.Env["IPBase"], '.') fields[3] = str(int(fields[3])+1) ip = string.join(fields, '.') self.CM.Env["IPBase"] = ip return ip class CIB10(CibBase): feature_set = "3.0" version = "pacemaker-1.0" cib_template = ''' ''' def _create(self, command): - fixed = "HOME=/root CIB_file="+self.cib_tmpfile+" crm --force configure " + command - rc = self.CM.rsh(self.target, fixed) + fixed = "HOME=/root CIB_file="+self.Factory.tmpfile+" crm --force configure " + command + rc = self.CM.rsh(self.Factory.target, fixed) if rc != 0: self.CM.log("Configure call failed: "+fixed) sys.exit(1) def _show(self, command=""): output = "" - (rc, result) = self.CM.rsh(self.target, "HOME=/root CIB_file="+self.cib_tmpfile+" crm configure show "+command, None, ) + (rc, result) = self.CM.rsh(self.Factory.target, "HOME=/root CIB_file="+self.Factory.tmpfile+" crm configure show "+command, None, ) for line in result: output += line self.CM.debug("Generated Config: "+line) return output def NewIP(self, name=None, standard="ocf:heartbeat"): ip = self.NextIP() if not name: name = "r"+ip if not standard: standard = "" else: standard += ":" self._create('''primitive %s %sIPaddr params ip=%s cidr_netmask=32 op monitor interval=5s''' % (name, standard, ip)) return name def install(self, target): - old = self.cib_tmpfile + old = self.Factory.tmpfile # Force a rebuild self.cts_cib = None - self.cib_tmpfile = CTSvars.CRM_CONFIG_DIR+"/cib.xml" + self.Factory.tmpfile = CTSvars.CRM_CONFIG_DIR+"/cib.xml" self.contents(target) - self.CM.rsh(self.target, "chown "+CTSvars.CRM_DAEMON_USER+" "+self.cib_tmpfile) + self.CM.rsh(self.Factory.target, "chown "+CTSvars.CRM_DAEMON_USER+" "+self.Factory.tmpfile) - self.cib_tmpfile = old + self.Factory.tmpfile = old def contents(self, target=None): # fencing resource if self.cts_cib: return self.cts_cib - if not target: - self.target = self.CM.Env["nodes"][0] - else: - self.target = target + if target: + self.Factory.target = target cib_base = self.cib_template % (self.feature_set, self.version, ''' remote-tls-port='9898' remote-clear-port='9999' ''') - self.CM.rsh(self.target, '''echo "%s" > %s''' % (cib_base, self.cib_tmpfile)) - #self.CM.rsh.cp(self.cib_tmpfile, "root@%s:%s" % (self.target, self.cib_tmpfile)) + self.CM.rsh(self.Factory.target, '''echo "%s" > %s''' % (cib_base, self.Factory.tmpfile)) + #self.CM.rsh.cp(self.Factory.tmpfile, "root@%s:%s" % (self.Factory.target, self.Factory.tmpfile)) nodelist = "" self.num_nodes = 0 for node in self.CM.Env["nodes"]: nodelist += node + " " self.num_nodes = self.num_nodes + 1 no_quorum = "stop" if self.num_nodes < 3: no_quorum = "ignore" self.CM.log("Cluster only has %d nodes, configuring: no-quroum-policy=ignore" % self.num_nodes) # The shell no longer functions when the lrmd isn't running, how wonderful # Start one here and let the cluster clean it up when the full stack starts # Just hope target has the same location for lrmd - self.CM.rsh(self.target, CTSvars.CRM_DAEMON_DIR+"/lrmd", synchronous=0) + self.CM.rsh(self.Factory.target, CTSvars.CRM_DAEMON_DIR+"/lrmd", synchronous=0) # Tell the shell to mind its own business, we know what we're doing - self.CM.rsh(self.target, "crm options check-mode relaxed") + self.CM.rsh(self.Factory.target, "crm options check-mode relaxed") # Fencing resource # Define first so that the shell doesn't reject every update if self.CM.Env["DoFencing"]: params = None entries = string.split(self.CM.Env["stonith-params"], ',') for entry in entries: (name, value) = string.split(entry, '=') if name == "hostlist" and value == "all": value = string.join(self.CM.Env["nodes"], " ") if params: params = ("""%s '%s="%s"' """ % (params, name, value)) else: params = ("""'%s="%s"' """ % (name, value)) if params: params = "params %s" % params else: params = "" # Set a threshold for unreliable stonith devices such as the vmware one self._create('''primitive Fencing stonith::%s %s meta migration-threshold=5 op monitor interval=120s timeout=300 op start interval=0 timeout=180s op stop interval=0 timeout=180s''' % (self.CM.Env["stonith-type"], params)) self._create('''property stonith-enabled=%s''' % (self.CM.Env["DoFencing"])) self._create('''property start-failure-is-fatal=false pe-input-series-max=5000 default-action-timeout=60s''') self._create('''property shutdown-escalation=5min batch-limit=10 dc-deadtime=5s''') self._create('''property no-quorum-policy=%s expected-quorum-votes=%d''' % (no_quorum, self.num_nodes)) if self.CM.Env["DoBSC"] == 1: self._create('''property ident-string="Linux-HA TEST configuration file - REMOVEME!!"''') # Add resources? if self.CM.Env["CIBResource"] == 1: self.add_resources() if self.CM.cluster_monitor == 1: self._create('''primitive cluster_mon ocf:pacemaker:ClusterMon params update=10 extra_options="-r -n" user=abeekhof htmlfile=/suse/abeekhof/Export/cluster.html op start interval=0 requires=nothing op monitor interval=5s requires=nothing''') self._create('''location prefer-dc cluster_mon rule -INFINITY: \#is_dc eq false''') # generate cib self.cts_cib = self._show("xml") - if self.cib_tmpfile != CTSvars.CRM_CONFIG_DIR+"/cib.xml": - self.CM.rsh(self.target, "rm -f "+self.cib_tmpfile) + if self.Factory.tmpfile != CTSvars.CRM_CONFIG_DIR+"/cib.xml": + self.CM.rsh(self.Factory.target, "rm -f "+self.Factory.tmpfile) return self.cts_cib def add_resources(self): # Group Resource r1 = self.NewIP() #ip = self.NextIP() #r2 = "r"+ip #self._create('''primitive %s heartbeat::IPaddr params 1=%s/32 op monitor interval=5s''' % (r2, ip)) r2 = self.NewIP() r3 = self.NewIP() self._create('''group group-1 %s %s %s''' % (r1, r2, r3)) # Per-node resources for node in self.CM.Env["nodes"]: r = self.NewIP("rsc_"+node) self._create('''location prefer-%s %s rule 100: \#uname eq %s''' % (node, r, node)) # LSB resource lsb_agent = self.CM.install_helper("LSBDummy") self._create('''primitive lsb-dummy lsb::''' +lsb_agent+ ''' op monitor interval=5s''') self._create('''colocation lsb-with-group INFINITY: lsb-dummy group-1''') self._create('''order lsb-after-group mandatory: group-1 lsb-dummy symmetrical=true''') # Migrator # Make this slightly sticky (since we have no other location constraints) to avoid relocation during Reattach self._create('''primitive migrator ocf:pacemaker:Dummy meta resource-stickiness=1 allow-migrate=1 op monitor interval=P10S''') # Ping the test master self._create('''primitive ping-1 ocf:pacemaker:ping params host_list=%s name=connected debug=true op monitor interval=60s''' % self.CM.Env["cts-master"]) self._create('''clone Connectivity ping-1 meta globally-unique=false''') #master slave resource self._create('''primitive stateful-1 ocf:pacemaker:Stateful op monitor interval=15s timeout=60s op monitor interval=16s role=Master timeout=60s ''') self._create('''ms master-1 stateful-1 meta clone-max=%d clone-node-max=%d master-max=%d master-node-max=%d''' % (self.num_nodes, 1, 1, 1)) # Require conectivity to run the master self._create('''location %s-is-connected %s rule -INFINITY: connected lt %d or not_defined connected''' % ("m1", "master-1", 1)) # Group with the master self._create('''colocation group-with-master INFINITY: group-1 master-1:Master''') self._create('''order group-after-master mandatory: master-1:promote group-1:start symmetrical=true''') +class Option: + def __init__(self, Factory, name, value, section="cib-bootstrap-options"): + self.Factory = Factory + self.id = "%s-%s" % (section, name) + self.section = section + self.name = name + self.value = value + + self.target = "pcmk-1" + self.cib_tmpfile = CTSvars.CRM_CONFIG_DIR+"/cib.xml" + + def show(self): + text = '''''' + text += ''' ''' % self.section + text += ''' ''' % (self.id, self.name, self.value) + text += ''' ''' + text += '''''' + return text + + def commit(self): + self.Factory.debug("Writing out %s" % self.id) + fixed = "HOME=/root CIB_file="+self.cib_tmpfile+" cibadmin --modify --xml-text '%s'" % self.show() + rc = self.Factory.rsh(self.target, fixed) + if rc != 0: + self.Factory.log("Configure call failed: "+fixed) + sys.exit(1) + +class CibXml: + def __init__(self, tag, name, **kwargs): + self.tag = tag + self.name = name + self.kwargs = kwargs + + def __setitem__(self, key, value): + self.kwargs[key] = value + + def show(self): + text = '''<%s id="%s"''' % (self.tag, self.name) + for k in self.kwargs.keys(): + text += ''' %s="%s"''' % (k, self.kwargs[k]) + text += '''/>''' + return text + +class Expression(CibXml): + def __init__(self, name, attr, op, value=None): + CibXml.__init__(self, "expression", name, attribute=attr, operation=op) + if value: + self["value"] = value + +class ResourceOp(CibXml): + def __init__(self, resource, name, interval, **kwargs): + CibXml.__init__(self, "op", "%s-%s-%s" % (resource, name, interval), **kwargs) + self["name"] = name + self["interval"] = interval + +class Rule: + def __init__(self, name, score, op="and", expr=None): + self.id = name + self.op = op + self.score = score + self.expr = [] + if expr: + self.add_exp(expr) + + def add_exp(self, e): + self.expr.append(e) + + def show(self): + text = '''''' % (self.id, self.score) + for e in self.expr: + text += e.show() + text += '''''' + return text + +class Resource: + def __init__(self, Factory, name, rtype, standard, provider=None): + self.Factory = Factory + + self.name = name + self.rtype = rtype + self.standard = standard + self.provider = provider + + self.op=[] + self.meta={} + self.param={} + + self.scores={} + self.needs={} + self.coloc={} + + if self.standard == "ocf" and not provider: + self.provider = "heartbeat" + elif self.standard == "lsb": + self.provider = None + + def __setitem__(self, key, value): + self.add_param(key, value) + + def add_op(self, name, interval, **kwargs): + self.op.append(ResourceOp(self.name, name, interval, **kwargs)) + + def add_param(self, name, value): + self.param[name] = value + + def add_meta(self, name, value): + self.meta[name] = value + + def prefer(self, node, score="INFINITY", rule=None): + if not rule: + rule = Rule("prefer-%s-r" % node, score, expr=Expression("prefer-%s-e" % node, "#uname", "eq", node)) + self.scores[node] = rule + + def _needs(self, resource, kind="Mandatory", first="start", then="start", **kwargs): + kargs = kwargs.copy() + kargs["kind"] = kind + if then: + kargs["first-action"] = "start" + kargs["then-action"] = then + + if first: + kargs["first-action"] = first + + self.needs[resource] = kargs + + def _coloc(self, resource, score="INFINITY", role=None, withrole=None, **kwargs): + kargs = kwargs.copy() + kargs["score"] = score + if role: + kargs["rsc-role"] = role + if withrole: + kargs["with-rsc-role"] = withrole + + self.coloc[resource] = kargs + + def constraints(self): + text = "" + + for k in self.scores.keys(): + text += '''''' % (k, self.name) + text += self.scores[k].show() + text += '''''' + + for k in self.needs.keys(): + text += '''''' + + for k in self.coloc.keys(): + text += '''''' + + text += "" + return text + + def show(self): + text = '''''' + + if len(self.meta) > 0: + text += '''''' % self.name + for p in self.meta.keys(): + text += '''''' % (self.name, p, p, self.meta[p]) + text += '''''' + + if len(self.param) > 0: + text += '''''' % self.name + for p in self.param.keys(): + text += '''''' % (self.name, p, p, self.param[p]) + text += '''''' + + if len(self.op) > 0: + text += '''''' + for o in self.op: + text += o.show() + text += '''''' + + text += '''''' + return text + + def commit(self): + self.Factory.debug("Writing out %s" % self.name) + fixed = "HOME=/root CIB_file="+self.Factory.tmpfile+" cibadmin --create --scope resources --xml-text '%s'" % self.show() + rc = self.Factory.rsh(self.Factory.target, fixed) + if rc != 0: + self.Factory.log("Configure call failed: "+fixed) + sys.exit(1) + + fixed = "HOME=/root CIB_file="+self.Factory.tmpfile+" cibadmin --modify --xml-text '%s'" % self.constraints() + rc = self.Factory.rsh(self.Factory.target, fixed) + if rc != 0: + self.Factory.log("Configure call failed: "+fixed) + sys.exit(1) + +class Group(Resource): + def __init__(self, Factory, name): + self.name = name + self.children = [] + self.object = "group" + Resource.__init__(self, Factory, name, None, None) + + def add_child(self, resource): + self.children.append(resource) + + def __setitem__(self, key, value): + self.add_meta(key, value) + + def show(self): + text = '''<%s id="%s">''' % (self.object, self.name) + + if len(self.meta) > 0: + text += '''''' % self.name + for p in self.meta.keys(): + text += '''''' % (self.name, p, p, self.meta[p]) + text += '''''' + + for c in self.children: + text += c.show() + text += '''''' % self.object + return text + +class Clone(Group): + def __init__(self, Factory, name, child=None): + Group.__init__(self, Factory, name) + self.object = "clone" + if child: + self.add_child(child) + + def add_child(self, resource): + if not self.children: + self.children.append(resource) + else: + self.Factory.log("Clones can only have a single child. Ignoring %s" % resource.name) + +class Master(Clone): + def __init__(self, Factory, name, child=None): + Clone.__init__(self, Factory, name, child) + self.object = "master" + +class CIB12(CibBase): + feature_set = "3.0" + version = "pacemaker-1.2" + + def _show(self, command=""): + output = "" + (rc, result) = self.Factory.rsh(self.Factory.target, "HOME=/root CIB_file="+self.Factory.tmpfile+" cibadmin -Ql "+command, None, ) + for line in result: + output += line + self.Factory.debug("Generated Config: "+line) + return output + + def NewIP(self, name=None, standard="ocf"): + ip = self.NextIP() + if not name: + name = "r"+ip + + r = Resource(self.Factory, name, "IPaddr2", standard) + r["ip"] = ip + r["cidr_netmask"] = "32" + r.add_op("monitor", "5s") + return r + + def install(self, target): + old = self.Factory.tmpfile + + # Force a rebuild + self.cts_cib = None + + self.Factory.tmpfile = CTSvars.CRM_CONFIG_DIR+"/cib.xml" + self.contents(target) + self.Factory.rsh(self.Factory.target, "chown "+CTSvars.CRM_DAEMON_USER+" "+self.Factory.tmpfile) + + self.Factory.tmpfile = old + + def contents(self, target=None): + # fencing resource + if self.cts_cib: + return self.cts_cib + + if target: + self.Factory.target = target + + self.Factory.rsh(self.Factory.target, "HOME=/root cibadmin --empty > %s" % self.Factory.tmpfile) + #cib_base = self.cib_template % (self.feature_set, self.version, ''' remote-tls-port='9898' remote-clear-port='9999' ''') + + nodelist = "" + self.num_nodes = 0 + for node in self.CM.Env["nodes"]: + nodelist += node + " " + self.num_nodes = self.num_nodes + 1 + + no_quorum = "stop" + if self.num_nodes < 3: + no_quorum = "ignore" + self.Factory.log("Cluster only has %d nodes, configuring: no-quroum-policy=ignore" % self.num_nodes) + + # Fencing resource + # Define first so that the shell doesn't reject every update + if self.CM.Env["DoFencing"]: + st = Resource(self.Factory, "Fencing", self.CM.Env["stonith-type"], "stonith") + # Set a threshold for unreliable stonith devices such as the vmware one + st.add_meta("migration-threshold", "5") + st.add_op("monitor", "120s", timeout="300s") + st.add_op("stop", "0", timeout="180s") + st.add_op("start", "0", timeout="180s") + + entries = string.split(self.CM.Env["stonith-params"], ',') + for entry in entries: + (name, value) = string.split(entry, '=') + if name == "hostlist" and value == "all": + value = string.join(self.CM.Env["nodes"], " ") + + st[name] = value + + st.commit() + + Option(self.Factory, "stonith-enabled", self.CM.Env["DoFencing"]).commit() + Option(self.Factory, "start-failure-is-fatal", "false").commit() + Option(self.Factory, "pe-input-series-max", "5000").commit() + Option(self.Factory, "default-action-timeout", "60s").commit() + Option(self.Factory, "shutdown-escalation", "5min").commit() + Option(self.Factory, "batch-limit", "10").commit() + Option(self.Factory, "dc-deadtime", "5s").commit() + Option(self.Factory, "no-quorum-policy", no_quorum).commit() + Option(self.Factory, "expected-quorum-votes", self.num_nodes).commit() + + if self.CM.Env["DoBSC"] == 1: + Option(self.Factory, "ident-string", "Linux-HA TEST configuration file - REMOVEME!!").commit() + + # Add resources? + if self.CM.Env["CIBResource"] == 1: + self.add_resources() + + if self.CM.cluster_monitor == 1: + mon = Resource(self.Factory, "cluster_mon", "ocf", "ClusterMon", "pacemaker") + mon.add_op("start", "0", requires="nothing") + mon.add_op("monitor", "5s", requires="nothing") + mon["update"] = "10" + mon["extra_options"] = "-r -n" + mon["user"] = "abeekhof" + mon["htmlfile"] = "/suse/abeekhof/Export/cluster.html" + mon.commit() + + #self._create('''location prefer-dc cluster_mon rule -INFINITY: \#is_dc eq false''') + + # generate cib + self.cts_cib = self._show() + + if self.Factory.tmpfile != CTSvars.CRM_CONFIG_DIR+"/cib.xml": + self.Factory.rsh(self.Factory.target, "rm -f "+self.Factory.tmpfile) + + return self.cts_cib + + def add_resources(self): + # Per-node resources + for node in self.CM.Env["nodes"]: + name = "rsc_"+node + r = self.NewIP(name) + r.prefer(node, "100") + r.commit() + + # Migrator + # Make this slightly sticky (since we have no other location constraints) to avoid relocation during Reattach + m = Resource(self.Factory, "migrator","Dummy", "ocf", "pacemaker") + m.add_meta("resource-stickiness","1") + m.add_meta("allow-migrate", "1") + m.add_op("monitor", "P10S") + m.commit() + + # Ping the test master + p = Resource(self.Factory, "ping-1","ping", "ocf", "pacemaker") + p.add_op("monitor", "60s") + p["host-list"] = self.CM.Env["cts-master"] + p["name"] = "connected" + p["debug"] = "true" + + c = Clone(self.Factory, "Connectivity", p) + c["globally-unique"] = "false" + c.commit() + + #master slave resource + s = Resource(self.Factory, "stateful-1", "Stateful", "ocf", "pacemaker") + s.add_op("monitor", "15s", timeout="60s") + s.add_op("monitor", "16s", timeout="60s", role="Master") + ms = Master(self.Factory, "master-1", s) + ms["clone-max"] = self.num_nodes + ms["master-max"] = 1 + ms["clone-node-max"] = 1 + ms["master-node-max"] = 1 + + # Require conectivity to run the master + r = Rule("connected", "-INFINITY", op="or") + r.add_exp(Expression("m1-connected-1", "connected", "lt", "1")) + r.add_exp(Expression("m1-connected-2", "connected", "not_defined", None)) + ms.prefer("connected", rule=r) + + ms.commit() + + # Group Resource + g = Group(self.Factory, "group-1") + g.add_child(self.NewIP()) + g.add_child(self.NewIP()) + g.add_child(self.NewIP()) + + # Group with the master + g._coloc("master-1", "INFINITY", withrole="Master") + g._needs("master-1", first="promote", then="start") + + g.commit() + + + # LSB resource + lsb_agent = self.CM.install_helper("LSBDummy") + + lsb = Resource(self.Factory, "lsb-dummy",lsb_agent, "lsb") + lsb.add_op("monitor", "5s") + + # LSB with group + lsb._needs("group-1") + lsb._coloc("group-1") + + lsb.commit() + class HASI(CIB10): def add_resources(self): # DLM resource self._create('''primitive dlm ocf:pacemaker:controld op monitor interval=120s''') self._create('''clone dlm-clone dlm meta globally-unique=false interleave=true''') # O2CB resource self._create('''primitive o2cb ocf:ocfs2:o2cb op monitor interval=120s''') self._create('''clone o2cb-clone o2cb meta globally-unique=false interleave=true''') self._create('''colocation o2cb-with-dlm INFINITY: o2cb-clone dlm-clone''') self._create('''order start-o2cb-after-dlm mandatory: dlm-clone o2cb-clone''') class ConfigFactory: def __init__(self, CM): self.CM = CM - self.register("pacemaker10", CIB10, CM) - self.register("hae", HASI, CM) + self.rsh = self.CM.rsh + self.register("pacemaker10", CIB10, CM, self) + self.register("pacemaker11", CIB12, CM, self) + self.register("hae", HASI, CM, self) + self.target = self.CM.Env["nodes"][0] + self.tmpfile = None + + def log(self, args): + self.CM.log("cib: %s" % args) + def debug(self, args): + self.CM.debug("cib: %s" % args) def register(self, methodName, constructor, *args, **kargs): """register a constructor""" _args = [constructor] _args.extend(args) setattr(self, methodName, apply(ConfigFactoryItem,_args, kargs)) def unregister(self, methodName): """unregister a constructor""" delattr(self, methodName) def createConfig(self, name="pacemaker-1.0"): if name == "pacemaker-1.0": name = "pacemaker10"; elif name == "pacemaker-1.1": name = "pacemaker11"; elif name == "pacemaker-1.2": name = "pacemaker12"; elif name == "hasi": name = "hae"; if hasattr(self, name): return getattr(self, name)() else: self.CM.log("Configuration variant '%s' is unknown. Defaulting to latest config" % name) return self.pacemaker10() class ConfigFactoryItem: def __init__(self, function, *args, **kargs): assert callable(function), "function should be a callable obj" self._function = function self._args = args self._kargs = kargs def __call__(self, *args, **kargs): """call function""" _args = list(self._args) _args.extend(args) _kargs = self._kargs.copy() _kargs.update(kargs) return apply(self._function,_args,_kargs) - -#CibFactory = ConfigFactory() +# Basic Sanity Testing +if __name__ == '__main__': + import CTSlab + env = CTSlab.LabEnvironment() + env["nodes"] = [] + env["nodes"].append("pcmk-1") + env["nodes"].append("pcmk-2") + env["nodes"].append("pcmk-3") + env["nodes"].append("pcmk-4") + + env["CIBResource"] = 1 + env["IPBase"] = "10.0.0.10" + env["DoStonith"]=1 + env["stonith-type"] = "fence_xvm" + env["stonith-params"] = "pcmk_arg_map=domain:uname" + + manager = ClusterManager(env) + manager.cluster_monitor = False + + CibFactory = ConfigFactory(manager) + cib = CibFactory.createConfig("pacemaker-1.1") + print cib.contents()