diff --git a/cts/cts-attrd.in b/cts/cts-attrd.in index ccc43e2f70..88e7964cf4 100644 --- a/cts/cts-attrd.in +++ b/cts/cts-attrd.in @@ -1,414 +1,414 @@ #!@PYTHON@ """Regression tests for Pacemaker's attribute daemon.""" # pylint doesn't like the module name "cts-attrd" which is an invalid complaint for this file # but probably something we want to continue warning about elsewhere # pylint: disable=invalid-name # pacemaker imports need to come after we modify sys.path, which pylint will complain about. # pylint: disable=wrong-import-position -__copyright__ = "Copyright 2023 the Pacemaker project contributors" +__copyright__ = "Copyright 2023-2025 the Pacemaker project contributors" __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" import argparse import os import subprocess import sys import tempfile # These imports allow running from a source checkout after running `make`. # Note that while this doesn't necessarily mean it will successfully run tests, # but being able to see --help output can be useful. if os.path.exists("@abs_top_srcdir@/python"): sys.path.insert(0, "@abs_top_srcdir@/python") # pylint: disable=comparison-of-constants,comparison-with-itself,condition-evals-to-constant if os.path.exists("@abs_top_builddir@/python") and "@abs_top_builddir@" != "@abs_top_srcdir@": sys.path.insert(0, "@abs_top_builddir@/python") from pacemaker.buildoptions import BuildOptions from pacemaker.exitstatus import ExitStatus from pacemaker._cts.corosync import Corosync from pacemaker._cts.process import killall, exit_if_proc_running from pacemaker._cts.test import Test, Tests TEST_DIR = sys.path[0] def update_path(): """Set the PATH environment variable appropriately for the tests.""" new_path = os.environ['PATH'] - if os.path.exists("%s/cts-attrd.in" % TEST_DIR): + if os.path.exists(f"{TEST_DIR}/cts-attrd.in"): # pylint: disable=protected-access - print("Running tests from the source tree: %s (%s)" % (BuildOptions._BUILD_DIR, TEST_DIR)) + print(f"Running tests from the source tree: {BuildOptions._BUILD_DIR} ({TEST_DIR})") # For pacemaker-attrd - new_path = "%s/daemons/attrd:%s" % (BuildOptions._BUILD_DIR, new_path) + new_path = f"{BuildOptions._BUILD_DIR}/daemons/attrd:{new_path}" else: - print("Running tests from the install tree: %s (not %s)" % (BuildOptions.DAEMON_DIR, TEST_DIR)) + print(f"Running tests from the install tree: {BuildOptions.DAEMON_DIR} (not {TEST_DIR})") # For pacemaker-attrd - new_path = "%s:%s" % (BuildOptions.DAEMON_DIR, new_path) + new_path = f"{BuildOptions.DAEMON_DIR}:{new_path}" - print('Using PATH="%s"' % new_path) + print(f'Using PATH="{new_path}"') os.environ['PATH'] = new_path class AttributeTest(Test): """Executor for a single test.""" def __init__(self, name, description, **kwargs): """ Create a new AttributeTest instance. Arguments: name -- A unique name for this test. This can be used on the command line to specify that only a specific test should be executed. description -- A meaningful description for the test. """ Test.__init__(self, name, description, **kwargs) self._daemon_location = "pacemaker-attrd" self._enable_corosync = True def _kill_daemons(self): killall([self._daemon_location]) def _start_daemons(self): if self.verbose: - print("Starting %s" % self._daemon_location) + print(f"Starting {self._daemon_location}") cmd = [self._daemon_location, "-s", "-l", self.logpath] # pylint: disable=consider-using-with self._daemon_process = subprocess.Popen(cmd) class AttributeTests(Tests): """Collection of all attribute regression tests.""" def __init__(self, **kwargs): """Create a new AttributeTests instance.""" Tests.__init__(self, **kwargs) self._corosync = Corosync(self.verbose, self.logdir, "cts-attrd") def new_test(self, name, description): """Create a named test.""" test = AttributeTest(name, description, verbose=self.verbose, logdir=self.logdir) self._tests.append(test) return test def setup_environment(self, use_corosync): """Prepare the host before executing any tests.""" if use_corosync: self._corosync.start(kill_first=True) def cleanup_environment(self, use_corosync): """Clean up the host after executing desired tests.""" if use_corosync: self._corosync.stop() def build_basic_tests(self): """Add basic tests - setting, querying, updating, and deleting attributes.""" test = self.new_test("set_attr_1", "Set and query an attribute") test.add_cmd("attrd_updater", args="--name AAA -U 111 --output-as=xml") test.add_cmd("attrd_updater", args="--name AAA -Q --output-as=xml", stdout_match='name="AAA" value="111"') test.add_cmd("attrd_updater", args="--name AAA -Q", stdout_match='name="AAA" host="[^"]+" value="111"', validate=False) test.add_log_pattern(r"Setting AAA\[.*\] in instance_attributes: \(unset\) -> 111", regex=True) # Setting the delay on an attribute that doesn't exist fails, but the failure is # not passed back to attrd_updater. test = self.new_test("set_attr_2", "Set an attribute's delay") test.add_cmd("attrd_updater", args="--name AAA -Y -d 5 --output-as=xml") test.add_log_pattern(r"Processed update-delay request from client .*: Error \(Attribute AAA does not exist\)", regex=True) test = self.new_test("set_attr_3", "Set and query an attribute's delay and value") test.add_cmd("attrd_updater", args="--name AAA -B 111 -d 5 --output-as=xml") test.add_cmd("attrd_updater", args="--name AAA -Q --output-as=xml", stdout_match='name="AAA" value="111"') test.add_cmd("attrd_updater", args="--name AAA -Q", stdout_match='name="AAA" host="[^"]+" value="111"', validate=False) test.add_log_pattern(r"Setting AAA\[.*\] in instance_attributes: \(unset\) -> 111 \| from .* with 5s write delay", regex=True) test = self.new_test("set_attr_4", "Update an attribute that does not exist with a delay") test.add_cmd("attrd_updater", args="--name BBB -U 999 -d 10 --output-as=xml") test.add_cmd("attrd_updater", args="--name BBB -Q --output-as=xml", stdout_match='name="BBB" value="999"') test.add_cmd("attrd_updater", args="--name BBB -Q", stdout_match='name="BBB" host="[^"]+" value="999"', validate=False) test.add_log_pattern(r"Setting BBB\[.*\] in instance_attributes: \(unset\) -> 999 \| from .* with 10s write delay", regex=True) test = self.new_test("update_attr_1", "Update an attribute that already exists") test.add_cmd("attrd_updater", args="--name BBB -U 222 --output-as=xml") test.add_cmd("attrd_updater", args="--name BBB -U 333 --output-as=xml") test.add_cmd("attrd_updater", args="--name BBB -Q --output-as=xml", stdout_match='name="BBB" value="333"') test.add_cmd("attrd_updater", args="--name BBB -Q", stdout_match='name="BBB" host="[^"]+" value="333"', validate=False) test.add_log_pattern(r"Setting BBB\[.*\] in instance_attributes: \(unset\) -> 222", regex=True) test.add_log_pattern(r"Setting BBB\[.*\] in instance_attributes: 222 -> 333", regex=True) test = self.new_test("update_attr_2", "Update an attribute using a delay other than its default") test.add_cmd("attrd_updater", args="--name BBB -U 777 -d 10 --output-as=xml") test.add_cmd("attrd_updater", args="--name BBB -U 888 -d 7 --output-as=xml") test.add_log_pattern(r"Setting BBB\[.*\] in instance_attributes: 777 -> 888 \| from .* with 10s write delay", regex=True) test = self.new_test("update_attr_delay_1", "Update the delay of an attribute that already exists") test.add_cmd("attrd_updater", args="--name BBB -U 222 --output-as=xml") test.add_cmd("attrd_updater", args="--name BBB -Y -d 5 --output-as=xml") test.add_log_pattern(r"Setting BBB\[.*\] in instance_attributes: \(unset\) -> 222", regex=True) test.add_log_pattern("Update attribute BBB delay to 5000ms (5)") test = self.new_test("update_attr_delay_2", "Update the delay and value of an attribute that already exists") test.add_cmd("attrd_updater", args="--name BBB -U 222 --output-as=xml") test.add_cmd("attrd_updater", args="--name BBB -B 333 -d 5 --output-as=xml") test.add_log_pattern(r"Setting BBB\[.*\] in instance_attributes: \(unset\) -> 222", regex=True) test.add_log_pattern("Update attribute BBB delay to 5000ms (5)") test.add_log_pattern(r"Setting BBB\[.*\] in instance_attributes: 222 -> 333", regex=True) test = self.new_test("missing_attr_1", "Query an attribute that does not exist") test.add_cmd("attrd_updater", args="--name NOSUCH --output-as=xml", expected_exitcode=ExitStatus.CONFIG) test = self.new_test("delete_attr_1", "Delete an existing attribute") test.add_cmd("attrd_updater", args="--name CCC -U 444 --output-as=xml") test.add_cmd("attrd_updater", args="--name CCC -D --output-as=xml") test.add_log_pattern(r"Setting CCC\[.*\] in instance_attributes: \(unset\) -> 444", regex=True) test.add_log_pattern(r"Setting CCC\[.*\] in instance_attributes: 444 -> \(unset\)", regex=True) test = self.new_test("missing_attr_2", "Delete an attribute that does not exist") test.add_cmd("attrd_updater", args="--name NOSUCH2 -D --output-as=xml") test = self.new_test("attr_in_set_1", "Set and query an attribute in a specific set") test.add_cmd("attrd_updater", args="--name DDD -U 555 --set=foo --output-as=xml") test.add_cmd("attrd_updater", args="--name DDD -Q --output-as=xml", stdout_match='name="DDD" value="555"') test.add_cmd("attrd_updater", args="--name DDD -Q", stdout_match='name="DDD" host="[^"]+" value="555"', validate=False) test.add_log_pattern("Processed 1 private change for DDD (set foo)") def build_multiple_query_tests(self): """Add tests that set and query an attribute across multiple nodes.""" # NOTE: These tests make use of the fact that nothing in attrd actually # cares about whether a node exists when you set or query an attribute. # It just keeps creating new hash tables for each node you ask it about. test = self.new_test("multi_query_1", "Query an attribute set across multiple nodes") test.add_cmd("attrd_updater", args="--name AAA -U 111 --node cluster1 --output-as=xml") test.add_cmd("attrd_updater", args="--name AAA -U 222 --node cluster2 --output-as=xml") test.add_cmd("attrd_updater", args="--name AAA -QA --output-as=xml", stdout_match=r'\n.*') test.add_cmd("attrd_updater", args="--name AAA -QA", stdout_match='name="AAA" host="cluster1" value="111"\nname="AAA" host="cluster2" value="222"', validate=False) test.add_cmd("attrd_updater", args="--name AAA -Q --node=cluster1 --output-as=xml", stdout_match='') test.add_cmd("attrd_updater", args="--name AAA -Q --node=cluster1", stdout_match='name="AAA" host="cluster1" value="111"', validate=False) test.add_cmd("attrd_updater", args="--name AAA -Q --node=cluster2 --output-as=xml", stdout_match='') test.add_cmd("attrd_updater", args="--name AAA -Q --node=cluster2", stdout_match='name="AAA" host="cluster2" value="222"', validate=False) test.add_cmd("attrd_updater", args="--name AAA -QA --output-as=xml", stdout_match=r'\n.*', env={"OCF_RESKEY_CRM_meta_on_node": "cluster1"}) test.add_cmd("attrd_updater", args="--name AAA -QA", stdout_match='name="AAA" host="cluster1" value="111"\nname="AAA" host="cluster2" value="222"', validate=False, env={"OCF_RESKEY_CRM_meta_on_node": "cluster1"}) test.add_cmd("attrd_updater", args="--name AAA -Q --output-as=xml", stdout_match='', env={"OCF_RESKEY_CRM_meta_on_node": "cluster1"}) test.add_cmd("attrd_updater", args="--name AAA -Q", stdout_match='name="AAA" host="cluster1" value="111"', validate=False, env={"OCF_RESKEY_CRM_meta_on_node": "cluster1"}) test.add_cmd("attrd_updater", args="--name AAA -Q --node=cluster2 --output-as=xml", stdout_match='', env={"OCF_RESKEY_CRM_meta_on_node": "cluster1"}) test.add_cmd("attrd_updater", args="--name AAA -Q --node=cluster2", stdout_match='name="AAA" host="cluster2" value="222"', validate=False, env={"OCF_RESKEY_CRM_meta_on_node": "cluster1"}) def build_regex_tests(self): """Add tests that use regexes.""" test = self.new_test("regex_update_1", "Update attributes using a regex") test.add_cmd("attrd_updater", args="--name AAA -U 111 --output-as=xml") test.add_cmd("attrd_updater", args="--name ABB -U 222 --output-as=xml") test.add_cmd("attrd_updater", args="-P 'A.*' -U 333 --output-as=xml") test.add_cmd("attrd_updater", args="--name AAA -Q --output-as=xml", stdout_match='name="AAA" value="333"') test.add_cmd("attrd_updater", args="--name ABB -Q --output-as=xml", stdout_match='name="ABB" value="333"') test.add_cmd("attrd_updater", args="--name AAA -Q", stdout_match='name="AAA" host="[^"]+" value="333"', validate=False) test.add_cmd("attrd_updater", args="--name ABB -Q", stdout_match='name="ABB" host="[^"]+" value="333"', validate=False) test.add_log_pattern(r"Setting AAA\[.*\] in instance_attributes: \(unset\) -> 111", regex=True) test.add_log_pattern(r"Setting ABB\[.*\] in instance_attributes: \(unset\) -> 222", regex=True) test.add_log_pattern(r"Setting ABB\[.*\] in instance_attributes: 222 -> 333", regex=True) test.add_log_pattern(r"Setting AAA\[.*\] in instance_attributes: 111 -> 333", regex=True) test = self.new_test("regex_delete_1", "Delete attributes using a regex") test.add_cmd("attrd_updater", args="--name XAX -U 444 --output-as=xml") test.add_cmd("attrd_updater", args="--name XBX -U 555 --output-as=xml") test.add_cmd("attrd_updater", args="-P 'X[A|B]X' -D --output-as=xml") test.add_log_pattern(r"Setting XAX\[.*\] in instance_attributes: \(unset\) -> 444", regex=True) test.add_log_pattern(r"Setting XBX\[.*\] in instance_attributes: \(unset\) -> 555", regex=True) test.add_log_pattern(r"Setting XBX\[.*\] in instance_attributes: 555 -> \(unset\)", regex=True) test.add_log_pattern(r"Setting XAX\[.*\] in instance_attributes: 444 -> \(unset\)", regex=True) def build_utilization_tests(self): """Add tests that involve utilization attributes.""" test = self.new_test("utilization_1", "Set and query a utilization attribute") test.add_cmd("attrd_updater", args="--name AAA -U ABC -z --output-as=xml") test.add_cmd("attrd_updater", args="--name AAA -Q --output-as=xml", stdout_match='name="AAA" value="ABC"') test.add_cmd("attrd_updater", args="--name AAA -Q", stdout_match='name="AAA" host="[^"]+" value="ABC"', validate=False) test.add_log_pattern(r"Setting AAA\[.*\] in utilization: \(unset\) -> ABC", regex=True) def build_sync_point_tests(self): """Add tests that involve sync points.""" test = self.new_test("local_sync_point", "Wait for a local sync point") test.add_cmd("attrd_updater", args="--name AAA -U 123 --wait=local --output-as=xml") test.add_cmd("attrd_updater", args="--name AAA -Q --output-as=xml", stdout_match='name="AAA" value="123"') test.add_cmd("attrd_updater", args="--name AAA -Q", stdout_match='name="AAA" host="[^"]+" value="123"', validate=False) test.add_log_pattern(r"Alerting client .* for reached local sync point", regex=True) test = self.new_test("cluster_sync_point", "Wait for a cluster-wide sync point") test.add_cmd("attrd_updater", args="--name BBB -U 456 --wait=cluster --output-as=xml") test.add_cmd("attrd_updater", args="--name BBB -Q --output-as=xml", stdout_match='name="BBB" value="456"') test.add_cmd("attrd_updater", args="--name BBB -Q", stdout_match='name="BBB" host="[^"]+" value="456"', validate=False) test.add_log_pattern(r"Alerting client .* for reached cluster sync point", regex=True) def build_options(): """Handle command line arguments.""" parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, description="Run pacemaker-attrd regression tests", epilog="Example: Run only the test 'start_stop'\n" "\t " + sys.argv[0] + " --run-only start_stop\n\n" "Example: Run only the tests with the string 'systemd' present in them\n" "\t " + sys.argv[0] + " --run-only-pattern systemd") parser.add_argument("-l", "--list-tests", action="store_true", help="Print out all registered tests") parser.add_argument("-p", "--run-only-pattern", metavar='PATTERN', help="Run only tests matching the given pattern") parser.add_argument("-r", "--run-only", metavar='TEST', help="Run a specific test") parser.add_argument("-V", "--verbose", action="store_true", help="Verbose output") args = parser.parse_args() return args def main(): """Run attrd regression tests as specified by arguments.""" update_path() # Ensure all command output is in portable locale for comparison os.environ['LC_ALL'] = "C" opts = build_options() exit_if_proc_running("pacemaker-attrd") # Create a temporary directory for log files (the directory and its # contents will automatically be erased when done) with tempfile.TemporaryDirectory(prefix="cts-attrd-") as logdir: tests = AttributeTests(verbose=opts.verbose, logdir=logdir) tests.build_basic_tests() tests.build_multiple_query_tests() tests.build_regex_tests() tests.build_utilization_tests() tests.build_sync_point_tests() if opts.list_tests: tests.print_list() sys.exit(ExitStatus.OK) print("Starting ...") try: tests.setup_environment(True) except TimeoutError: print("corosync did not start in time, exiting") sys.exit(ExitStatus.TIMEOUT) if opts.run_only_pattern: tests.run_tests_matching(opts.run_only_pattern) tests.print_results() elif opts.run_only: tests.run_single(opts.run_only) tests.print_results() else: tests.run_tests() tests.print_results() tests.cleanup_environment(True) tests.exit() if __name__ == "__main__": main() diff --git a/cts/cts-exec.in b/cts/cts-exec.in index bcf2dd72bd..dcc3c37f6d 100644 --- a/cts/cts-exec.in +++ b/cts/cts-exec.in @@ -1,929 +1,930 @@ #!@PYTHON@ """Regression tests for Pacemaker's pacemaker-execd.""" # pylint doesn't like the module name "cts-execd" which is an invalid complaint for this file # but probably something we want to continue warning about elsewhere # pylint: disable=invalid-name # pacemaker imports need to come after we modify sys.path, which pylint will complain about. # pylint: disable=wrong-import-position -__copyright__ = "Copyright 2012-2024 the Pacemaker project contributors" +__copyright__ = "Copyright 2012-2025 the Pacemaker project contributors" __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" import argparse import os import stat import sys import subprocess import shutil import tempfile # Where to find test binaries # Prefer the source tree if available TEST_DIR = sys.path[0] # These imports allow running from a source checkout after running `make`. # Note that while this doesn't necessarily mean it will successfully run tests, # but being able to see --help output can be useful. if os.path.exists("@abs_top_srcdir@/python"): sys.path.insert(0, "@abs_top_srcdir@/python") # pylint: disable=comparison-of-constants,comparison-with-itself,condition-evals-to-constant if os.path.exists("@abs_top_builddir@/python") and "@abs_top_builddir@" != "@abs_top_srcdir@": sys.path.insert(0, "@abs_top_builddir@/python") from pacemaker.buildoptions import BuildOptions from pacemaker.exitstatus import ExitStatus from pacemaker._cts.corosync import Corosync from pacemaker._cts.process import killall, exit_if_proc_running, stdout_from_command from pacemaker._cts.test import Test, Tests # File permissions for executable scripts we create EXECMODE = stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH def update_path(): # pylint: disable=protected-access """Set the PATH environment variable appropriately for the tests.""" new_path = os.environ['PATH'] - if os.path.exists("%s/cts-exec.in" % TEST_DIR): - print("Running tests from the source tree: %s (%s)" % (BuildOptions._BUILD_DIR, TEST_DIR)) + if os.path.exists(f"{TEST_DIR}/cts-exec.in"): + print(f"Running tests from the source tree: {BuildOptions._BUILD_DIR} ({TEST_DIR})") # For pacemaker-execd, cts-exec-helper, and pacemaker-remoted - new_path = "%s/daemons/execd:%s" % (BuildOptions._BUILD_DIR, new_path) - new_path = "%s/tools:%s" % (BuildOptions._BUILD_DIR, new_path) # For crm_resource + new_path = f"{BuildOptions._BUILD_DIR}/daemons/execd:{new_path}" + new_path = f"{BuildOptions._BUILD_DIR}/tools:{new_path}" # For crm_resource # For pacemaker-fenced - new_path = "%s/daemons/fenced:%s" % (BuildOptions._BUILD_DIR, new_path) + new_path = f"{BuildOptions._BUILD_DIR}/daemons/fenced:{new_path}" # For cts-support - new_path = "%s/cts/support:%s" % (BuildOptions._BUILD_DIR, new_path) + new_path = f"{BuildOptions._BUILD_DIR}/cts/support:{new_path}" else: - print("Running tests from the install tree: %s (not %s)" % (BuildOptions.DAEMON_DIR, TEST_DIR)) + print(f"Running tests from the install tree: {BuildOptions.DAEMON_DIR} (not {TEST_DIR})") # For cts-exec-helper, cts-support, pacemaker-execd, pacemaker-fenced, # and pacemaker-remoted - new_path = "%s:%s" % (BuildOptions.DAEMON_DIR, new_path) + new_path = f"{BuildOptions.DAEMON_DIR}:{new_path}" - print('Using PATH="%s"' % new_path) + print(f'Using PATH="{new_path}"') os.environ['PATH'] = new_path class ExecTest(Test): """Executor for a single pacemaker-execd regression test.""" def __init__(self, name, description, **kwargs): """Create a new ExecTest instance. Arguments: name -- A unique name for this test. This can be used on the command line to specify that only a specific test should be executed. description -- A meaningful description for the test. Keyword arguments: tls -- Enable pacemaker-remoted. """ Test.__init__(self, name, description, **kwargs) self.tls = kwargs.get("tls", False) # If we are going to run the stonith resource tests, we will need to # launch and track Corosync and pacemaker-fenced. self._corosync = None self._fencer = None self._is_stonith_test = "stonith" in self.name if self.tls: self._daemon_location = "pacemaker-remoted" else: self._daemon_location = "pacemaker-execd" if self._is_stonith_test: self._corosync = Corosync(self.verbose, self.logdir, "cts-exec") self._test_tool_location = "cts-exec-helper" def _kill_daemons(self): killall([ "corosync", "pacemaker-fenced", "lt-pacemaker-fenced", "pacemaker-execd", "lt-pacemaker-execd", "cts-exec-helper", "lt-cts-exec-helper", "pacemaker-remoted", ]) def _start_daemons(self): if self._corosync: self._corosync.start(kill_first=True) # pylint: disable=consider-using-with self._fencer = subprocess.Popen(["pacemaker-fenced", "-s"]) cmd = [self._daemon_location, "-l", self.logpath] if self.verbose: cmd += ["-V"] # pylint: disable=consider-using-with self._daemon_process = subprocess.Popen(cmd) def clean_environment(self): """Clean up the host after running a test.""" if self._daemon_process: self._daemon_process.terminate() self._daemon_process.wait() if self.verbose: print("Daemon Output Start") with open(self.logpath, "rt", errors="replace", encoding="utf-8") as logfile: for line in logfile: print(line.strip()) print("Daemon Output End") if self._corosync: self._fencer.terminate() self._fencer.wait() self._corosync.stop() self._daemon_process = None self._fencer = None self._corosync = None def add_cmd(self, cmd=None, **kwargs): """Add a cts-exec-helper command to be executed as part of this test.""" if cmd is None: cmd = self._test_tool_location if cmd == self._test_tool_location: if self.verbose: kwargs["args"] += " -V " if self.tls: kwargs["args"] += " -S " kwargs["validate"] = False kwargs["check_rng"] = False kwargs["check_stderr"] = False Test.add_cmd(self, cmd, **kwargs) def run(self): """Execute this test.""" if self.tls and self._is_stonith_test: - self._result_txt = "SKIPPED - '%s' - disabled when testing pacemaker_remote" % (self.name) + self._result_txt = f"SKIPPED - '{self.name}' - disabled when testing pacemaker_remote" print(self._result_txt) return Test.run(self) class ExecTests(Tests): """Collection of all pacemaker-execd regression tests.""" def __init__(self, **kwargs): """ Create a new ExecTests instance. Keyword arguments: tls -- Enable pacemaker-remoted. """ Tests.__init__(self, **kwargs) self.tls = kwargs.get("tls", False) self._action_timeout = " -t 9000 " self._installed_files = [] self._rsc_classes = self._setup_rsc_classes() - print("Testing resource classes %r" % self._rsc_classes) + print(f"Testing resource classes {self._rsc_classes!r}") if "lsb" in self._rsc_classes: service_agent = "LSBDummy" elif "systemd" in self._rsc_classes: service_agent = "pacemaker-cts-dummyd@3" else: service_agent = "unsupported" self._common_cmds = { "ocf_reg_line": '-c register_rsc -r ocf_test_rsc ' + self._action_timeout + ' -C ocf -P pacemaker -T Dummy', "ocf_reg_event": '-l "NEW_EVENT event_type:register rsc_id:ocf_test_rsc action:none rc:ok op_status:Done"', "ocf_unreg_line": '-c unregister_rsc -r ocf_test_rsc ' + self._action_timeout, "ocf_unreg_event": '-l "NEW_EVENT event_type:unregister rsc_id:ocf_test_rsc action:none rc:ok op_status:Done"', "ocf_start_line": '-c exec -r ocf_test_rsc -a start ' + self._action_timeout, "ocf_start_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:start rc:ok op_status:Done" ', "ocf_stop_line": '-c exec -r ocf_test_rsc -a stop ' + self._action_timeout, "ocf_stop_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:stop rc:ok op_status:Done" ', "ocf_monitor_line": '-c exec -r ocf_test_rsc -a monitor -i 2s ' + self._action_timeout, "ocf_monitor_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:monitor rc:ok op_status:Done" ' + self._action_timeout, "ocf_cancel_line": '-c cancel -r ocf_test_rsc -a monitor -i 2s ' + self._action_timeout, "ocf_cancel_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:monitor rc:ok op_status:Cancelled" ', "systemd_reg_line": '-c register_rsc -r systemd_test_rsc ' + self._action_timeout + ' -C systemd -T pacemaker-cts-dummyd@3', "systemd_reg_event": '-l "NEW_EVENT event_type:register rsc_id:systemd_test_rsc action:none rc:ok op_status:Done"', "systemd_unreg_line": '-c unregister_rsc -r systemd_test_rsc ' + self._action_timeout, "systemd_unreg_event": '-l "NEW_EVENT event_type:unregister rsc_id:systemd_test_rsc action:none rc:ok op_status:Done"', "systemd_start_line": '-c exec -r systemd_test_rsc -a start ' + self._action_timeout, "systemd_start_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:start rc:ok op_status:Done" ', "systemd_stop_line": '-c exec -r systemd_test_rsc -a stop ' + self._action_timeout, "systemd_stop_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:stop rc:ok op_status:Done" ', "systemd_monitor_line": '-c exec -r systemd_test_rsc -a monitor -i 2s ' + self._action_timeout, "systemd_monitor_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:monitor rc:ok op_status:Done" -t 15000 ', "systemd_cancel_line": '-c cancel -r systemd_test_rsc -a monitor -i 2s ' + self._action_timeout, "systemd_cancel_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:monitor rc:ok op_status:Cancelled" ', - "service_reg_line": '-c register_rsc -r service_test_rsc ' + self._action_timeout - + ' -C service -T %s' % service_agent, + "service_reg_line": f"-c register_rsc -r service_test_rsc {self._action_timeout} -C service -T {service_agent}", "service_reg_event": '-l "NEW_EVENT event_type:register rsc_id:service_test_rsc action:none rc:ok op_status:Done"', "service_unreg_line": '-c unregister_rsc -r service_test_rsc ' + self._action_timeout, "service_unreg_event": '-l "NEW_EVENT event_type:unregister rsc_id:service_test_rsc action:none rc:ok op_status:Done"', "service_start_line": '-c exec -r service_test_rsc -a start ' + self._action_timeout, "service_start_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:start rc:ok op_status:Done" ', "service_stop_line": '-c exec -r service_test_rsc -a stop ' + self._action_timeout, "service_stop_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:stop rc:ok op_status:Done" ', "service_monitor_line": '-c exec -r service_test_rsc -a monitor -i 2s ' + self._action_timeout, "service_monitor_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:monitor rc:ok op_status:Done" ' + self._action_timeout, "service_cancel_line": '-c cancel -r service_test_rsc -a monitor -i 2s ' + self._action_timeout, "service_cancel_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:monitor rc:ok op_status:Cancelled" ', "lsb_reg_line": '-c register_rsc -r lsb_test_rsc ' + self._action_timeout + ' -C lsb -T LSBDummy', "lsb_reg_event": '-l "NEW_EVENT event_type:register rsc_id:lsb_test_rsc action:none rc:ok op_status:Done" ', "lsb_unreg_line": '-c unregister_rsc -r lsb_test_rsc ' + self._action_timeout, "lsb_unreg_event": '-l "NEW_EVENT event_type:unregister rsc_id:lsb_test_rsc action:none rc:ok op_status:Done"', "lsb_start_line": '-c exec -r lsb_test_rsc -a start ' + self._action_timeout, "lsb_start_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:start rc:ok op_status:Done" ', "lsb_stop_line": '-c exec -r lsb_test_rsc -a stop ' + self._action_timeout, "lsb_stop_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:stop rc:ok op_status:Done" ', "lsb_monitor_line": '-c exec -r lsb_test_rsc -a status -i 2s ' + self._action_timeout, "lsb_monitor_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:status rc:ok op_status:Done" ' + self._action_timeout, "lsb_cancel_line": '-c cancel -r lsb_test_rsc -a status -i 2s ' + self._action_timeout, "lsb_cancel_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:status rc:ok op_status:Cancelled" ', "stonith_reg_line": '-c register_rsc -r stonith_test_rsc ' + self._action_timeout + ' -C stonith -P pacemaker -T fence_dummy', "stonith_reg_event": '-l "NEW_EVENT event_type:register rsc_id:stonith_test_rsc action:none rc:ok op_status:Done" ', "stonith_unreg_line": '-c unregister_rsc -r stonith_test_rsc ' + self._action_timeout, "stonith_unreg_event": '-l "NEW_EVENT event_type:unregister rsc_id:stonith_test_rsc action:none rc:ok op_status:Done"', "stonith_start_line": '-c exec -r stonith_test_rsc -a start ' + self._action_timeout, "stonith_start_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:start rc:ok op_status:Done" ', "stonith_stop_line": '-c exec -r stonith_test_rsc -a stop ' + self._action_timeout, "stonith_stop_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:stop rc:ok op_status:Done" ', "stonith_monitor_line": '-c exec -r stonith_test_rsc -a monitor -i 2s ' + self._action_timeout, "stonith_monitor_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:ok op_status:Done" ' + self._action_timeout, "stonith_cancel_line": '-c cancel -r stonith_test_rsc -a monitor -i 2s ' + self._action_timeout, "stonith_cancel_event": '-l "NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:ok op_status:Cancelled" ', } def _setup_rsc_classes(self): """Determine which resource classes are supported.""" classes = stdout_from_command(["crm_resource", "--list-standards"]) # Strip trailing empty line classes = classes[:-1] if self.tls: classes.remove("stonith") if "systemd" in classes: try: # This code doesn't need this import, but pacemaker-cts-dummyd # does, so ensure the dependency is available rather than cause # all systemd tests to fail. # pylint: disable=import-outside-toplevel,unused-import import systemd.daemon except ImportError: print("Python systemd bindings not found.") print("The tests for systemd class are not going to be run.") classes.remove("systemd") return classes def new_test(self, name, description): """Create a named test.""" test = ExecTest(name, description, verbose=self.verbose, tls=self.tls, timeout=self.timeout, force_wait=self.force_wait, logdir=self.logdir) self._tests.append(test) return test def setup_environment(self): """Prepare the host before executing any tests.""" if BuildOptions.REMOTE_ENABLED: # @TODO Use systemctl when available, and use the subprocess module # with an argument array instead of os.system() os.system("service pacemaker_remote stop") self.cleanup_environment() # @TODO Support the option of using specified existing certificates - authkey = "%s/authkey" % BuildOptions.PACEMAKER_CONFIG_DIR + authkey = f"{BuildOptions.PACEMAKER_CONFIG_DIR}/authkey" if self.tls and not os.path.isfile(authkey): - print("Installing %s ..." % authkey) + print(f"Installing {authkey} ...") # @TODO Use os.mkdir() instead - os.system("mkdir -p %s" % BuildOptions.PACEMAKER_CONFIG_DIR) + os.system(f"mkdir -p {BuildOptions.PACEMAKER_CONFIG_DIR}") # @TODO Use the subprocess module with an argument array instead - os.system("dd if=/dev/urandom of=%s bs=4096 count=1" % authkey) + os.system(f"dd if=/dev/urandom of={authkey} bs=4096 count=1") self._installed_files.append(authkey) # If we're in build directory, install agents if not already installed # pylint: disable=protected-access - if os.path.exists("%s/cts/cts-exec.in" % BuildOptions._BUILD_DIR): + if os.path.exists(f"{BuildOptions._BUILD_DIR}/cts/cts-exec.in"): - if not os.path.exists("%s/pacemaker" % BuildOptions.OCF_RA_INSTALL_DIR): + if not os.path.exists(f"{BuildOptions.OCF_RA_INSTALL_DIR}/pacemaker"): # @TODO remember which components were created and remove them - os.makedirs("%s/pacemaker" % BuildOptions.OCF_RA_INSTALL_DIR, 0o755) + os.makedirs(f"{BuildOptions.OCF_RA_INSTALL_DIR}/pacemaker", 0o755) for agent in ["Dummy", "Stateful", "ping"]: - agent_source = "%s/extra/resources/%s" % (BuildOptions._BUILD_DIR, agent) - agent_dest = "%s/pacemaker/%s" % (BuildOptions.OCF_RA_INSTALL_DIR, agent) + agent_source = f"{BuildOptions._BUILD_DIR}/extra/resources/{agent}" + agent_dest = f"{BuildOptions.OCF_RA_INSTALL_DIR}/pacemaker/{agent}" if not os.path.exists(agent_dest): - print("Installing %s ..." % agent_dest) + print(f"Installing {agent_dest} ...") shutil.copyfile(agent_source, agent_dest) os.chmod(agent_dest, EXECMODE) self._installed_files.append(agent_dest) subprocess.call(["cts-support", "install"]) def cleanup_environment(self): """Clean up the host after executing desired tests.""" for installed_file in self._installed_files: - print("Removing %s ..." % installed_file) + print(f"Removing {installed_file} ...") os.remove(installed_file) subprocess.call(["cts-support", "uninstall"]) def _build_cmd_str(self, rsc, ty): """Construct a command string for the given resource and type.""" - return "%s %s" % (self._common_cmds["%s_%s_line" % (rsc, ty)], self._common_cmds["%s_%s_event" % (rsc, ty)]) + return f"{self._common_cmds[f'{rsc}_{ty}_line']} {self._common_cmds[f'{rsc}_{ty}_event']}" def build_generic_tests(self): """Register tests that apply to all resource classes.""" common_cmds = self._common_cmds # register/unregister tests for rsc in self._rsc_classes: - test = self.new_test("generic_registration_%s" % rsc, - "Simple resource registration test for %s standard" % rsc) + test = self.new_test(f"generic_registration_{rsc}", + f"Simple resource registration test for {rsc} standard") test.add_cmd(args=self._build_cmd_str(rsc, "reg")) test.add_cmd(args=self._build_cmd_str(rsc, "unreg")) # start/stop tests for rsc in self._rsc_classes: - test = self.new_test("generic_start_stop_%s" % rsc, "Simple start and stop test for %s standard" % rsc) + test = self.new_test(f"generic_start_stop_{rsc}", + f"Simple start and stop test for {rsc} standard") test.add_cmd(args=self._build_cmd_str(rsc, "reg")) test.add_cmd(args=self._build_cmd_str(rsc, "start")) test.add_cmd(args=self._build_cmd_str(rsc, "stop")) test.add_cmd(args=self._build_cmd_str(rsc, "unreg")) # monitor cancel test for rsc in self._rsc_classes: - test = self.new_test("generic_monitor_cancel_%s" % rsc, - "Simple monitor cancel test for %s standard" % rsc) + test = self.new_test(f"generic_monitor_cancel_{rsc}", + f"Simple monitor cancel test for {rsc} standard") test.add_cmd(args=self._build_cmd_str(rsc, "reg")) test.add_cmd(args=self._build_cmd_str(rsc, "start")) test.add_cmd(args=self._build_cmd_str(rsc, "monitor")) # If this fails, that means the monitor may not be getting rescheduled - test.add_cmd(args=common_cmds["%s_monitor_event" % rsc]) + test.add_cmd(args=common_cmds[f"{rsc}_monitor_event"]) # If this fails, that means the monitor may not be getting rescheduled - test.add_cmd(args=common_cmds["%s_monitor_event" % rsc]) + test.add_cmd(args=common_cmds[f"{rsc}_monitor_event"]) test.add_cmd(args=self._build_cmd_str(rsc, "cancel")) # If this happens the monitor did not actually cancel correctly - test.add_cmd(args=common_cmds["%s_monitor_event" % rsc], + test.add_cmd(args=common_cmds[f"{rsc}_monitor_event"], expected_exitcode=ExitStatus.TIMEOUT) # If this happens the monitor did not actually cancel correctly - test.add_cmd(args=common_cmds["%s_monitor_event" % rsc], + test.add_cmd(args=common_cmds[f"{rsc}_monitor_event"], expected_exitcode=ExitStatus.TIMEOUT) test.add_cmd(args=self._build_cmd_str(rsc, "stop")) test.add_cmd(args=self._build_cmd_str(rsc, "unreg")) # monitor duplicate test for rsc in self._rsc_classes: - test = self.new_test("generic_monitor_duplicate_%s" % rsc, - "Test creation and canceling of duplicate monitors for %s standard" % rsc) + test = self.new_test(f"generic_monitor_duplicate_{rsc}", + f"Test creation and canceling of duplicate monitors for {rsc} standard") test.add_cmd(args=self._build_cmd_str(rsc, "reg")) test.add_cmd(args=self._build_cmd_str(rsc, "start")) test.add_cmd(args=self._build_cmd_str(rsc, "monitor")) # If this fails, that means the monitor may not be getting rescheduled - test.add_cmd(args=common_cmds["%s_monitor_event" % rsc]) + test.add_cmd(args=common_cmds[f"{rsc}_monitor_event"]) # If this fails, that means the monitor may not be getting rescheduled - test.add_cmd(args=common_cmds["%s_monitor_event" % rsc]) + test.add_cmd(args=common_cmds[f"{rsc}_monitor_event"]) # Add the duplicate monitors test.add_cmd(args=self._build_cmd_str(rsc, "monitor")) test.add_cmd(args=self._build_cmd_str(rsc, "monitor")) test.add_cmd(args=self._build_cmd_str(rsc, "monitor")) test.add_cmd(args=self._build_cmd_str(rsc, "monitor")) # verify we still get update events # If this fails, that means the monitor may not be getting rescheduled - test.add_cmd(args=common_cmds["%s_monitor_event" % rsc]) + test.add_cmd(args=common_cmds[f"{rsc}_monitor_event"]) # cancel the monitor, if the duplicate merged with the original, we should no longer see monitor updates test.add_cmd(args=self._build_cmd_str(rsc, "cancel")) # If this happens the monitor did not actually cancel correctly - test.add_cmd(args=common_cmds["%s_monitor_event" % rsc], + test.add_cmd(args=common_cmds[f"{rsc}_monitor_event"], expected_exitcode=ExitStatus.TIMEOUT) # If this happens the monitor did not actually cancel correctly - test.add_cmd(args=common_cmds["%s_monitor_event" % rsc], + test.add_cmd(args=common_cmds[f"{rsc}_monitor_event"], expected_exitcode=ExitStatus.TIMEOUT) test.add_cmd(args=self._build_cmd_str(rsc, "stop")) test.add_cmd(args=self._build_cmd_str(rsc, "unreg")) # stop implies cancel test for rsc in self._rsc_classes: - test = self.new_test("generic_stop_implies_cancel_%s" % rsc, - "Verify stopping a resource implies cancel of recurring ops for %s standard" % rsc) + test = self.new_test(f"generic_stop_implies_cancel_{rsc}", + f"Verify stopping a resource implies cancel of recurring ops for {rsc} standard") test.add_cmd(args=self._build_cmd_str(rsc, "reg")) test.add_cmd(args=self._build_cmd_str(rsc, "start")) test.add_cmd(args=self._build_cmd_str(rsc, "monitor")) # If this fails, that means the monitor may not be getting rescheduled - test.add_cmd(args=common_cmds["%s_monitor_event" % rsc]) + test.add_cmd(args=common_cmds[f"{rsc}_monitor_event"]) # If this fails, that means the monitor may not be getting rescheduled - test.add_cmd(args=common_cmds["%s_monitor_event" % rsc]) + test.add_cmd(args=common_cmds[f"{rsc}_monitor_event"]) test.add_cmd(args=self._build_cmd_str(rsc, "stop")) # If this happens the monitor did not actually cancel correctly - test.add_cmd(args=common_cmds["%s_monitor_event" % rsc], + test.add_cmd(args=common_cmds[f"{rsc}_monitor_event"], expected_exitcode=ExitStatus.TIMEOUT) # If this happens the monitor did not actually cancel correctly - test.add_cmd(args=common_cmds["%s_monitor_event" % rsc], + test.add_cmd(args=common_cmds[f"{rsc}_monitor_event"], expected_exitcode=ExitStatus.TIMEOUT) test.add_cmd(args=self._build_cmd_str(rsc, "unreg")) def build_multi_rsc_tests(self): """Register complex tests that involve managing multiple resouces of different types.""" common_cmds = self._common_cmds # do not use service and systemd at the same time, it is the same resource. # register start monitor stop unregister resources of each type at the same time test = self.new_test("multi_rsc_start_stop_all_including_stonith", "Start, monitor, and stop resources of multiple types and classes") for rsc in self._rsc_classes: test.add_cmd(args=self._build_cmd_str(rsc, "reg")) for rsc in self._rsc_classes: test.add_cmd(args=self._build_cmd_str(rsc, "start")) for rsc in self._rsc_classes: test.add_cmd(args=self._build_cmd_str(rsc, "monitor")) for rsc in self._rsc_classes: # If this fails, that means the monitor is not being rescheduled - test.add_cmd(args=common_cmds["%s_monitor_event" % rsc]) + test.add_cmd(args=common_cmds[f"{rsc}_monitor_event"]) for rsc in self._rsc_classes: test.add_cmd(args=self._build_cmd_str(rsc, "cancel")) for rsc in self._rsc_classes: test.add_cmd(args=self._build_cmd_str(rsc, "stop")) for rsc in self._rsc_classes: test.add_cmd(args=self._build_cmd_str(rsc, "unreg")) def build_negative_tests(self): """Register tests related to how pacemaker-execd handles failures.""" # ocf start timeout test test = self.new_test("ocf_start_timeout", "Force start timeout to occur, verify start failure.") test.add_cmd(args='-c register_rsc -r test_rsc -C ocf -P pacemaker -T Dummy ' + self._action_timeout + '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done" ') # -t must be less than self._action_timeout test.add_cmd(args='-c exec -r test_rsc -a start -k op_sleep -v 5 -t 1000 -w') test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:Error occurred op_status:Timed out" ' + self._action_timeout) test.add_cmd(args='-c exec -r test_rsc -a stop ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:Done" ') test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done" ') # stonith start timeout test test = self.new_test("stonith_start_timeout", "Force start timeout to occur, verify start failure.") test.add_cmd(args='-c register_rsc -r test_rsc -C stonith -P pacemaker -T fence_dummy ' + self._action_timeout + '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done"') # -t must be less than self._action_timeout test.add_cmd(args='-c exec -r test_rsc -a start -k monitor_delay -v 30 -t 1000 -w') test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:Error occurred op_status:Timed out" ' + self._action_timeout) test.add_cmd(args='-c exec -r test_rsc -a stop ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:Done" ') test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done" ') # stonith component fail test = self.new_test("stonith_component_fail", "Kill stonith component after pacemaker-execd connects") test.add_cmd(args=self._build_cmd_str("stonith", "reg")) test.add_cmd(args=self._build_cmd_str("stonith", "start")) test.add_cmd(args='-c exec -r stonith_test_rsc -a monitor -i 600s ' '-l "NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:ok op_status:Done" ' + self._action_timeout) test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:Error occurred op_status:error" -t 15000', kill="killall -9 -q pacemaker-fenced lt-pacemaker-fenced") test.add_cmd(args=self._build_cmd_str("stonith", "unreg")) # monitor fail for ocf resources test = self.new_test("monitor_fail_ocf", "Force ocf monitor to fail, verify failure is reported.") test.add_cmd(args='-c register_rsc -r test_rsc -C ocf -P pacemaker -T Dummy ' + self._action_timeout + '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a start ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a start ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a monitor -i 1s ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done"') test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done"' + self._action_timeout) test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done"' + self._action_timeout) test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Done" ' + self._action_timeout, - kill="rm -f %s/run/Dummy-test_rsc.state" % BuildOptions.LOCAL_STATE_DIR) + kill=f"rm -f {BuildOptions.LOCAL_STATE_DIR}/run/Dummy-test_rsc.state") test.add_cmd(args='-c cancel -r test_rsc -a monitor -i 1s ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled" ') test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Done" ' + self._action_timeout, expected_exitcode=ExitStatus.TIMEOUT) test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done" ' + self._action_timeout, expected_exitcode=ExitStatus.TIMEOUT) test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done" ') # verify notify changes only for monitor operation test = self.new_test("monitor_changes_only", "Verify when flag is set, only monitor changes are notified.") test.add_cmd(args='-c register_rsc -r test_rsc -C ocf -P pacemaker -T Dummy ' + self._action_timeout + '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a start ' + self._action_timeout + ' -o ' '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a monitor -i 1s ' + self._action_timeout + ' -o -l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done" ') test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done" ' + self._action_timeout, expected_exitcode=ExitStatus.TIMEOUT) test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Done"' + self._action_timeout, - kill='rm -f %s/run/Dummy-test_rsc.state' % BuildOptions.LOCAL_STATE_DIR) + kill=f"rm -f {BuildOptions.LOCAL_STATE_DIR}/run/Dummy-test_rsc.state") test.add_cmd(args='-c cancel -r test_rsc -a monitor -i 1s' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled" ') test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Done" ' + self._action_timeout, expected_exitcode=ExitStatus.TIMEOUT) test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done" ' + self._action_timeout, expected_exitcode=ExitStatus.TIMEOUT) test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done"') # monitor fail for systemd resource if "systemd" in self._rsc_classes: test = self.new_test("monitor_fail_systemd", "Force systemd monitor to fail, verify failure is reported..") test.add_cmd(args='-c register_rsc -r test_rsc -C systemd -T pacemaker-cts-dummyd@3 ' + self._action_timeout + '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a start ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a start ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a monitor -i 1s ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done" ') test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done" ' + self._action_timeout) test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done" ' + self._action_timeout) test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Done"' + self._action_timeout, kill="pkill -9 -f pacemaker-cts-dummyd") test.add_cmd(args='-c cancel -r test_rsc -a monitor -i 1s' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled" ') test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Done" ' + self._action_timeout, expected_exitcode=ExitStatus.TIMEOUT) test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done" ' + self._action_timeout, expected_exitcode=ExitStatus.TIMEOUT) test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done" ') # Cancel non-existent operation on a resource test = self.new_test("cancel_non_existent_op", "Attempt to cancel the wrong monitor operation, verify expected failure") test.add_cmd(args='-c register_rsc -r test_rsc -C ocf -P pacemaker -T Dummy ' + self._action_timeout + '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a start ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a start ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a monitor -i 1s ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done" ') test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done" ' + self._action_timeout) # interval is wrong, should fail test.add_cmd(args='-c cancel -r test_rsc -a monitor -i 2s' + self._action_timeout + ' -l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled" ', expected_exitcode=ExitStatus.ERROR) # action name is wrong, should fail test.add_cmd(args='-c cancel -r test_rsc -a stop -i 1s' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled" ', expected_exitcode=ExitStatus.ERROR) test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done" ') # Attempt to invoke non-existent rsc id test = self.new_test("invoke_non_existent_rsc", "Attempt to perform operations on a non-existent rsc id.") test.add_cmd(args='-c exec -r test_rsc -a start ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:error op_status:Done" ', expected_exitcode=ExitStatus.ERROR) test.add_cmd(args='-c exec -r test_rsc -a stop ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:Done" ', expected_exitcode=ExitStatus.ERROR) test.add_cmd(args='-c exec -r test_rsc -a monitor -i 6s ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done" ', expected_exitcode=ExitStatus.ERROR) test.add_cmd(args='-c cancel -r test_rsc -a start ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Cancelled" ', expected_exitcode=ExitStatus.ERROR) test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done" ') # Register and start a resource that doesn't exist, systemd if "systemd" in self._rsc_classes: test = self.new_test("start_uninstalled_systemd", "Register uninstalled systemd agent, try to start, verify expected failure") test.add_cmd(args='-c register_rsc -r test_rsc -C systemd -T this_is_fake1234 ' + self._action_timeout + '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a start ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed" ') test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done" ') # Register and start a resource that doesn't exist, ocf test = self.new_test("start_uninstalled_ocf", "Register uninstalled ocf agent, try to start, verify expected failure.") test.add_cmd(args='-c register_rsc -r test_rsc -C ocf -P pacemaker -T this_is_fake1234 ' + self._action_timeout + '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a start ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed" ') test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done" ') # Register ocf with non-existent provider test = self.new_test("start_ocf_bad_provider", "Register ocf agent with a non-existent provider, verify expected failure.") test.add_cmd(args='-c register_rsc -r test_rsc -C ocf -P pancakes -T Dummy ' + self._action_timeout + '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a start ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed" ') test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done" ') # Register ocf with empty provider field test = self.new_test("start_ocf_no_provider", "Register ocf agent with a no provider, verify expected failure.") test.add_cmd(args='-c register_rsc -r test_rsc -C ocf -T Dummy ' + self._action_timeout + '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done" ', expected_exitcode=ExitStatus.ERROR) test.add_cmd(args='-c exec -r test_rsc -a start ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Error" ', expected_exitcode=ExitStatus.ERROR) test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done" ') def build_stress_tests(self): """Register stress tests.""" timeout = "-t 20000" iterations = 25 test = self.new_test("ocf_stress", "Verify OCF agent handling works under load") for i in range(iterations): - test.add_cmd(args='-c register_rsc -r rsc_%s %s -C ocf -P heartbeat -T Dummy -l "NEW_EVENT event_type:register rsc_id:rsc_%s action:none rc:ok op_status:Done"' % (i, timeout, i)) - test.add_cmd(args='-c exec -r rsc_%s -a start %s -l "NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:start rc:ok op_status:Done"' % (i, timeout, i)) - test.add_cmd(args='-c exec -r rsc_%s -a monitor %s -i 1s ' - '-l "NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:monitor rc:ok op_status:Done"' % (i, timeout, i)) + test.add_cmd(args=f'-c register_rsc -r rsc_{i} {timeout} -C ocf -P heartbeat -T Dummy -l "NEW_EVENT event_type:register rsc_id:rsc_{i} action:none rc:ok op_status:Done"') + test.add_cmd(args=f'-c exec -r rsc_{i} -a start {timeout} -l "NEW_EVENT event_type:exec_complete rsc_id:rsc_{i} action:start rc:ok op_status:Done"') + test.add_cmd(args=f'-c exec -r rsc_{i} -a monitor {timeout} -i 1s ' + f'-l "NEW_EVENT event_type:exec_complete rsc_id:rsc_{i} action:monitor rc:ok op_status:Done"') + for i in range(iterations): - test.add_cmd(args='-c exec -r rsc_%s -a stop %s -l "NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:stop rc:ok op_status:Done"' % (i, timeout, i)) - test.add_cmd(args='-c unregister_rsc -r rsc_%s %s -l "NEW_EVENT event_type:unregister rsc_id:rsc_%s action:none rc:ok op_status:Done"' % (i, timeout, i)) + test.add_cmd(args=f'-c exec -r rsc_{i} -a stop {timeout} -l "NEW_EVENT event_type:exec_complete rsc_id:rsc_{i} action:stop rc:ok op_status:Done"') + test.add_cmd(args=f'-c unregister_rsc -r rsc_{i} {timeout} -l "NEW_EVENT event_type:unregister rsc_id:rsc_{i} action:none rc:ok op_status:Done"') if "systemd" in self._rsc_classes: test = self.new_test("systemd_stress", "Verify systemd dbus connection works under load") for i in range(iterations): - test.add_cmd(args='-c register_rsc -r rsc_%s %s -C systemd -T pacemaker-cts-dummyd@3 -l "NEW_EVENT event_type:register rsc_id:rsc_%s action:none rc:ok op_status:Done"' % (i, timeout, i)) - test.add_cmd(args='-c exec -r rsc_%s -a start %s -l "NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:start rc:ok op_status:Done"' % (i, timeout, i)) - test.add_cmd(args='-c exec -r rsc_%s -a monitor %s -i 1s ' - '-l "NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:monitor rc:ok op_status:Done"' % (i, timeout, i)) + test.add_cmd(args=f'-c register_rsc -r rsc_{i} {timeout} -C systemd -T pacemaker-cts-dummyd@3 -l "NEW_EVENT event_type:register rsc_id:rsc_{i} action:none rc:ok op_status:Done"') + test.add_cmd(args=f'-c exec -r rsc_{i} -a start {timeout} -l "NEW_EVENT event_type:exec_complete rsc_id:rsc_{i} action:start rc:ok op_status:Done"') + test.add_cmd(args=f'-c exec -r rsc_{i} -a monitor {timeout} -i 1s ' + f'-l "NEW_EVENT event_type:exec_complete rsc_id:rsc_{i} action:monitor rc:ok op_status:Done"') for i in range(iterations): - test.add_cmd(args='-c exec -r rsc_%s -a stop %s -l "NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:stop rc:ok op_status:Done"' % (i, timeout, i)) - test.add_cmd(args='-c unregister_rsc -r rsc_%s %s -l "NEW_EVENT event_type:unregister rsc_id:rsc_%s action:none rc:ok op_status:Done"' % (i, timeout, i)) + test.add_cmd(args=f'-c exec -r rsc_{i} -a stop {timeout} -l "NEW_EVENT event_type:exec_complete rsc_id:rsc_{i} action:stop rc:ok op_status:Done"') + test.add_cmd(args=f'-c unregister_rsc -r rsc_{i} {timeout} -l "NEW_EVENT event_type:unregister rsc_id:rsc_{i} action:none rc:ok op_status:Done"') iterations = 9 timeout = "-t 30000" # Verify recurring op in-flight collision is handled in series properly test = self.new_test("rsc_inflight_collision", "Verify recurring ops do not collide with other operations for the same rsc.") test.add_cmd(args='-c register_rsc -r test_rsc -P pacemaker -C ocf -T Dummy ' '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done" ' + self._action_timeout) - test.add_cmd(args='-c exec -r test_rsc -a start %s -k op_sleep -v 1 -l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Done"' % timeout) + test.add_cmd(args=f'-c exec -r test_rsc -a start {timeout} -k op_sleep -v 1 -l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Done"') for i in range(iterations): - test.add_cmd(args='-c exec -r test_rsc -a monitor %s -i 100%dms -k op_sleep -v 2 ' - '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done"' % (timeout, i)) + test.add_cmd(args=f'-c exec -r test_rsc -a monitor {timeout} -i 100{i}ms -k op_sleep -v 2 ' + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done"') - test.add_cmd(args='-c exec -r test_rsc -a stop %s -l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:Done"' % timeout) - test.add_cmd(args='-c unregister_rsc -r test_rsc %s -l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done"' % timeout) + test.add_cmd(args=f'-c exec -r test_rsc -a stop {timeout} -l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:Done"') + test.add_cmd(args=f'-c unregister_rsc -r test_rsc {timeout} -l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done"') def build_custom_tests(self): """Register tests that target specific cases.""" # verify resource temporary folder is created and used by OCF agents test = self.new_test("rsc_tmp_dir", "Verify creation and use of rsc temporary state directory") - test.add_cmd("ls", args="-al %s" % BuildOptions.RSC_TMP_DIR) + test.add_cmd("ls", args=f"-al {BuildOptions.RSC_TMP_DIR}") test.add_cmd(args='-c register_rsc -r test_rsc -P heartbeat -C ocf -T Dummy ' '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done" ' + self._action_timeout) test.add_cmd(args='-c exec -r test_rsc -a start -t 4000') - test.add_cmd("ls", args="-al %s" % BuildOptions.RSC_TMP_DIR) - test.add_cmd("ls", args="%s/Dummy-test_rsc.state" % BuildOptions.RSC_TMP_DIR) + test.add_cmd("ls", args=f"-al {BuildOptions.RSC_TMP_DIR}") + test.add_cmd("ls", args=f"{BuildOptions.RSC_TMP_DIR}/Dummy-test_rsc.state") test.add_cmd(args='-c exec -r test_rsc -a stop -t 4000') test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done" ') # start delay then stop test test = self.new_test("start_delay", "Verify start delay works as expected.") test.add_cmd(args='-c register_rsc -r test_rsc -P pacemaker -C ocf -T Dummy ' '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done" ' + self._action_timeout) test.add_cmd(args='-c exec -r test_rsc -s 6000 -a start -w -t 6000') test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Done" -t 2000', expected_exitcode=ExitStatus.TIMEOUT) test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Done" -t 6000') test.add_cmd(args='-c exec -r test_rsc -a stop ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:Done" ') test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done" ') # start delay, but cancel before it gets a chance to start test = self.new_test("start_delay_cancel", "Using start_delay, start a rsc, but cancel the start op before execution.") test.add_cmd(args='-c register_rsc -r test_rsc -P pacemaker -C ocf -T Dummy ' '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done" ' + self._action_timeout) test.add_cmd(args='-c exec -r test_rsc -s 5000 -a start -w -t 4000') test.add_cmd(args='-c cancel -r test_rsc -a start ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Cancelled" ') test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Done" -t 5000', expected_exitcode=ExitStatus.TIMEOUT) test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done" ') # Register a bunch of resources, verify we can get info on them test = self.new_test("verify_get_rsc_info", "Register multiple resources, verify retrieval of rsc info.") if "systemd" in self._rsc_classes: test.add_cmd(args='-c register_rsc -r rsc1 -C systemd -T pacemaker-cts-dummyd@3 ' + self._action_timeout) test.add_cmd(args='-c get_rsc_info -r rsc1 ') test.add_cmd(args='-c unregister_rsc -r rsc1 ' + self._action_timeout) test.add_cmd(args='-c get_rsc_info -r rsc1 ', expected_exitcode=ExitStatus.ERROR) test.add_cmd(args='-c register_rsc -r rsc2 -C ocf -T Dummy -P pacemaker ' + self._action_timeout) test.add_cmd(args='-c get_rsc_info -r rsc2 ') test.add_cmd(args='-c unregister_rsc -r rsc2 ' + self._action_timeout) test.add_cmd(args='-c get_rsc_info -r rsc2 ', expected_exitcode=ExitStatus.ERROR) # Register duplicate, verify only one entry exists and can still be removed test = self.new_test("duplicate_registration", "Register resource multiple times, verify only one entry exists and can be removed.") test.add_cmd(args='-c register_rsc -r rsc2 -C ocf -T Dummy -P pacemaker ' + self._action_timeout) test.add_cmd(args="-c get_rsc_info -r rsc2 ", stdout_match="id:rsc2 class:ocf provider:pacemaker type:Dummy") test.add_cmd(args='-c register_rsc -r rsc2 -C ocf -T Dummy -P pacemaker ' + self._action_timeout) test.add_cmd(args="-c get_rsc_info -r rsc2 ", stdout_match="id:rsc2 class:ocf provider:pacemaker type:Dummy") test.add_cmd(args='-c register_rsc -r rsc2 -C ocf -T Stateful -P pacemaker ' + self._action_timeout) test.add_cmd(args="-c get_rsc_info -r rsc2 ", stdout_match="id:rsc2 class:ocf provider:pacemaker type:Stateful") test.add_cmd(args='-c unregister_rsc -r rsc2 ' + self._action_timeout) test.add_cmd(args='-c get_rsc_info -r rsc2 ', expected_exitcode=ExitStatus.ERROR) # verify the option to only send notification to the original client test = self.new_test("notify_orig_client_only", "Verify option to only send notifications to the client originating the action.") test.add_cmd(args='-c register_rsc -r test_rsc -C ocf -P pacemaker -T Dummy ' + self._action_timeout + '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a start ' + self._action_timeout + '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Done" ') test.add_cmd(args='-c exec -r test_rsc -a monitor -i 1s ' + self._action_timeout + ' -n ' '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done"') # this will fail because the monitor notifications should only go to the original caller, which no longer exists. test.add_cmd(args='-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:Done" ' + self._action_timeout, expected_exitcode=ExitStatus.TIMEOUT) test.add_cmd(args='-c cancel -r test_rsc -a monitor -i 1s -t 6000 ') test.add_cmd(args='-c unregister_rsc -r test_rsc ' + self._action_timeout + '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:Done" ') # get metadata test = self.new_test("get_ocf_metadata", "Retrieve metadata for a resource") test.add_cmd(args="-c metadata -C ocf -P pacemaker -T Dummy", stdout_match="resource-agent name=\"Dummy\"") test.add_cmd(args="-c metadata -C ocf -P pacemaker -T Stateful") test.add_cmd(args="-c metadata -P pacemaker -T Stateful", expected_exitcode=ExitStatus.ERROR) test.add_cmd(args="-c metadata -C ocf -P pacemaker -T fake_agent", expected_exitcode=ExitStatus.ERROR) # get stonith metadata test = self.new_test("get_stonith_metadata", "Retrieve stonith metadata for a resource") test.add_cmd(args="-c metadata -C stonith -P pacemaker -T fence_dummy", stdout_match="resource-agent name=\"fence_dummy\"") # get lsb metadata if "lsb" in self._rsc_classes: test = self.new_test("get_lsb_metadata", "Retrieve metadata for an LSB resource") test.add_cmd(args="-c metadata -C lsb -T LSBDummy", stdout_match="resource-agent name='LSBDummy'") # get metadata if "systemd" in self._rsc_classes: test = self.new_test("get_systemd_metadata", "Retrieve metadata for a resource") test.add_cmd(args="-c metadata -C systemd -T pacemaker-cts-dummyd@", stdout_match="resource-agent name=\"pacemaker-cts-dummyd@\"") # get ocf providers test = self.new_test("list_ocf_providers", "Retrieve list of available resource providers, verifies pacemaker is a provider.") test.add_cmd(args="-c list_ocf_providers ", stdout_match="pacemaker") test.add_cmd(args="-c list_ocf_providers -T ping", stdout_match="pacemaker") # Verify agents only exist in their lists test = self.new_test("verify_agent_lists", "Verify the agent lists contain the right data.") if "ocf" in self._rsc_classes: test.add_cmd(args="-c list_agents ", stdout_match="Stateful") test.add_cmd(args="-c list_agents -C ocf", stdout_match="Stateful", stdout_no_match="pacemaker-cts-dummyd@|fence_dummy") if "service" in self._rsc_classes: test.add_cmd(args="-c list_agents -C service", stdout_match="", stdout_no_match="Stateful|fence_dummy") if "lsb" in self._rsc_classes: test.add_cmd(args="-c list_agents", stdout_match="LSBDummy") test.add_cmd(args="-c list_agents -C lsb", stdout_match="LSBDummy", stdout_no_match="pacemaker-cts-dummyd@|Stateful|fence_dummy") test.add_cmd(args="-c list_agents -C service", stdout_match="LSBDummy") if "systemd" in self._rsc_classes: test.add_cmd(args="-c list_agents ", stdout_match="pacemaker-cts-dummyd@") # systemd test.add_cmd(args="-c list_agents -C systemd", stdout_match="", stdout_no_match="Stateful") # should not exist test.add_cmd(args="-c list_agents -C systemd", stdout_match="pacemaker-cts-dummyd@") test.add_cmd(args="-c list_agents -C systemd", stdout_match="", stdout_no_match="fence_dummy") # should not exist if "stonith" in self._rsc_classes: test.add_cmd(args="-c list_agents -C stonith", stdout_match="fence_dummy") # stonith test.add_cmd(args="-c list_agents -C stonith", stdout_match="", # should not exist stdout_no_match="pacemaker-cts-dummyd@") test.add_cmd(args="-c list_agents -C stonith", stdout_match="", stdout_no_match="Stateful") # should not exist test.add_cmd(args="-c list_agents ", stdout_match="fence_dummy") def build_options(): """Handle command line arguments.""" parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, description="Run pacemaker-execd regression tests", epilog="Example: Run only the test 'start_stop'\n" "\t " + sys.argv[0] + " --run-only start_stop\n\n" "Example: Run only the tests with the string 'systemd' present in them\n" "\t " + sys.argv[0] + " --run-only-pattern systemd") parser.add_argument("-l", "--list-tests", action="store_true", help="Print out all registered tests") parser.add_argument("-p", "--run-only-pattern", metavar='PATTERN', help="Run only tests matching the given pattern") parser.add_argument("-r", "--run-only", metavar='TEST', help="Run a specific test") parser.add_argument("-t", "--timeout", type=float, default=2, help="Up to how many seconds each test case waits for the daemon to " "be initialized. Defaults to 2. The value 0 means no limit.") parser.add_argument("-w", "--force-wait", action="store_true", help="Each test case waits the default/specified --timeout for the " "daemon without tracking the log") if BuildOptions.REMOTE_ENABLED: parser.add_argument("-R", "--pacemaker-remote", action="store_true", help="Test pacemaker-remoted binary instead of pacemaker-execd") parser.add_argument("-V", "--verbose", action="store_true", help="Verbose output") args = parser.parse_args() return args def main(): """Run pacemaker-execd regression tests as specified by arguments.""" update_path() # Ensure all command output is in portable locale for comparison os.environ['LC_ALL'] = "C" opts = build_options() if opts.pacemaker_remote: exit_if_proc_running("pacemaker-remoted") else: exit_if_proc_running("corosync") exit_if_proc_running("pacemaker-execd") exit_if_proc_running("pacemaker-fenced") # Create a temporary directory for log files (the directory will # automatically be erased when done) with tempfile.TemporaryDirectory(prefix="cts-exec-") as logdir: tests = ExecTests(verbose=opts.verbose, tls=opts.pacemaker_remote, timeout=opts.timeout, force_wait=opts.force_wait, logdir=logdir) tests.build_generic_tests() tests.build_multi_rsc_tests() tests.build_negative_tests() tests.build_custom_tests() tests.build_stress_tests() if opts.list_tests: tests.print_list() sys.exit(ExitStatus.OK) print("Starting ...") tests.setup_environment() if opts.run_only_pattern: tests.run_tests_matching(opts.run_only_pattern) tests.print_results() elif opts.run_only: tests.run_single(opts.run_only) tests.print_results() else: tests.run_tests() tests.print_results() tests.cleanup_environment() tests.exit() if __name__ == "__main__": main() diff --git a/cts/cts-fencing.in b/cts/cts-fencing.in index e104701411..058d757b87 100644 --- a/cts/cts-fencing.in +++ b/cts/cts-fencing.in @@ -1,953 +1,953 @@ #!@PYTHON@ """ Regression tests for Pacemaker's fencer """ -__copyright__ = "Copyright 2012-2024 the Pacemaker project contributors" +__copyright__ = "Copyright 2012-2025 the Pacemaker project contributors" __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" import argparse import os import sys import subprocess import tempfile # These imports allow running from a source checkout after running `make`. # Note that while this doesn't necessarily mean it will successfully run tests, # but being able to see --help output can be useful. if os.path.exists("@abs_top_srcdir@/python"): sys.path.insert(0, "@abs_top_srcdir@/python") if os.path.exists("@abs_top_builddir@/python") and "@abs_top_builddir@" != "@abs_top_srcdir@": sys.path.insert(0, "@abs_top_builddir@/python") from pacemaker.buildoptions import BuildOptions from pacemaker.exitstatus import ExitStatus from pacemaker._cts.corosync import Corosync, localname from pacemaker._cts.errors import ExitCodeError, OutputFoundError, OutputNotFoundError, XmlValidationError from pacemaker._cts.process import killall, exit_if_proc_running from pacemaker._cts.test import Test, Tests TEST_DIR = sys.path[0] def update_path(): """ Set the PATH environment variable appropriately for the tests """ new_path = os.environ['PATH'] - if os.path.exists("%s/cts-fencing.in" % TEST_DIR): - print("Running tests from the source tree: %s (%s)" % (BuildOptions._BUILD_DIR, TEST_DIR)) + if os.path.exists(f"{TEST_DIR}/cts-fencing.in"): + print(f"Running tests from the source tree: {BuildOptions._BUILD_DIR} ({TEST_DIR})") # For pacemaker-fenced and cts-fence-helper - new_path = "%s/daemons/fenced:%s" % (BuildOptions._BUILD_DIR, new_path) - new_path = "%s/tools:%s" % (BuildOptions._BUILD_DIR, new_path) # For stonith_admin - new_path = "%s/cts/support:%s" % (BuildOptions._BUILD_DIR, new_path) # For cts-support + new_path = f"{BuildOptions._BUILD_DIR}/daemons/fenced:{new_path}" + new_path = f"{BuildOptions._BUILD_DIR}/tools:{new_path}" # For stonith_admin + new_path = f"{BuildOptions._BUILD_DIR}/cts/support:{new_path}" # For cts-support else: - print("Running tests from the install tree: %s (not %s)" % (BuildOptions.DAEMON_DIR, TEST_DIR)) + print(f"Running tests from the install tree: {BuildOptions.DAEMON_DIR} (not {TEST_DIR})") # For pacemaker-fenced, cts-fence-helper, and cts-support - new_path = "%s:%s" % (BuildOptions.DAEMON_DIR, new_path) + new_path = f"{BuildOptions.DAEMON_DIR}:{new_path}" - print('Using PATH="%s"' % new_path) + print(f'Using PATH="{new_path}"') os.environ['PATH'] = new_path class FenceTest(Test): """ Executor for a single test """ def __init__(self, name, description, **kwargs): Test.__init__(self, name, description, **kwargs) self._daemon_location = "pacemaker-fenced" def _kill_daemons(self): killall(["pacemakerd", "pacemaker-fenced"]) def _start_daemons(self): cmd = ["pacemaker-fenced", "--stand-alone", "--logfile", self.logpath] if self.verbose: cmd += ["-V"] - print("Starting %s" % " ".join(cmd)) + s = " ".join(cmd) + print(f"Starting {s}") self._daemon_process = subprocess.Popen(cmd) class FenceTests(Tests): """ Collection of all fencing regression tests """ def __init__(self, **kwargs): Tests.__init__(self, **kwargs) self._corosync = Corosync(self.verbose, self.logdir, "cts-fencing") def new_test(self, name, description): """ Create a named test """ test = FenceTest(name, description, verbose=self.verbose, timeout=self.timeout, force_wait=self.force_wait, logdir=self.logdir) self._tests.append(test) return test def build_api_sanity_tests(self): """ Register tests to verify basic API usage """ verbose_arg = "" if self.verbose: verbose_arg = "-V" test = self.new_test("low_level_api_test", "Sanity-test client API") - test.add_cmd("cts-fence-helper", args="-t %s" % verbose_arg, validate=False) + test.add_cmd("cts-fence-helper", args=f"-t {verbose_arg}", validate=False) test = self.new_test("low_level_api_mainloop_test", "Sanity-test client API using mainloop") - test.add_cmd("cts-fence-helper", args="-m %s" % verbose_arg, validate=False) + test.add_cmd("cts-fence-helper", args=f"-m {verbose_arg}", validate=False) def build_custom_timeout_tests(self): """ Register tests to verify custom timeout usage """ # custom timeout without topology test = self.new_test("custom_timeout_1", "Verify per device timeouts work as expected without using topology") test.add_cmd('stonith_admin', args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"') test.add_cmd('stonith_admin', args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_list=node3 -o pcmk_off_timeout=1') test.add_cmd('stonith_admin', args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o pcmk_host_list=node3 -o pcmk_off_timeout=4') test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5") # timeout is 5+1+4 = 10 test.add_log_pattern("Total timeout set to 12s") # custom timeout _WITH_ topology test = self.new_test("custom_timeout_2", "Verify per device timeouts work as expected _WITH_ topology") test.add_cmd('stonith_admin', args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"') test.add_cmd('stonith_admin', args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_list=node3 -o pcmk_off_timeout=1000ms') test.add_cmd('stonith_admin', args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o pcmk_host_list=node3 -o pcmk_off_timeout=4000s') test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false1") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true1") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 3 -v false2") test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5") # timeout is 5+1+4000 = 4006 test.add_log_pattern("Total timeout set to 4807s") def build_fence_merge_tests(self): """ Register tests to verify when fence operations should be merged """ ### Simple test that overlapping fencing operations get merged test = self.new_test("custom_merge_single", "Verify overlapping identical fencing operations are merged, no fencing levels used") test.add_cmd("stonith_admin", args="--output-as=xml -R false1 -a fence_dummy -o mode=fail -o pcmk_host_list=node3") test.add_cmd("stonith_admin", args="--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_list=node3") test.add_cmd("stonith_admin", args="--output-as=xml -R false2 -a fence_dummy -o mode=fail -o pcmk_host_list=node3") test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True) test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10") ### one merger will happen test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") ### the pattern below signifies that both the original and duplicate operation completed test.add_log_pattern("Operation 'off' targeting node3 by ") test.add_log_pattern("Operation 'off' targeting node3 by ") ### Test that multiple mergers occur test = self.new_test("custom_merge_multiple", "Verify multiple overlapping identical fencing operations are merged") test.add_cmd("stonith_admin", args="--output-as=xml -R false1 -a fence_dummy -o mode=fail -o pcmk_host_list=node3") test.add_cmd("stonith_admin", args="--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o delay=2 -o pcmk_host_list=node3") test.add_cmd("stonith_admin", args="--output-as=xml -R false2 -a fence_dummy -o mode=fail -o pcmk_host_list=node3") test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True) test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True) test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True) test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True) test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10") ### 4 mergers should occur test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") ### the pattern below signifies that both the original and duplicate operation completed test.add_log_pattern("Operation 'off' targeting node3 by ") test.add_log_pattern("Operation 'off' targeting node3 by ") test.add_log_pattern("Operation 'off' targeting node3 by ") test.add_log_pattern("Operation 'off' targeting node3 by ") test.add_log_pattern("Operation 'off' targeting node3 by ") ### Test that multiple mergers occur with topologies used test = self.new_test("custom_merge_with_topology", "Verify multiple overlapping identical fencing operations are merged with fencing levels") test.add_cmd("stonith_admin", args="--output-as=xml -R false1 -a fence_dummy -o mode=fail -o pcmk_host_list=node3") test.add_cmd("stonith_admin", args="--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_list=node3") test.add_cmd("stonith_admin", args="--output-as=xml -R false2 -a fence_dummy -o mode=fail -o pcmk_host_list=node3") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false1") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false2") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true1") test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True) test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True) test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True) test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True) test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10") ### 4 mergers should occur test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") ### the pattern below signifies that both the original and duplicate operation completed test.add_log_pattern("Operation 'off' targeting node3 by ") test.add_log_pattern("Operation 'off' targeting node3 by ") test.add_log_pattern("Operation 'off' targeting node3 by ") test.add_log_pattern("Operation 'off' targeting node3 by ") test.add_log_pattern("Operation 'off' targeting node3 by ") def build_fence_no_merge_tests(self): """ Register tests to verify when fence operations should not be merged """ test = self.new_test("custom_no_merge", "Verify differing fencing operations are not merged") test.add_cmd("stonith_admin", args="--output-as=xml -R false1 -a fence_dummy -o mode=fail -o pcmk_host_list=node3 node2") test.add_cmd("stonith_admin", args="--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_list=node3 node2") test.add_cmd("stonith_admin", args="--output-as=xml -R false2 -a fence_dummy -o mode=fail -o pcmk_host_list=node3 node2") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false1") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false2") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true1") test.add_cmd("stonith_admin", args="--output-as=xml -F node2 -t 10", no_wait=True) test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10") test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client", negative=True) def build_standalone_tests(self): """ Register a grab bag of tests """ # test what happens when all devices timeout test = self.new_test("fence_multi_device_failure", "Verify that all devices timeout, a fencing failure is returned") test.add_cmd("stonith_admin", args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R false3 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 2", expected_exitcode=ExitStatus.TIMEOUT) test.add_log_pattern("Total timeout set to 7s") test.add_log_pattern("targeting node3 using false1 returned ") test.add_log_pattern("targeting node3 using false2 returned ") test.add_log_pattern("targeting node3 using false3 returned ") # test what happens when multiple devices can fence a node, but the first device fails test = self.new_test("fence_device_failure_rollover", "Verify that when one fence device fails for a node, the others are tried") test.add_cmd("stonith_admin", args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5") test.add_log_pattern("Total timeout set to 18s") # test what happens when we try to use a missing fence-agent test = self.new_test("fence_missing_agent", "Verify proper error-handling when using a non-existent fence-agent") test.add_cmd("stonith_admin", args="--output-as=xml -R true1 -a fence_missing -o mode=pass -o pcmk_host_list=node3") test.add_cmd("stonith_admin", args="--output-as=xml -R true2 -a fence_dummy -o mode=pass -o pcmk_host_list=node2") test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5", expected_exitcode=ExitStatus.NOSUCH) test.add_cmd("stonith_admin", args="--output-as=xml -F node2 -t 5") # simple topology test for one device test = self.new_test("topology_simple", "Verify all fencing devices at a level are used") test.add_cmd("stonith_admin", args='--output-as=xml -R true -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v true") test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5") test.add_log_pattern("Total timeout set to 6s") test.add_log_pattern("targeting node3 using true returned 0") # add topology, delete topology, verify fencing still works test = self.new_test("topology_add_remove", "Verify fencing occurrs after all topology levels are removed") test.add_cmd("stonith_admin", args='--output-as=xml -R true -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v true") test.add_cmd("stonith_admin", args="--output-as=xml -d node3 -i 1") test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5") test.add_log_pattern("Total timeout set to 6s") test.add_log_pattern("targeting node3 using true returned 0") # test what happens when the first fencing level has multiple devices test = self.new_test("topology_device_fails", "Verify if one device in a level fails, the other is tried") test.add_cmd("stonith_admin", args='--output-as=xml -R false -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R true -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true") test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 20") test.add_log_pattern("Total timeout set to 48s") test.add_log_pattern("targeting node3 using false returned 1") test.add_log_pattern("targeting node3 using true returned 0") # test what happens when the first fencing level fails test = self.new_test("topology_multi_level_fails", "Verify if one level fails, the next leve is tried") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R true3 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R true4 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false1") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v true1") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true2") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v false2") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 3 -v true3") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 3 -v true4") test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 3") test.add_log_pattern("Total timeout set to 21s") test.add_log_pattern("targeting node3 using false1 returned 1") test.add_log_pattern("targeting node3 using false2 returned 1") test.add_log_pattern("targeting node3 using true3 returned 0") test.add_log_pattern("targeting node3 using true4 returned 0") # test what happens when the first fencing level had devices that no one has registered test = self.new_test("topology_missing_devices", "Verify topology can continue with missing devices") test.add_cmd("stonith_admin", args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R true3 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R true4 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false1") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v true1") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true2") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v false2") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 3 -v true3") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 3 -v true4") test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5") # Test what happens if multiple fencing levels are defined, and then the first one is removed test = self.new_test("topology_level_removal", "Verify level removal works") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R true3 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R true4 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false1") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v true1") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true2") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v false2") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 3 -v true3") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 3 -v true4") # Now remove level 2, verify none of the devices in level two are hit test.add_cmd("stonith_admin", args="--output-as=xml -d node3 -i 2") test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 20") test.add_log_pattern("Total timeout set to 96s") test.add_log_pattern("targeting node3 using false1 returned 1") test.add_log_pattern("targeting node3 using false2 returned ", negative=True) test.add_log_pattern("targeting node3 using true3 returned 0") test.add_log_pattern("targeting node3 using true4 returned 0") # Test targeting a topology level by node name pattern test = self.new_test("topology_level_pattern", "Verify targeting topology by node name pattern works") test.add_cmd("stonith_admin", args='--output-as=xml -R true -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args="--output-as=xml -r '@node.*' -i 1 -v true") test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5") test.add_log_pattern("targeting node3 using true returned 0") # test allowing commas and semicolons as delimiters in pcmk_host_list test = self.new_test("host_list_delimiters", "Verify commas and semicolons can be used as pcmk_host_list delimiters") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1,node2,node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list=pcmk1;pcmk2;pcmk3"') test.add_cmd("stonith_admin", args="stonith_admin --output-as=xml -F node2 -t 5") test.add_cmd("stonith_admin", args="stonith_admin --output-as=xml -F pcmk3 -t 5") test.add_log_pattern("targeting node2 using true1 returned 0") test.add_log_pattern("targeting pcmk3 using true2 returned 0") # test the stonith builds the correct list of devices that can fence a node test = self.new_test("list_devices", "Verify list of devices that can fence a node is correct") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R true3 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args="--output-as=xml -l node1 -V", stdout_match="true2", stdout_no_match="true1") test.add_cmd("stonith_admin", args="--output-as=xml -l node1 -V", stdout_match="true3", stdout_no_match="true1") # simple test of device monitor test = self.new_test("monitor", "Verify device is reachable") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=node3"') test.add_cmd("stonith_admin", args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node3"') test.add_cmd("stonith_admin", args="--output-as=xml -Q true1") test.add_cmd("stonith_admin", args="--output-as=xml -Q false1") test.add_cmd("stonith_admin", args="--output-as=xml -Q true2", expected_exitcode=ExitStatus.NOSUCH) # Verify monitor occurs for duration of timeout period on failure test = self.new_test("monitor_timeout", "Verify monitor uses duration of timeout period given") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=fail -o monitor_mode=fail -o pcmk_host_list=node3') test.add_cmd("stonith_admin", args="--output-as=xml -Q true1 -t 5", expected_exitcode=ExitStatus.ERROR) test.add_log_pattern("Attempt 2 to execute") # Verify monitor occurs for duration of timeout period on failure, but stops at max retries test = self.new_test("monitor_timeout_max_retries", "Verify monitor retries until max retry value or timeout is hit") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=fail -o monitor_mode=fail -o pcmk_host_list=node3') test.add_cmd("stonith_admin", args="--output-as=xml -Q true1 -t 15", expected_exitcode=ExitStatus.ERROR) test.add_log_pattern("Attempted to execute agent fence_dummy (list) the maximum number of times") # simple register test test = self.new_test("register", "Verify devices can be registered and un-registered") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_list=node3') test.add_cmd("stonith_admin", args="--output-as=xml -Q true1") test.add_cmd("stonith_admin", args="--output-as=xml -D true1") test.add_cmd("stonith_admin", args="--output-as=xml -Q true1", expected_exitcode=ExitStatus.NOSUCH) # simple reboot test test = self.new_test("reboot", "Verify devices can be rebooted") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_list=node3') test.add_cmd("stonith_admin", args="--output-as=xml -B node3 -t 5") test.add_cmd("stonith_admin", args="--output-as=xml -D true1") test.add_cmd("stonith_admin", args="--output-as=xml -Q true1", expected_exitcode=ExitStatus.NOSUCH) # test fencing history test = self.new_test("fence_history", "Verify last fencing operation is returned") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_list=node3') test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5 -V") test.add_cmd("stonith_admin", args="--output-as=xml -H node3", stdout_match='action="off" target="node3" .* status="success"') # simple test of dynamic list query test = self.new_test("dynamic_list_query", "Verify dynamic list of fencing devices can be retrieved") test.add_cmd("stonith_admin", args="--output-as=xml -R true1 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1") test.add_cmd("stonith_admin", args="--output-as=xml -R true2 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1") test.add_cmd("stonith_admin", args="--output-as=xml -R true3 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1") test.add_cmd("stonith_admin", args="--output-as=xml -l fake_port_1", stdout_match='count="3"') # fence using dynamic list query test = self.new_test("fence_dynamic_list_query", "Verify dynamic list of fencing devices can be retrieved") test.add_cmd("stonith_admin", args="--output-as=xml -R true1 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1") test.add_cmd("stonith_admin", args="--output-as=xml -R true2 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1") test.add_cmd("stonith_admin", args="--output-as=xml -R true3 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1") test.add_cmd("stonith_admin", args="--output-as=xml -F fake_port_1 -t 5 -V") # simple test of query using status action test = self.new_test("status_query", "Verify dynamic list of fencing devices can be retrieved") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_check=status') test.add_cmd("stonith_admin", args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o pcmk_host_check=status') test.add_cmd("stonith_admin", args='--output-as=xml -R true3 -a fence_dummy -o mode=pass -o pcmk_host_check=status') test.add_cmd("stonith_admin", args="--output-as=xml -l fake_port_1", stdout_match='count="3"') # test what happens when no reboot action is advertised test = self.new_test("no_reboot_support", "Verify reboot action defaults to off when no reboot action is advertised by agent") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy_no_reboot -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args="--output-as=xml -B node1 -t 5 -V") test.add_log_pattern("does not support reboot") test.add_log_pattern("using true1 returned 0") # make sure reboot is used when reboot action is advertised test = self.new_test("with_reboot_support", "Verify reboot action can be used when metadata advertises it") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args="--output-as=xml -B node1 -t 5 -V") test.add_log_pattern("does not advertise support for 'reboot', performing 'off'", negative=True) test.add_log_pattern("using true1 returned 0") # make sure all fencing delays are applied correctly and taken into account by fencing timeouts with topology test = self.new_test("topology_delays", "Verify all fencing delays are applied correctly and taken into account by fencing timeouts with topology") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3" -o pcmk_delay_base=1') test.add_cmd("stonith_admin", args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3" -o pcmk_delay_base=1') # Resulting "random" delay will always be 1 since (rand() % (delay_max - delay_base)) is always 0 here test.add_cmd("stonith_admin", args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3" -o pcmk_delay_base=1 -o pcmk_delay_max=2') test.add_cmd("stonith_admin", args='--output-as=xml -R true3 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"') test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v true1") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false1") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true2") test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true3") test.add_cmd("stonith_admin", args="--output-as=xml -F node3 --delay 1") # Total fencing timeout takes all fencing delays into account test.add_log_pattern("Total timeout set to 582s") # Fencing timeout for the first device takes the requested fencing delay # and pcmk_delay_base into account test.add_log_pattern(r"Requesting that .* perform 'off' action targeting node3 using true1 .*146s.*", regex=True) # Requested fencing delay is applied only for the first device in the # first level, with the static delay from pcmk_delay_base added test.add_log_pattern("Delaying 'off' action targeting node3 using true1 for 2s | timeout=120s requested_delay=1s base=1s max=1s") # Fencing timeout no longer takes the requested fencing delay into account for further devices test.add_log_pattern(r"Requesting that .* perform 'off' action targeting node3 using false1 .*145s.*", regex=True) # Requested fencing delay is no longer applied for further devices test.add_log_pattern("Delaying 'off' action targeting node3 using false1 for 1s | timeout=120s requested_delay=0s base=1s max=1s") # Fencing timeout takes pcmk_delay_max into account test.add_log_pattern(r"Requesting that .* perform 'off' action targeting node3 using true2 .*146s.*", regex=True) test.add_log_pattern("Delaying 'off' action targeting node3 using true2 for 1s | timeout=120s requested_delay=0s base=1s max=2s") test.add_log_pattern("Delaying 'off' action targeting node3 using true3", negative=True) def build_nodeid_tests(self): """ Register tests that use a corosync node id """ our_uname = localname() ### verify nodeid is supplied when nodeid is in the metadata parameters test = self.new_test("supply_nodeid", "Verify nodeid is given when fence agent has nodeid as parameter") test.add_cmd("stonith_admin", - args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=%s"' % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -F %s -t 3" % our_uname) - test.add_log_pattern("as nodeid with fence action 'off' targeting %s" % (our_uname)) + args=f'--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list={our_uname}"') + test.add_cmd("stonith_admin", args=f"--output-as=xml -F {our_uname} -t 3") + test.add_log_pattern(f"as nodeid with fence action 'off' targeting {our_uname}") ### verify nodeid is _NOT_ supplied when nodeid is not in the metadata parameters test = self.new_test("do_not_supply_nodeid", "Verify nodeid is _NOT_ given when fence agent does not have nodeid as parameter") # use a host name that won't be in corosync.conf test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy_no_nodeid ' - '-o mode=pass -o pcmk_host_list="regr-test %s"' - % our_uname) + f'-o mode=pass -o pcmk_host_list="regr-test {our_uname}"') test.add_cmd("stonith_admin", args="--output-as=xml -F regr-test -t 3") test.add_log_pattern("as nodeid with fence action 'off' targeting regr-test", negative=True) - test.add_cmd("stonith_admin", args="--output-as=xml -F %s -t 3" % our_uname) - test.add_log_pattern("as nodeid with fence action 'off' targeting %s" % our_uname, + test.add_cmd("stonith_admin", args=f"--output-as=xml -F {our_uname} -t 3") + test.add_log_pattern("as nodeid with fence action 'off' targeting {our_uname}", negative=True) def build_unfence_tests(self): """ Register tests that verify unfencing """ our_uname = localname() ### verify unfencing using automatic unfencing test = self.new_test("unfence_required_1", "Verify require unfencing on all devices when automatic=true in agent's metadata") test.add_cmd('stonith_admin', - args='--output-as=xml -R true1 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list=%s"' % our_uname) + args=f'--output-as=xml -R true1 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname}"') test.add_cmd('stonith_admin', - args='--output-as=xml -R true2 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list=%s"' % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -U %s -t 3" % our_uname) + args=f'--output-as=xml -R true2 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname}"') + test.add_cmd("stonith_admin", args=f"--output-as=xml -U {our_uname} -t 3") # both devices should be executed test.add_log_pattern("using true1 returned 0") test.add_log_pattern("using true2 returned 0") ### verify unfencing using automatic unfencing fails if any of the required agents fail test = self.new_test("unfence_required_2", "Verify require unfencing on all devices when automatic=true in agent's metadata") test.add_cmd('stonith_admin', - args='--output-as=xml -R true1 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list=%s"' % our_uname) + args=f'--output-as=xml -R true1 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname}"') test.add_cmd('stonith_admin', - args='--output-as=xml -R true2 -a fence_dummy_auto_unfence -o mode=fail -o "pcmk_host_list=%s"' % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -U %s -t 6" % our_uname, expected_exitcode=ExitStatus.ERROR) + args=f'--output-as=xml -R true2 -a fence_dummy_auto_unfence -o mode=fail -o "pcmk_host_list={our_uname}"') + test.add_cmd("stonith_admin", args=f"--output-as=xml -U {our_uname} -t 6", expected_exitcode=ExitStatus.ERROR) ### verify unfencing using automatic devices with topology test = self.new_test("unfence_required_3", "Verify require unfencing on all devices even when at different topology levels") test.add_cmd('stonith_admin', - args='--output-as=xml -R true1 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list=%s node3"' % our_uname) + args=f'--output-as=xml -R true1 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname} node3"') test.add_cmd('stonith_admin', - args='--output-as=xml -R true2 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list=%s node3"' % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -r %s -i 1 -v true1" % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -r %s -i 2 -v true2" % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -U %s -t 3" % our_uname) + args=f'--output-as=xml -R true2 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname} node3"') + test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 1 -v true1") + test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 2 -v true2") + test.add_cmd("stonith_admin", args=f"--output-as=xml -U {our_uname} -t 3") test.add_log_pattern("using true1 returned 0") test.add_log_pattern("using true2 returned 0") ### verify unfencing using automatic devices with topology test = self.new_test("unfence_required_4", "Verify all required devices are executed even with topology levels fail") test.add_cmd('stonith_admin', - args='--output-as=xml -R true1 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list=%s node3"' % our_uname) + args=f'--output-as=xml -R true1 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname} node3"') test.add_cmd('stonith_admin', - args='--output-as=xml -R true2 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list=%s node3"' % our_uname) + args=f'--output-as=xml -R true2 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname} node3"') test.add_cmd('stonith_admin', - args='--output-as=xml -R true3 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list=%s node3"' % our_uname) + args=f'--output-as=xml -R true3 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname} node3"') test.add_cmd('stonith_admin', - args='--output-as=xml -R true4 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list=%s node3"' % our_uname) + args=f'--output-as=xml -R true4 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname} node3"') test.add_cmd('stonith_admin', - args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=%s node3"' % our_uname) + args=f'--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list={our_uname} node3"') test.add_cmd('stonith_admin', - args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o "pcmk_host_list=%s node3"' % our_uname) + args=f'--output-as=xml -R false2 -a fence_dummy -o mode=fail -o "pcmk_host_list={our_uname} node3"') test.add_cmd('stonith_admin', - args='--output-as=xml -R false3 -a fence_dummy -o mode=fail -o "pcmk_host_list=%s node3"' % our_uname) + args=f'--output-as=xml -R false3 -a fence_dummy -o mode=fail -o "pcmk_host_list={our_uname} node3"') test.add_cmd('stonith_admin', - args='--output-as=xml -R false4 -a fence_dummy -o mode=fail -o "pcmk_host_list=%s node3"' % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -r %s -i 1 -v true1" % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -r %s -i 1 -v false1" % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -r %s -i 2 -v false2" % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -r %s -i 2 -v true2" % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -r %s -i 2 -v false3" % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -r %s -i 2 -v true3" % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -r %s -i 3 -v false4" % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -r %s -i 4 -v true4" % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -U %s -t 3" % our_uname) + args=f'--output-as=xml -R false4 -a fence_dummy -o mode=fail -o "pcmk_host_list={our_uname} node3"') + test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 1 -v true1") + test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 1 -v false1") + test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 2 -v false2") + test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 2 -v true2") + test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 2 -v false3") + test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 2 -v true3") + test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 3 -v false4") + test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 4 -v true4") + test.add_cmd("stonith_admin", args=f"--output-as=xml -U {our_uname} -t 3") test.add_log_pattern("using true1 returned 0") test.add_log_pattern("using true2 returned 0") test.add_log_pattern("using true3 returned 0") test.add_log_pattern("using true4 returned 0") def build_unfence_on_target_tests(self): """ Register tests that verify unfencing that runs on the target """ our_uname = localname() ### verify unfencing using on_target device test = self.new_test("unfence_on_target_1", "Verify unfencing with on_target = true") test.add_cmd("stonith_admin", - args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=%s"' % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -U %s -t 3" % our_uname) + args=f'--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list={our_uname}"') + test.add_cmd("stonith_admin", args=f"--output-as=xml -U {our_uname} -t 3") test.add_log_pattern("(on) to be executed on target") ### verify failure of unfencing using on_target device test = self.new_test("unfence_on_target_2", "Verify failure unfencing with on_target = true") test.add_cmd("stonith_admin", - args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=%s node_fake_1234"' % our_uname) + args=f'--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list={our_uname} node_fake_1234"') test.add_cmd("stonith_admin", args="--output-as=xml -U node_fake_1234 -t 3", expected_exitcode=ExitStatus.NOSUCH) test.add_log_pattern("(on) to be executed on target") ### verify unfencing using on_target device with topology test = self.new_test("unfence_on_target_3", "Verify unfencing with on_target = true using topology") test.add_cmd("stonith_admin", - args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=%s node3"' % our_uname) + args=f'--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list={our_uname} node3"') test.add_cmd("stonith_admin", - args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list=%s node3"' % our_uname) + args=f'--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list={our_uname} node3"') - test.add_cmd("stonith_admin", args="--output-as=xml -r %s -i 1 -v true1" % our_uname) - test.add_cmd("stonith_admin", args="--output-as=xml -r %s -i 2 -v true2" % our_uname) + test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 1 -v true1") + test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 2 -v true2") - test.add_cmd("stonith_admin", args="--output-as=xml -U %s -t 3" % our_uname) + test.add_cmd("stonith_admin", args=f"--output-as=xml -U {our_uname} -t 3") test.add_log_pattern("(on) to be executed on target") ### verify unfencing using on_target device with topology fails when target node doesn't exist test = self.new_test("unfence_on_target_4", "Verify unfencing failure with on_target = true using topology") test.add_cmd("stonith_admin", - args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=%s node_fake"' % our_uname) + args=f'--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list={our_uname} node_fake"') test.add_cmd("stonith_admin", - args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list=%s node_fake"' % our_uname) + args=f'--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list={our_uname} node_fake"') test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 1 -v true1") test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 2 -v true2") test.add_cmd("stonith_admin", args="--output-as=xml -U node_fake -t 3", expected_exitcode=ExitStatus.NOSUCH) test.add_log_pattern("(on) to be executed on target") def build_remap_tests(self): """ Register tests that verify remapping of reboots to off-on """ test = self.new_test("remap_simple", "Verify sequential topology reboot is remapped to all-off-then-all-on") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_list=node_fake ' '-o pcmk_off_timeout=1 -o pcmk_reboot_timeout=10') test.add_cmd("stonith_admin", args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o pcmk_host_list=node_fake ' '-o pcmk_off_timeout=2 -o pcmk_reboot_timeout=20') test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 1 -v true1 -v true2") test.add_cmd("stonith_admin", args="--output-as=xml -B node_fake -t 5") test.add_log_pattern("Remapping multiple-device reboot targeting node_fake") # timeout should be sum of off timeouts (1+2=3), not reboot timeouts (10+20=30) test.add_log_pattern("Total timeout set to 3s for peer's fencing targeting node_fake") test.add_log_pattern("perform 'off' action targeting node_fake using true1") test.add_log_pattern("perform 'off' action targeting node_fake using true2") test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'") # fence_dummy sets "on" as an on_target action test.add_log_pattern("Ignoring true1 'on' failure (no capable peers) targeting node_fake") test.add_log_pattern("Ignoring true2 'on' failure (no capable peers) targeting node_fake") test.add_log_pattern("Undoing remap of reboot targeting node_fake") test = self.new_test("remap_simple_off", "Verify sequential topology reboot skips 'on' if " "pcmk_reboot_action=off or agent doesn't support " "'on'") test.add_cmd("stonith_admin", args="--output-as=xml -R true1 -a fence_dummy -o mode=pass " "-o pcmk_host_list=node_fake -o pcmk_off_timeout=1 " "-o pcmk_reboot_timeout=10 -o pcmk_reboot_action=off") test.add_cmd("stonith_admin", args="--output-as=xml -R true2 -a fence_dummy_no_on " "-o mode=pass -o pcmk_host_list=node_fake " "-o pcmk_off_timeout=2 -o pcmk_reboot_timeout=20") test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 1 -v true1 -v true2") test.add_cmd("stonith_admin", args="--output-as=xml -B node_fake -t 5") test.add_log_pattern("Remapping multiple-device reboot targeting node_fake") # timeout should be sum of off timeouts (1+2=3), not reboot timeouts (10+20=30) test.add_log_pattern("Total timeout set to 3s for peer's fencing targeting node_fake") test.add_log_pattern("perform 'off' action targeting node_fake using true1") test.add_log_pattern("perform 'off' action targeting node_fake using true2") test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'") # "on" should be skipped test.add_log_pattern("Not turning node_fake back on using " "true1 because the device is configured " "to stay off") test.add_log_pattern("Not turning node_fake back on using true2" " because the agent doesn't support 'on'") test.add_log_pattern("Undoing remap of reboot targeting node_fake") test = self.new_test("remap_automatic", "Verify remapped topology reboot skips automatic 'on'") test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy_auto_unfence ' '-o mode=pass -o pcmk_host_list=node_fake') test.add_cmd("stonith_admin", args='--output-as=xml -R true2 -a fence_dummy_auto_unfence ' '-o "mode=pass" -o pcmk_host_list=node_fake') test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 1 -v true1 -v true2") test.add_cmd("stonith_admin", args="--output-as=xml -B node_fake -t 5") test.add_log_pattern("Remapping multiple-device reboot targeting node_fake") test.add_log_pattern("perform 'off' action targeting node_fake using true1") test.add_log_pattern("perform 'off' action targeting node_fake using true2") test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'") test.add_log_pattern("Undoing remap of reboot targeting node_fake") test.add_log_pattern("perform 'on' action targeting node_fake using", negative=True) test.add_log_pattern("'on' failure", negative=True) test = self.new_test("remap_complex_1", "Verify remapped topology reboot in second level works if non-remapped first level fails") test.add_cmd("stonith_admin", args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o pcmk_host_list=node_fake') test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_list=node_fake') test.add_cmd("stonith_admin", args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o pcmk_host_list=node_fake') test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 1 -v false1") test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 2 -v true1 -v true2") test.add_cmd("stonith_admin", args="--output-as=xml -B node_fake -t 5") test.add_log_pattern("perform 'reboot' action targeting node_fake using false1") test.add_log_pattern("Remapping multiple-device reboot targeting node_fake") test.add_log_pattern("perform 'off' action targeting node_fake using true1") test.add_log_pattern("perform 'off' action targeting node_fake using true2") test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'") test.add_log_pattern("Ignoring true1 'on' failure (no capable peers) targeting node_fake") test.add_log_pattern("Ignoring true2 'on' failure (no capable peers) targeting node_fake") test.add_log_pattern("Undoing remap of reboot targeting node_fake") test = self.new_test("remap_complex_2", "Verify remapped topology reboot failure in second level proceeds to third level") test.add_cmd("stonith_admin", args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o pcmk_host_list=node_fake') test.add_cmd("stonith_admin", args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o pcmk_host_list=node_fake') test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_list=node_fake') test.add_cmd("stonith_admin", args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o pcmk_host_list=node_fake') test.add_cmd("stonith_admin", args='--output-as=xml -R true3 -a fence_dummy -o mode=pass -o pcmk_host_list=node_fake') test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 1 -v false1") test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 2 -v true1 -v false2 -v true3") test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 3 -v true2") test.add_cmd("stonith_admin", args="--output-as=xml -B node_fake -t 5") test.add_log_pattern("perform 'reboot' action targeting node_fake using false1") test.add_log_pattern("Remapping multiple-device reboot targeting node_fake") test.add_log_pattern("perform 'off' action targeting node_fake using true1") test.add_log_pattern("perform 'off' action targeting node_fake using false2") test.add_log_pattern("Attempted to execute agent fence_dummy (off) the maximum number of times") test.add_log_pattern("Undoing remap of reboot targeting node_fake") test.add_log_pattern("perform 'reboot' action targeting node_fake using true2") test.add_log_pattern("node_fake with true3", negative=True) def build_query_tests(self): """ run stonith_admin --metadata for the fence_dummy agent and check command output """ test = self.new_test("get_metadata", "Run stonith_admin --metadata for the fence_dummy agent") test.add_cmd("stonith_admin", args="--output-as=xml -a fence_dummy --metadata", stdout_match=' 1.""" if n == 1: return "" return "S" if __name__ == '__main__': environment = CtsLab(sys.argv[1:]) iters = environment["iterations"] tests = [] # Set the signal handler signal.signal(15, sig_handler) signal.signal(10, sig_handler) # Create the Cluster Manager object cm = None if environment["Stack"] == "corosync 2+": cm = Corosync2() else: - LogFactory().log("Unknown stack: %s" % environment["stack"]) + LogFactory().log(f"Unknown stack: {environment['stack']}") sys.exit(1) if environment["TruncateLog"]: if environment["OutputFile"] is None: LogFactory().log("Ignoring truncate request because no output file specified") else: - LogFactory().log("Truncating %s" % environment["OutputFile"]) + LogFactory().log(f"Truncating {environment['OutputFile']}") with open(environment["OutputFile"], "w", encoding="utf-8") as outputfile: outputfile.truncate(0) audits = audit_list(cm) if environment["ListTests"]: tests = test_list(cm, audits) - LogFactory().log("Total %d tests" % len(tests)) + LogFactory().log(f"Total {len(tests)} tests") for test in tests: LogFactory().log(test.name) sys.exit(0) elif len(environment["tests"]) == 0: tests = test_list(cm, audits) else: chosen = environment["tests"] for test_case in chosen: match = None for test in test_list(cm, audits): if test.name == test_case: match = test if not match: LogFactory().log("--choose: No applicable/valid tests chosen") sys.exit(1) else: tests.append(match) # Scenario selection if environment["scenario"] == "all-once": iters = len(tests) scenario = AllOnce(cm, [BootCluster(cm, environment)], audits, tests) elif environment["scenario"] == "sequence": scenario = Sequence(cm, [BootCluster(cm, environment)], audits, tests) elif environment["scenario"] == "boot": scenario = Boot(cm, [LeaveBooted(cm, environment)], audits, []) else: scenario = RandomTests(cm, [BootCluster(cm, environment)], audits, tests) - LogFactory().log(">>>>>>>>>>>>>>>> BEGINNING %r TEST%s" % (iters, plural_s(iters))) - LogFactory().log("Stack: %s (%s)" % (environment["Stack"], environment["Name"])) - LogFactory().log("Schema: %s" % environment["Schema"]) - LogFactory().log("Scenario: %s" % scenario.__doc__) - LogFactory().log("CTS Exerciser: %s" % environment["cts-exerciser"]) - LogFactory().log("CTS Logfile: %s" % environment["OutputFile"]) - LogFactory().log("Random Seed: %s" % environment["RandSeed"]) + LogFactory().log(f">>>>>>>>>>>>>>>> BEGINNING {iters!r} TEST{plural_s(iters)}") + LogFactory().log(f"Stack: {environment['Stack']} ({environment['Name']})") + LogFactory().log(f"Schema: {environment['Schema']}") + LogFactory().log(f"Scenario: {scenario.__doc__}") + LogFactory().log(f"CTS Exerciser: {environment['cts-exerciser']}") + LogFactory().log(f"CTS Logfile: {environment['OutputFile']}") + LogFactory().log(f"Random Seed: {environment['RandSeed']}") if "syslogd" in environment: - LogFactory().log("Syslog variant: %s" % environment["syslogd"].strip()) - LogFactory().log("System log files: %s" % environment["LogFileName"]) + LogFactory().log(f"Syslog variant: {environment['syslogd'].strip()}") + LogFactory().log(f"System log files: {environment['LogFileName']}") if "IPBase" in environment: - LogFactory().log("Base IP for resources: %s" % environment["IPBase"]) + LogFactory().log(f"Base IP for resources: {environment['IPBase']}") - LogFactory().log("Cluster starts at boot: %d" % environment["at-boot"]) + LogFactory().log(f"Cluster starts at boot: {environment['at-boot']}") environment.dump() rc = environment.run(scenario, iters) sys.exit(rc) # vim: set filetype=python expandtab tabstop=4 softtabstop=4 shiftwidth=4 textwidth=120: diff --git a/cts/cts-regression.in b/cts/cts-regression.in index c6837c4614..69fb7a9a02 100644 --- a/cts/cts-regression.in +++ b/cts/cts-regression.in @@ -1,293 +1,291 @@ #!@PYTHON@ """Convenience wrapper for running Pacemaker regression tests. Usage: cts-regression [-h] [-V] [-v] [COMPONENT ...] """ -__copyright__ = 'Copyright 2012-2023 the Pacemaker project contributors' +__copyright__ = 'Copyright 2012-2025 the Pacemaker project contributors' __license__ = 'GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY' import argparse import os import subprocess import sys import textwrap # These imports allow running from a source checkout after running `make`. # Note that while this doesn't necessarily mean it will successfully run tests, # but being able to see --help output can be useful. if os.path.exists("@abs_top_srcdir@/python"): sys.path.insert(0, "@abs_top_srcdir@/python") if os.path.exists("@abs_top_builddir@/python") and "@abs_top_builddir@" != "@abs_top_srcdir@": sys.path.insert(0, "@abs_top_builddir@/python") from pacemaker.buildoptions import BuildOptions from pacemaker.exitstatus import ExitStatus class Component(): """A class for running regression tests on a component. "Component" refers to a Pacemaker component, such as the scheduler. :attribute name: The name of the component. :type name: str :attribute description: The description of the component. :type description: str :attribute requires_root: Whether the component's tests must be run as root. :type requires_root: bool :attribute supports_valgrind: Whether the component's tests support running under valgrind. :type supports_valgrind: bool :attribute cmd: The command to run the component's tests, along with any required options. :type cmd: list[str] :method run([verbose=False], [valgrind=False]): Run the component's regression tests and return the result. """ def __init__(self, name, description, test_home, requires_root=False, supports_valgrind=False): """Constructor for the :class:`Component` class. :param name: The name of the component. :type name: str :param description: The description of the component. :type description: str :param test_home: The directory where the component's tests reside. :type test_home: str :param requires_root: Whether the component's tests must be run as root. :type requires_root: bool :param supports_valgrind: Whether the component's tests support running under valgrind. :type supports_valgrind: bool """ self.name = name self.description = description self.requires_root = requires_root self.supports_valgrind = supports_valgrind if self.name == 'pacemaker_remote': self.cmd = [os.path.join(test_home, 'cts-exec'), '-R'] else: - self.cmd = [os.path.join(test_home, 'cts-%s' % self.name)] + self.cmd = [os.path.join(test_home, f"cts-{self.name}")] def run(self, verbose=False, valgrind=False): """Run the component's regression tests and return the result. :param verbose: Whether to increase test output verbosity. :type verbose: bool :param valgrind: Whether to run the test under valgrind. :type valgrind: bool :return: The exit code from the component's test suite. :rtype: :class:`ExitStatus` """ - print('Executing the %s regression tests' % self.name) + print(f"Executing the {self.name} regression tests") print('=' * 60) cmd = self.cmd if self.requires_root and os.geteuid() != 0: print('Enter the sudo password if prompted') cmd = ['sudo'] + self.cmd if verbose: cmd.append('--verbose') if self.supports_valgrind and valgrind: cmd.append('--valgrind') try: rc = ExitStatus(subprocess.call(cmd)) except OSError as err: - error_print('Failed to execute %s tests: %s' % (self.name, err)) + error_print(f"Failed to execute {self.name} tests: {err}") rc = ExitStatus.NOT_INSTALLED print('=' * 60 + '\n\n') return rc class ComponentsArgAction(argparse.Action): """A class to handle `components` arguments. This class handles special cases and cleans up the `components` list. Specifically, it does the following: * Enforce a default value of ['cli', 'scheduler']. * Replace the 'all' alias with the components that it represents. * Get rid of duplicates. The main motivation is that when the `choices` argument of :meth:`parser.add_argument()` is specified, the `default` argument must contain exactly one value (not `None` and not a list). We want our default to be a list of components, namely `cli` and `scheduler`. """ def __call__(self, parser, namespace, values, option_string=None): all_components = ['attrd', 'cli', 'exec', 'fencing', 'scheduler'] default_components = ['cli', 'scheduler'] if not values: setattr(namespace, self.dest, default_components) return # If no argument is specified, the default gets passed as a # string 'default' instead of as a list ['default']. Probably # a bug in argparse. The below gives us a list. if not isinstance(values, list): values = [values] components = set(values) # If 'all', is found, replace it with the components it represents. try: components.remove('all') components.update(set(all_components)) except KeyError: pass # Same for 'default' try: components.remove('default') components.update(set(default_components)) except KeyError: pass setattr(namespace, self.dest, sorted(list(components))) def error_print(msg): """Print an error message. :param msg: Message to print. :type msg: str """ - print(' * ERROR: %s' % msg) + print(f" * ERROR: {msg}") def run_components(components, verbose=False, valgrind=False): """Run components' regression tests and report results for each. :param components: A list of names of components for which to run tests. :type components: list[:class:`Component`] :return: :attr:`ExitStatus.OK` if all tests were successful, :attr:`ExitStatus.ERROR` otherwise. :rtype: :class:`ExitStatus` """ failed = [] for comp in components: rc = comp.run(verbose, valgrind) if rc != ExitStatus.OK: - error_print('%s regression tests failed (%s)' % (comp.name, rc)) + error_print(f"{comp.name} regression tests failed ({rc})") failed.append(comp.name) if failed: print('Failed regression tests:', end='') for comp in failed: - print(' %s' % comp, end='') + print(f" {comp}", end='') print() return ExitStatus.ERROR return ExitStatus.OK def main(): """Run Pacemaker regression tests as specified by arguments.""" try: test_home = os.path.dirname(os.readlink(sys.argv[0])) except OSError: test_home = os.path.dirname(sys.argv[0]) # Available components components = { 'attrd': Component( 'attrd', 'Attribute manager', test_home, requires_root=True, supports_valgrind=False, ), 'cli': Component( 'cli', 'Command-line tools', test_home, requires_root=False, supports_valgrind=True, ), 'exec': Component( 'exec', 'Local resource agent executor', test_home, requires_root=True, supports_valgrind=False, ), 'fencing': Component( 'fencing', 'Fencer', test_home, requires_root=True, supports_valgrind=False, ), 'scheduler': Component( 'scheduler', 'Action scheduler', test_home, requires_root=False, supports_valgrind=True, ), } if BuildOptions.REMOTE_ENABLED: components['pacemaker_remote'] = Component( 'pacemaker_remote', 'Resource agent executor in remote mode', test_home, requires_root=True, supports_valgrind=False, ) # Build up program description description = textwrap.dedent('''\ Run Pacemaker regression tests. Available components (default components are 'cli scheduler'): ''') for name, comp in sorted(components.items()): - description += '\n {:<20} {}'.format(name, comp.description) + description += f"\n {name:<20} {comp.description}" - description += ( - '\n {:<20} Synonym for "cli exec fencing scheduler"'.format('all') - ) + description += f'\n {"all":<20} Synonym for "cli exec fencing scheduler"' # Parse the arguments parser = argparse.ArgumentParser( description=description, formatter_class=argparse.RawDescriptionHelpFormatter, ) choices = sorted(components.keys()) + ['all', 'default'] parser.add_argument('-V', '--verbose', action='store_true', help='Increase test verbosity') parser.add_argument('-v', '--valgrind', action='store_true', help='Run test commands under valgrind') parser.add_argument('components', nargs='*', choices=choices, default='default', action=ComponentsArgAction, metavar='COMPONENT', help="One of the components to test, or 'all'") args = parser.parse_args() # Run the tests selected = [components[x] for x in args.components] rc = run_components(selected, args.verbose, args.valgrind) sys.exit(rc) if __name__ == '__main__': main() diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in index bd714faa70..450f0bd4d7 100644 --- a/cts/cts-scheduler.in +++ b/cts/cts-scheduler.in @@ -1,1731 +1,1718 @@ #!@PYTHON@ """ Regression tests for Pacemaker's scheduler """ -__copyright__ = "Copyright 2004-2024 the Pacemaker project contributors" +__copyright__ = "Copyright 2004-2025 the Pacemaker project contributors" __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" import io import os import re import sys import stat import shlex import shutil import argparse import subprocess import platform import tempfile # These imports allow running from a source checkout after running `make`. # Note that while this doesn't necessarily mean it will successfully run tests, # but being able to see --help output can be useful. if os.path.exists("@abs_top_srcdir@/python"): sys.path.insert(0, "@abs_top_srcdir@/python") if os.path.exists("@abs_top_builddir@/python") and "@abs_top_builddir@" != "@abs_top_srcdir@": sys.path.insert(0, "@abs_top_builddir@/python") from pacemaker.buildoptions import BuildOptions from pacemaker.exitstatus import ExitStatus DESC = """Regression tests for Pacemaker's scheduler""" class SchedulerTest: def __init__(self, name, desc, args=None): self.name = name self.desc = desc if args is None: self.args = [] else: self.args = args class SchedulerTestGroup: def __init__(self, tests): self.tests = tests # Each entry in TESTS is a group of tests, where each test consists of a # test base name, test description, and additional test arguments. # Test groups will be separated by newlines in output. TESTS = [ SchedulerTestGroup([ SchedulerTest("simple1", "Offline"), SchedulerTest("simple2", "Start"), SchedulerTest("simple3", "Start 2"), SchedulerTest("simple4", "Start Failed"), SchedulerTest("simple6", "Stop Start"), SchedulerTest("simple7", "Shutdown"), SchedulerTest("simple11", "Priority (ne)"), SchedulerTest("simple12", "Priority (eq)"), SchedulerTest("simple8", "Stickiness"), ]), SchedulerTestGroup([ SchedulerTest("group1", "Group"), SchedulerTest("group2", "Group + Native"), SchedulerTest("group3", "Group + Group"), SchedulerTest("group4", "Group + Native (nothing)"), SchedulerTest("group5", "Group + Native (move)"), SchedulerTest("group6", "Group + Group (move)"), SchedulerTest("group7", "Group colocation"), SchedulerTest("group13", "Group colocation (cant run)"), SchedulerTest("group8", "Group anti-colocation"), SchedulerTest("group9", "Group recovery"), SchedulerTest("group10", "Group partial recovery"), SchedulerTest("group11", "Group target_role"), SchedulerTest("group14", "Group stop (graph terminated)"), SchedulerTest("group15", "Negative group colocation"), SchedulerTest("bug-1573", "Partial stop of a group with two children"), SchedulerTest("bug-1718", "Mandatory group ordering - Stop group_FUN"), SchedulerTest("failed-sticky-group", "Move group on last member failure despite infinite stickiness"), SchedulerTest("failed-sticky-anticolocated-group", "Move group on last member failure despite infinite stickiness and optional anti-colocation"), SchedulerTest("bug-lf-2619", "Move group on clone failure"), SchedulerTest("group-fail", "Ensure stop order is preserved for partially active groups"), SchedulerTest("group-unmanaged", "No need to restart r115 because r114 is unmanaged"), SchedulerTest("group-unmanaged-stopped", "Make sure r115 is stopped when r114 fails"), SchedulerTest("partial-unmanaged-group", "New member in partially unmanaged group"), SchedulerTest("group-dependents", "Account for the location preferences of things colocated with a group"), SchedulerTest("group-stop-ordering", "Ensure blocked group member stop does not force other member stops"), SchedulerTest("colocate-unmanaged-group", "Respect mandatory colocations even if earlier group member is unmanaged"), SchedulerTest("coloc-with-inner-group-member", "Consider explicit colocations with inner group members"), SchedulerTest("banned-group-inner-constraints", "Group banned from current node, inner member constrained"), ]), SchedulerTestGroup([ SchedulerTest("rsc_dep1", "Must not"), SchedulerTest("rsc_dep3", "Must"), SchedulerTest("rsc_dep5", "Must not 3"), SchedulerTest("rsc_dep7", "Must 3"), SchedulerTest("rsc_dep10", "Must (but cant)"), SchedulerTest("rsc_dep2", "Must (running)"), SchedulerTest("rsc_dep8", "Must (running : alt)"), SchedulerTest("rsc_dep4", "Must (running + move)"), SchedulerTest("asymmetric", "Asymmetric - require explicit location constraints"), ]), SchedulerTestGroup([ SchedulerTest("orphan-0", "Orphan ignore"), SchedulerTest("orphan-1", "Orphan stop"), SchedulerTest("orphan-2", "Orphan stop, remove failcount"), ]), SchedulerTestGroup([ SchedulerTest("params-0", "Params: No change"), SchedulerTest("params-1", "Params: Changed"), SchedulerTest("params-2", "Params: Resource definition"), SchedulerTest("params-3", "Params: Restart instead of reload if start pending"), SchedulerTest("params-4", "Params: Reload"), SchedulerTest("params-5", "Params: Restart based on probe digest"), SchedulerTest("novell-251689", "Resource definition change + target_role=stopped"), SchedulerTest("bug-lf-2106", "Restart all anonymous clone instances after config change"), SchedulerTest("params-6", "Params: Detect reload in previously migrated resource"), SchedulerTest("nvpair-id-ref", "Support id-ref in nvpair with optional name"), SchedulerTest("not-reschedule-unneeded-monitor", "Do not reschedule unneeded monitors while resource definitions have changed"), SchedulerTest("reload-becomes-restart", "Cancel reload if restart becomes required"), SchedulerTest("restart-with-extra-op-params", "Restart if with extra operation parameters upon changes of any"), ]), SchedulerTestGroup([ SchedulerTest("target-0", "Target Role : baseline"), SchedulerTest("target-1", "Target Role : promoted"), SchedulerTest("target-2", "Target Role : invalid"), ]), SchedulerTestGroup([ SchedulerTest("base-score", "Set a node's default score for all nodes"), ]), SchedulerTestGroup([ SchedulerTest("date-1", "Dates", ["-t", "2005-020"]), SchedulerTest("date-2", "Date Spec - Pass", ["-t", "2005-020T12:30"]), SchedulerTest("date-3", "Date Spec - Fail", ["-t", "2005-020T11:30"]), SchedulerTest("origin", "Timing of recurring operations", ["-t", "2014-05-07 00:28:00"]), SchedulerTest("probe-0", "Probe (anon clone)"), SchedulerTest("probe-1", "Pending Probe"), SchedulerTest("probe-2", "Correctly re-probe cloned groups"), SchedulerTest("probe-3", "Probe (pending node)"), SchedulerTest("probe-4", "Probe (pending node + stopped resource)"), SchedulerTest("probe-pending-node", "Probe (pending node + unmanaged resource)"), SchedulerTest("failed-probe-primitive", "Maskable vs. unmaskable probe failures on primitive resources"), SchedulerTest("failed-probe-clone", "Maskable vs. unmaskable probe failures on cloned resources"), SchedulerTest("expired-failed-probe-primitive", "Maskable, expired probe failure on primitive resources"), SchedulerTest("standby", "Standby"), SchedulerTest("comments", "Comments"), ]), SchedulerTestGroup([ SchedulerTest("one-or-more-0", "Everything starts"), SchedulerTest("one-or-more-1", "Nothing starts because of A"), SchedulerTest("one-or-more-2", "D can start because of C"), SchedulerTest("one-or-more-3", "D cannot start because of B and C"), SchedulerTest("one-or-more-4", "D cannot start because of target-role"), SchedulerTest("one-or-more-5", "Start A and F even though C and D are stopped"), SchedulerTest("one-or-more-6", "Leave A running even though B is stopped"), SchedulerTest("one-or-more-7", "Leave A running even though C is stopped"), SchedulerTest("bug-5140-require-all-false", "Allow basegrp:0 to stop"), SchedulerTest("clone-require-all-1", "clone B starts node 3 and 4"), SchedulerTest("clone-require-all-2", "clone B remains stopped everywhere"), SchedulerTest("clone-require-all-3", "clone B stops everywhere because A stops everywhere"), SchedulerTest("clone-require-all-4", "clone B remains on node 3 and 4 with only one instance of A remaining"), SchedulerTest("clone-require-all-5", "clone B starts on node 1 3 and 4"), SchedulerTest("clone-require-all-6", "clone B remains active after shutting down instances of A"), SchedulerTest("clone-require-all-7", "clone A and B both start at the same time. all instances of A start before B"), SchedulerTest("clone-require-all-no-interleave-1", "C starts everywhere after A and B"), SchedulerTest("clone-require-all-no-interleave-2", "C starts on nodes 1, 2, and 4 with only one active instance of B"), SchedulerTest("clone-require-all-no-interleave-3", "C remains active when instance of B is stopped on one node and started on another"), SchedulerTest("one-or-more-unrunnable-instances", "Avoid dependencies on instances that won't ever be started"), ]), SchedulerTestGroup([ SchedulerTest("location-date-rules-1", "Use location constraints with ineffective date-based rules"), SchedulerTest("location-date-rules-2", "Use location constraints with effective date-based rules"), SchedulerTest("nvpair-date-rules-1", "Use nvpair blocks with a variety of date-based rules"), SchedulerTest("value-source", "Use location constraints with node attribute expressions using value-source"), SchedulerTest("rule-dbl-as-auto-number-match", "Floating-point rule values default to number comparison: match"), SchedulerTest("rule-dbl-as-auto-number-no-match", "Floating-point rule values default to number comparison: no match"), SchedulerTest("rule-dbl-as-integer-match", "Floating-point rule values set to integer comparison: match"), SchedulerTest("rule-dbl-as-integer-no-match", "Floating-point rule values set to integer comparison: no match"), SchedulerTest("rule-dbl-as-number-match", "Floating-point rule values set to number comparison: match"), SchedulerTest("rule-dbl-as-number-no-match", "Floating-point rule values set to number comparison: no match"), SchedulerTest("rule-dbl-parse-fail-default-str-match", "Floating-point rule values fail to parse, default to string " "comparison: match"), SchedulerTest("rule-dbl-parse-fail-default-str-no-match", "Floating-point rule values fail to parse, default to string " "comparison: no match"), SchedulerTest("rule-int-as-auto-integer-match", "Integer rule values default to integer comparison: match"), SchedulerTest("rule-int-as-auto-integer-no-match", "Integer rule values default to integer comparison: no match"), SchedulerTest("rule-int-as-integer-match", "Integer rule values set to integer comparison: match"), SchedulerTest("rule-int-as-integer-no-match", "Integer rule values set to integer comparison: no match"), SchedulerTest("rule-int-as-number-match", "Integer rule values set to number comparison: match"), SchedulerTest("rule-int-as-number-no-match", "Integer rule values set to number comparison: no match"), SchedulerTest("rule-int-parse-fail-default-str-match", "Integer rule values fail to parse, default to string " "comparison: match"), SchedulerTest("rule-int-parse-fail-default-str-no-match", "Integer rule values fail to parse, default to string " "comparison: no match"), ]), SchedulerTestGroup([ SchedulerTest("order1", "Order start 1"), SchedulerTest("order2", "Order start 2"), SchedulerTest("order3", "Order stop"), SchedulerTest("order4", "Order (multiple)"), SchedulerTest("order5", "Order (move)"), SchedulerTest("order6", "Order (move w/ restart)"), SchedulerTest("order7", "Order (mandatory)"), SchedulerTest("order-optional", "Order (score=0)"), SchedulerTest("order-required", "Order (score=INFINITY)"), SchedulerTest("bug-lf-2171", "Prevent group start when clone is stopped"), SchedulerTest("order-clone", "Clone ordering should be able to prevent startup of dependent clones"), SchedulerTest("order-sets", "Ordering for resource sets"), SchedulerTest("order-serialize", "Serialize resources without inhibiting migration"), SchedulerTest("order-serialize-set", "Serialize a set of resources without inhibiting migration"), SchedulerTest("clone-order-primitive", "Order clone start after a primitive"), SchedulerTest("clone-order-16instances", "Verify ordering of 16 cloned resources"), SchedulerTest("order-optional-keyword", "Order (optional keyword)"), SchedulerTest("order-mandatory", "Order (mandatory keyword)"), SchedulerTest("bug-lf-2493", "Don't imply colocation requirements when applying ordering constraints with clones"), SchedulerTest("ordered-set-basic-startup", "Constraint set with default order settings"), SchedulerTest("ordered-set-natural", "Allow natural set ordering"), SchedulerTest("order-wrong-kind", "Order (error)"), ]), SchedulerTestGroup([ SchedulerTest("coloc-loop", "Colocation - loop"), SchedulerTest("coloc-many-one", "Colocation - many-to-one"), SchedulerTest("coloc-list", "Colocation - many-to-one with list"), SchedulerTest("coloc-group", "Colocation - groups"), SchedulerTest("coloc-unpromoted-anti", "Anti-colocation with unpromoted shouldn't prevent promoted colocation"), SchedulerTest("coloc-attr", "Colocation based on node attributes"), SchedulerTest("coloc-negative-group", "Negative colocation with a group"), SchedulerTest("coloc-intra-set", "Intra-set colocation"), SchedulerTest("bug-lf-2435", "Colocation sets with a negative score"), SchedulerTest("coloc-clone-stays-active", "Ensure clones don't get stopped/demoted because a dependent must stop"), SchedulerTest("coloc_fp_logic", "Verify floating point calculations in colocation are working"), SchedulerTest("colo_promoted_w_native", "cl#5070 - Verify promotion order is affected when colocating promoted with primitive"), SchedulerTest("colo_unpromoted_w_native", "cl#5070 - Verify promotion order is affected when colocating unpromoted with primitive"), SchedulerTest("anti-colocation-order", "cl#5187 - Prevent resources in an anti-colocation from even temporarily running on a same node"), SchedulerTest("anti-colocation-promoted", "Organize order of actions for promoted resources in anti-colocations"), SchedulerTest("anti-colocation-unpromoted", "Organize order of actions for unpromoted resources in anti-colocations"), SchedulerTest("group-anticolocation", "Group with failed last member anti-colocated with another group"), SchedulerTest("group-anticolocation-2", "Group with failed last member anti-colocated with another sticky group"), SchedulerTest("group-anticolocation-3", "Group with failed last member mandatorily anti-colocated with another group"), SchedulerTest("group-anticolocation-4", "Group with failed last member anti-colocated without influence with another group"), SchedulerTest("group-anticolocation-5", "Group with failed last member anti-colocated with another group (third node allowed)"), SchedulerTest("group-colocation-failure", "Group with sole member failed, colocated with another group"), SchedulerTest("enforce-colo1", "Always enforce B with A INFINITY"), SchedulerTest("complex_enforce_colo", "Always enforce B with A INFINITY. (make sure heat-engine stops)"), SchedulerTest("coloc-dependee-should-stay", "Stickiness outweighs group colocation"), SchedulerTest("coloc-dependee-should-move", "Group colocation outweighs stickiness"), SchedulerTest("colocation-influence", "Respect colocation influence"), SchedulerTest("colocation-priority-group", "Apply group colocations in order of primary priority"), SchedulerTest("colocation-vs-stickiness", "Group stickiness outweighs anti-colocation score"), SchedulerTest("promoted-with-blocked", "Promoted role colocated with a resource with blocked start"), SchedulerTest("primitive-with-group-with-clone", "Consider group dependent when colocating with clone"), SchedulerTest("primitive-with-group-with-promoted", "Consider group dependent when colocating with promoted role"), SchedulerTest("primitive-with-unrunnable-group", "Block primitive colocated with group that can't start"), SchedulerTest("coloc-cloned-group-promoted-dependent1", "Cloned group promoted role with primitive (mandatory)"), SchedulerTest("coloc-cloned-group-promoted-dependent2", "Cloned group promoted role with primitive (optional)"), SchedulerTest("coloc-optional-promoted-dependent-moves-1", "Colocation score less than promotion score " + "difference: move"), SchedulerTest("coloc-optional-promoted-dependent-moves-2", "Colocation score greater than promotion score " + "difference: move"), SchedulerTest("coloc-optional-promoted-dependent-stays-1", "Colocation score greater than promotion score " + "difference: stay"), SchedulerTest("coloc-optional-promoted-dependent-stays-2", "Colocation score less than promotion score " + "difference: stay"), ]), SchedulerTestGroup([ SchedulerTest("rsc-sets-seq-true", "Resource Sets - sequential=false"), SchedulerTest("rsc-sets-seq-false", "Resource Sets - sequential=true"), SchedulerTest("rsc-sets-clone", "Resource Sets - Clone"), SchedulerTest("rsc-sets-promoted", "Resource Sets - Promoted"), SchedulerTest("rsc-sets-clone-1", "Resource Sets - Clone (lf#2404)"), ]), SchedulerTestGroup([ SchedulerTest("attrs1", "string: eq (and)"), SchedulerTest("attrs2", "string: lt / gt (and)"), SchedulerTest("attrs3", "string: ne (or)"), SchedulerTest("attrs4", "string: exists"), SchedulerTest("attrs5", "string: not_exists"), SchedulerTest("attrs6", "is_dc: true"), SchedulerTest("attrs7", "is_dc: false"), SchedulerTest("attrs8", "score_attribute"), SchedulerTest("per-node-attrs", "Per node resource parameters"), ]), SchedulerTestGroup([ SchedulerTest("mon-rsc-1", "Schedule Monitor - start"), SchedulerTest("mon-rsc-2", "Schedule Monitor - move"), SchedulerTest("mon-rsc-3", "Schedule Monitor - pending start"), SchedulerTest("mon-rsc-4", "Schedule Monitor - move/pending start"), ]), SchedulerTestGroup([ SchedulerTest("rec-rsc-0", "Resource Recover - no start"), SchedulerTest("rec-rsc-1", "Resource Recover - start"), SchedulerTest("rec-rsc-2", "Resource Recover - monitor"), SchedulerTest("rec-rsc-3", "Resource Recover - stop - ignore"), SchedulerTest("rec-rsc-4", "Resource Recover - stop - block"), SchedulerTest("rec-rsc-5", "Resource Recover - stop - fence"), SchedulerTest("rec-rsc-6", "Resource Recover - multiple - restart"), SchedulerTest("rec-rsc-7", "Resource Recover - multiple - stop"), SchedulerTest("rec-rsc-8", "Resource Recover - multiple - block"), SchedulerTest("rec-rsc-9", "Resource Recover - group/group"), SchedulerTest("stop-unexpected", "Recover multiply active group with stop_unexpected"), SchedulerTest("stop-unexpected-2", "Resource multiply active primitve with stop_unexpected"), SchedulerTest("monitor-recovery", "on-fail=block + resource recovery detected by recurring monitor"), SchedulerTest("stop-failure-no-quorum", "Stop failure without quorum"), SchedulerTest("stop-failure-no-fencing", "Stop failure without fencing available"), SchedulerTest("stop-failure-with-fencing", "Stop failure with fencing available"), SchedulerTest("multiple-active-block-group", "Support of multiple-active=block for resource groups"), SchedulerTest("multiple-monitor-one-failed", "Consider resource failed if any of the configured monitor operations failed"), ]), SchedulerTestGroup([ SchedulerTest("quorum-1", "No quorum - ignore"), SchedulerTest("quorum-2", "No quorum - freeze"), SchedulerTest("quorum-3", "No quorum - stop"), SchedulerTest("quorum-4", "No quorum - start anyway"), SchedulerTest("quorum-5", "No quorum - start anyway (group)"), SchedulerTest("quorum-6", "No quorum - start anyway (clone)"), SchedulerTest("bug-cl-5212", "No promotion with no-quorum-policy=freeze"), SchedulerTest("suicide-needed-inquorate", "no-quorum-policy=suicide: suicide necessary"), SchedulerTest("suicide-not-needed-initial-quorum", "no-quorum-policy=suicide: suicide not necessary at initial quorum"), SchedulerTest("suicide-not-needed-never-quorate", "no-quorum-policy=suicide: suicide not necessary if never quorate"), SchedulerTest("suicide-not-needed-quorate", "no-quorum-policy=suicide: suicide necessary if quorate"), ]), SchedulerTestGroup([ SchedulerTest("rec-node-1", "Node Recover - Startup - no fence"), SchedulerTest("rec-node-2", "Node Recover - Startup - fence"), SchedulerTest("rec-node-3", "Node Recover - HA down - no fence"), SchedulerTest("rec-node-4", "Node Recover - HA down - fence"), SchedulerTest("rec-node-5", "Node Recover - CRM down - no fence"), SchedulerTest("rec-node-6", "Node Recover - CRM down - fence"), SchedulerTest("rec-node-7", "Node Recover - no quorum - ignore"), SchedulerTest("rec-node-8", "Node Recover - no quorum - freeze"), SchedulerTest("rec-node-9", "Node Recover - no quorum - stop"), SchedulerTest("rec-node-10", "Node Recover - no quorum - stop w/fence"), SchedulerTest("rec-node-11", "Node Recover - CRM down w/ group - fence"), SchedulerTest("rec-node-12", "Node Recover - nothing active - fence"), SchedulerTest("rec-node-13", "Node Recover - failed resource + shutdown - fence"), SchedulerTest("rec-node-15", "Node Recover - unknown lrm section"), SchedulerTest("rec-node-14", "Serialize all stonith's"), ]), SchedulerTestGroup([ SchedulerTest("multi1", "Multiple Active (stop/start)"), ]), SchedulerTestGroup([ SchedulerTest("migrate-begin", "Normal migration"), SchedulerTest("migrate-success", "Completed migration"), SchedulerTest("migrate-partial-1", "Completed migration, missing stop on source"), SchedulerTest("migrate-partial-2", "Successful migrate_to only"), SchedulerTest("migrate-partial-3", "Successful migrate_to only, target down"), SchedulerTest("migrate-partial-4", "Migrate from the correct host after migrate_to+migrate_from"), SchedulerTest("bug-5186-partial-migrate", "Handle partial migration when src node loses membership"), SchedulerTest("migrate-fail-2", "Failed migrate_from"), SchedulerTest("migrate-fail-3", "Failed migrate_from + stop on source"), SchedulerTest("migrate-fail-4", "Failed migrate_from + stop on target - ideally we wouldn't need to re-stop on target"), SchedulerTest("migrate-fail-5", "Failed migrate_from + stop on source and target"), SchedulerTest("migrate-fail-6", "Failed migrate_to"), SchedulerTest("migrate-fail-7", "Failed migrate_to + stop on source"), SchedulerTest("migrate-fail-8", "Failed migrate_to + stop on target - ideally we wouldn't need to re-stop on target"), SchedulerTest("migrate-fail-9", "Failed migrate_to + stop on source and target"), SchedulerTest("migration-ping-pong", "Old migrate_to failure + successful migrate_from on same node"), SchedulerTest("migrate-stop", "Migration in a stopping stack"), SchedulerTest("migrate-start", "Migration in a starting stack"), SchedulerTest("migrate-stop_start", "Migration in a restarting stack"), SchedulerTest("migrate-stop-complex", "Migration in a complex stopping stack"), SchedulerTest("migrate-start-complex", "Migration in a complex starting stack"), SchedulerTest("migrate-stop-start-complex", "Migration in a complex moving stack"), SchedulerTest("migrate-shutdown", "Order the post-migration 'stop' before node shutdown"), SchedulerTest("migrate-1", "Migrate (migrate)"), SchedulerTest("migrate-2", "Migrate (stable)"), SchedulerTest("migrate-3", "Migrate (failed migrate_to)"), SchedulerTest("migrate-4", "Migrate (failed migrate_from)"), SchedulerTest("novell-252693", "Migration in a stopping stack"), SchedulerTest("novell-252693-2", "Migration in a starting stack"), SchedulerTest("novell-252693-3", "Non-Migration in a starting and stopping stack"), SchedulerTest("bug-1820", "Migration in a group"), SchedulerTest("bug-1820-1", "Non-migration in a group"), SchedulerTest("migrate-5", "Primitive migration with a clone"), SchedulerTest("migrate-fencing", "Migration after Fencing"), SchedulerTest("migrate-both-vms", "Migrate two VMs that have no colocation"), SchedulerTest("migration-behind-migrating-remote", "Migrate resource behind migrating remote connection"), SchedulerTest("1-a-then-bm-move-b", "Advanced migrate logic. A then B. migrate B"), SchedulerTest("2-am-then-b-move-a", "Advanced migrate logic, A then B, migrate A without stopping B"), SchedulerTest("3-am-then-bm-both-migrate", "Advanced migrate logic. A then B. migrate both"), SchedulerTest("4-am-then-bm-b-not-migratable", "Advanced migrate logic, A then B, B not migratable"), SchedulerTest("5-am-then-bm-a-not-migratable", "Advanced migrate logic. A then B. move both, a not migratable"), SchedulerTest("6-migrate-group", "Advanced migrate logic, migrate a group"), SchedulerTest("7-migrate-group-one-unmigratable", "Advanced migrate logic, migrate group mixed with allow-migrate true/false"), SchedulerTest("8-am-then-bm-a-migrating-b-stopping", "Advanced migrate logic, A then B, A migrating, B stopping"), SchedulerTest("9-am-then-bm-b-migrating-a-stopping", "Advanced migrate logic, A then B, B migrate, A stopping"), SchedulerTest("10-a-then-bm-b-move-a-clone", "Advanced migrate logic, A clone then B, migrate B while stopping A"), SchedulerTest("11-a-then-bm-b-move-a-clone-starting", "Advanced migrate logic, A clone then B, B moving while A is start/stopping"), SchedulerTest("a-promote-then-b-migrate", "A promote then B start. migrate B"), SchedulerTest("a-demote-then-b-migrate", "A demote then B stop. migrate B"), SchedulerTest("probe-target-of-failed-migrate_to-1", "Failed migrate_to, target rejoins"), SchedulerTest("probe-target-of-failed-migrate_to-2", "Failed migrate_to, target rejoined and probed"), SchedulerTest("partial-live-migration-multiple-active", "Prevent running on multiple nodes due to partial live migration"), SchedulerTest("migration-intermediary-cleaned", "Probe live-migration intermediary with no history"), SchedulerTest("bug-lf-2422", "Dependency on partially active group - stop ocfs:*"), ]), SchedulerTestGroup([ SchedulerTest("clone-anon-probe-1", "Probe the correct (anonymous) clone instance for each node"), SchedulerTest("clone-anon-probe-2", "Avoid needless re-probing of anonymous clones"), SchedulerTest("clone-anon-failcount", "Merge failcounts for anonymous clones"), SchedulerTest("force-anon-clone-max", "Update clone-max properly when forcing a clone to be anonymous"), SchedulerTest("anon-instance-pending", "Assign anonymous clone instance numbers properly when action pending"), SchedulerTest("inc0", "Incarnation start"), SchedulerTest("inc1", "Incarnation start order"), SchedulerTest("inc2", "Incarnation silent restart, stop, move"), SchedulerTest("inc3", "Inter-incarnation ordering, silent restart, stop, move"), SchedulerTest("inc4", "Inter-incarnation ordering, silent restart, stop, move (ordered)"), SchedulerTest("inc5", "Inter-incarnation ordering, silent restart, stop, move (restart 1)"), SchedulerTest("inc6", "Inter-incarnation ordering, silent restart, stop, move (restart 2)"), SchedulerTest("inc7", "Clone colocation"), SchedulerTest("inc8", "Clone anti-colocation"), SchedulerTest("inc9", "Non-unique clone"), SchedulerTest("inc10", "Non-unique clone (stop)"), SchedulerTest("inc11", "Primitive colocation with clones"), SchedulerTest("inc12", "Clone shutdown"), SchedulerTest("cloned-group", "Make sure only the correct number of cloned groups are started"), SchedulerTest("cloned-group-stop", "Ensure stopping qpidd also stops glance and cinder"), SchedulerTest("clone-no-shuffle", "Don't prioritize allocation of instances that must be moved"), SchedulerTest("clone-recover-no-shuffle-1", "Don't shuffle instances when starting a new primitive instance"), SchedulerTest("clone-recover-no-shuffle-2", "Don't shuffle instances when starting a new group instance"), SchedulerTest("clone-recover-no-shuffle-3", "Don't shuffle instances when starting a new bundle instance"), SchedulerTest("clone-recover-no-shuffle-4", "Don't shuffle instances when starting a new primitive instance with " "location preference "), SchedulerTest("clone-recover-no-shuffle-5", "Don't shuffle instances when starting a new group instance with " "location preference"), SchedulerTest("clone-recover-no-shuffle-6", "Don't shuffle instances when starting a new bundle instance with " "location preference"), SchedulerTest("clone-recover-no-shuffle-7", "Don't shuffle instances when starting a new primitive instance that " "will be promoted"), SchedulerTest("clone-recover-no-shuffle-8", "Don't shuffle instances when starting a new group instance that " "will be promoted "), SchedulerTest("clone-recover-no-shuffle-9", "Don't shuffle instances when starting a new bundle instance that " "will be promoted "), SchedulerTest("clone-recover-no-shuffle-10", "Don't shuffle instances when starting a new primitive instance that " "won't be promoted"), SchedulerTest("clone-recover-no-shuffle-11", "Don't shuffle instances when starting a new group instance that " "won't be promoted "), SchedulerTest("clone-recover-no-shuffle-12", "Don't shuffle instances when starting a new bundle instance that " "won't be promoted "), SchedulerTest("clone-max-zero", "Orphan processing with clone-max=0"), SchedulerTest("clone-anon-dup", "Bug LF#2087 - Correctly parse the state of anonymous clones that are active more than once per node"), SchedulerTest("bug-lf-2160", "Don't shuffle clones due to colocation"), SchedulerTest("bug-lf-2213", "clone-node-max enforcement for cloned groups"), SchedulerTest("bug-lf-2153", "Clone ordering constraints"), SchedulerTest("bug-lf-2361", "Ensure clones observe mandatory ordering constraints if the LHS is unrunnable"), SchedulerTest("bug-lf-2317", "Avoid needless restart of primitive depending on a clone"), SchedulerTest("bug-lf-2453", "Enforce mandatory clone ordering without colocation"), SchedulerTest("bug-lf-2508", "Correctly reconstruct the status of anonymous cloned groups"), SchedulerTest("bug-lf-2544", "Balanced clone placement"), SchedulerTest("bug-lf-2445", "Redistribute clones with node-max > 1 and stickiness = 0"), SchedulerTest("bug-lf-2574", "Avoid clone shuffle"), SchedulerTest("bug-lf-2581", "Avoid group restart due to unrelated clone (re)start"), SchedulerTest("bug-cl-5168", "Don't shuffle clones"), SchedulerTest("bug-cl-5170", "Prevent clone from starting with on-fail=block"), SchedulerTest("clone-fail-block-colocation", "Move colocated group when failed clone has on-fail=block"), SchedulerTest("clone-interleave-1", "Clone-3 cannot start on pcmk-1 due to interleaved ordering (no colocation)"), SchedulerTest("clone-interleave-2", "Clone-3 must stop on pcmk-1 due to interleaved ordering (no colocation)"), SchedulerTest("clone-interleave-3", "Clone-3 must be recovered on pcmk-1 due to interleaved ordering (no colocation)"), SchedulerTest("rebalance-unique-clones", "Rebalance unique clone instances with no stickiness"), SchedulerTest("clone-requires-quorum-recovery", "Clone with requires=quorum on failed node needing recovery"), SchedulerTest("clone-requires-quorum", "Clone with requires=quorum with presumed-inactive instance on failed node"), ]), SchedulerTestGroup([ SchedulerTest("cloned_start_one", "order first clone then clone... first clone_min=2"), SchedulerTest("cloned_start_two", "order first clone then clone... first clone_min=2"), SchedulerTest("cloned_stop_one", "order first clone then clone... first clone_min=2"), SchedulerTest("cloned_stop_two", "order first clone then clone... first clone_min=2"), SchedulerTest("clone_min_interleave_start_one", "order first clone then clone... first clone_min=2 and then has interleave=true"), SchedulerTest("clone_min_interleave_start_two", "order first clone then clone... first clone_min=2 and then has interleave=true"), SchedulerTest("clone_min_interleave_stop_one", "order first clone then clone... first clone_min=2 and then has interleave=true"), SchedulerTest("clone_min_interleave_stop_two", "order first clone then clone... first clone_min=2 and then has interleave=true"), SchedulerTest("clone_min_start_one", "order first clone then primitive... first clone_min=2"), SchedulerTest("clone_min_start_two", "order first clone then primitive... first clone_min=2"), SchedulerTest("clone_min_stop_all", "order first clone then primitive... first clone_min=2"), SchedulerTest("clone_min_stop_one", "order first clone then primitive... first clone_min=2"), SchedulerTest("clone_min_stop_two", "order first clone then primitive... first clone_min=2"), ]), SchedulerTestGroup([ SchedulerTest("unfence-startup", "Clean unfencing"), SchedulerTest("unfence-definition", "Unfencing when the agent changes"), SchedulerTest("unfence-parameters", "Unfencing when the agent parameters changes"), SchedulerTest("unfence-device", "Unfencing when a cluster has only fence devices"), ]), SchedulerTestGroup([ SchedulerTest("promoted-0", "Stopped -> Unpromoted"), SchedulerTest("promoted-1", "Stopped -> Promote"), SchedulerTest("promoted-2", "Stopped -> Promote : notify"), SchedulerTest("promoted-3", "Stopped -> Promote : promoted location"), SchedulerTest("promoted-4", "Started -> Promote : promoted location"), SchedulerTest("promoted-5", "Promoted -> Promoted"), SchedulerTest("promoted-6", "Promoted -> Promoted (2)"), SchedulerTest("promoted-7", "Promoted -> Fenced"), SchedulerTest("promoted-8", "Promoted -> Fenced -> Moved"), SchedulerTest("promoted-9", "Stopped + Promotable + No quorum"), SchedulerTest("promoted-10", "Stopped -> Promotable : notify with monitor"), SchedulerTest("promoted-11", "Stopped -> Promote : colocation"), SchedulerTest("novell-239082", "Demote/Promote ordering"), SchedulerTest("novell-239087", "Stable promoted placement"), SchedulerTest("promoted-12", "Promotion based solely on rsc_location constraints"), SchedulerTest("promoted-13", "Include preferences of colocated resources when placing promoted"), SchedulerTest("promoted-demote", "Ordering when actions depends on demoting an unpromoted resource"), SchedulerTest("promoted-ordering", "Prevent resources from starting that need a promoted"), SchedulerTest("bug-1765", "Verify promoted-with-promoted colocation does not stop unpromoted instances"), SchedulerTest("promoted-group", "Promotion of cloned groups"), SchedulerTest("bug-lf-1852", "Don't shuffle promotable instances unnecessarily"), SchedulerTest("promoted-failed-demote", "Don't retry failed demote actions"), SchedulerTest("promoted-failed-demote-2", "Don't retry failed demote actions (notify=false)"), SchedulerTest("promoted-depend", "Ensure resources that depend on promoted instance don't get allocated until that does"), SchedulerTest("promoted-reattach", "Re-attach to a running promoted"), SchedulerTest("promoted-allow-start", "Don't include promoted score if it would prevent allocation"), SchedulerTest("promoted-colocation", "Allow promoted instances placemaker to be influenced by colocation constraints"), SchedulerTest("promoted-pseudo", "Make sure promote/demote pseudo actions are created correctly"), SchedulerTest("promoted-role", "Prevent target-role from promoting more than promoted-max instances"), SchedulerTest("bug-lf-2358", "Anti-colocation of promoted instances"), SchedulerTest("promoted-promotion-constraint", "Mandatory promoted colocation constraints"), SchedulerTest("unmanaged-promoted", "Ensure role is preserved for unmanaged resources"), SchedulerTest("promoted-unmanaged-monitor", "Start correct monitor for unmanaged promoted instances"), SchedulerTest("promoted-demote-2", "Demote does not clear past failure"), SchedulerTest("promoted-move", "Move promoted based on failure of colocated group"), SchedulerTest("promoted-probed-score", "Observe the promotion score of probed resources"), SchedulerTest("colocation_constraint_stops_promoted", "cl#5054 - Ensure promoted is demoted when stopped by colocation constraint"), SchedulerTest("colocation_constraint_stops_unpromoted", "cl#5054 - Ensure unpromoted is not demoted when stopped by colocation constraint"), SchedulerTest("order_constraint_stops_promoted", "cl#5054 - Ensure promoted is demoted when stopped by order constraint"), SchedulerTest("order_constraint_stops_unpromoted", "cl#5054 - Ensure unpromoted is not demoted when stopped by order constraint"), SchedulerTest("promoted_monitor_restart", "cl#5072 - Ensure promoted monitor operation will start after promotion"), SchedulerTest("bug-rh-880249", "Handle replacement of an m/s resource with a primitive"), SchedulerTest("bug-5143-ms-shuffle", "Prevent promoted instance shuffling due to promotion score"), SchedulerTest("promoted-demote-block", "Block promotion if demote fails with on-fail=block"), SchedulerTest("promoted-dependent-ban", "Don't stop instances from being active because a dependent is banned from that host"), SchedulerTest("promoted-stop", "Stop instances due to location constraint with role=Started"), SchedulerTest("promoted-partially-demoted-group", "Allow partially demoted group to finish demoting"), SchedulerTest("bug-cl-5213", "Ensure role colocation with -INFINITY is enforced"), SchedulerTest("bug-cl-5219", "Allow unrelated resources with a common colocation target to remain promoted"), SchedulerTest("promoted-asymmetrical-order", "Fix the behaviors of multi-state resources with asymmetrical ordering"), SchedulerTest("promoted-notify", "Promotion with notifications"), SchedulerTest("promoted-score-startup", "Use permanent promoted scores without LRM history"), SchedulerTest("failed-demote-recovery", "Recover resource in unpromoted role after demote fails"), SchedulerTest("failed-demote-recovery-promoted", "Recover resource in promoted role after demote fails"), SchedulerTest("on_fail_demote1", "Recovery with on-fail=\"demote\" on healthy cluster, remote, guest, and bundle nodes"), SchedulerTest("on_fail_demote2", "Recovery with on-fail=\"demote\" with promotion on different node"), SchedulerTest("on_fail_demote3", "Recovery with on-fail=\"demote\" with no promotion"), SchedulerTest("on_fail_demote4", "Recovery with on-fail=\"demote\" on failed cluster, remote, guest, and bundle nodes"), SchedulerTest("no_quorum_demote", "Promotable demotion and primitive stop with no-quorum-policy=\"demote\""), SchedulerTest("no-promote-on-unrunnable-guest", "Don't select bundle instance for promotion when container can't run"), SchedulerTest("leftover-pending-monitor", "Prevent a leftover pending monitor from causing unexpected stop of other instances"), ]), SchedulerTestGroup([ SchedulerTest("history-1", "Correctly parse stateful-1 resource state"), ]), SchedulerTestGroup([ SchedulerTest("managed-0", "Managed (reference)"), SchedulerTest("managed-1", "Not managed - down"), SchedulerTest("managed-2", "Not managed - up"), SchedulerTest("bug-5028", "Shutdown should block if anything depends on an unmanaged resource"), SchedulerTest("bug-5028-detach", "Ensure detach still works"), SchedulerTest("bug-5028-bottom", "Ensure shutdown still blocks if the blocked resource is at the bottom of the stack"), SchedulerTest("unmanaged-stop-1", "cl#5155 - Block the stop of resources if any depending resource is unmanaged"), SchedulerTest("unmanaged-stop-2", "cl#5155 - Block the stop of resources if the first resource in a mandatory stop order is unmanaged"), SchedulerTest("unmanaged-stop-3", "cl#5155 - Block the stop of resources if any depending resource in a group is unmanaged"), SchedulerTest("unmanaged-stop-4", "cl#5155 - Block the stop of resources if any depending resource in the middle of a group is unmanaged"), SchedulerTest("unmanaged-block-restart", "Block restart of resources if any dependent resource in a group is unmanaged"), ]), SchedulerTestGroup([ SchedulerTest("interleave-0", "Interleave (reference)"), SchedulerTest("interleave-1", "coloc - not interleaved"), SchedulerTest("interleave-2", "coloc - interleaved"), SchedulerTest("interleave-3", "coloc - interleaved (2)"), SchedulerTest("interleave-pseudo-stop", "Interleaved clone during stonith"), SchedulerTest("interleave-stop", "Interleaved clone during stop"), SchedulerTest("interleave-restart", "Interleaved clone during dependency restart"), ]), SchedulerTestGroup([ SchedulerTest("notify-0", "Notify reference"), SchedulerTest("notify-1", "Notify simple"), SchedulerTest("notify-2", "Notify simple, confirm"), SchedulerTest("notify-3", "Notify move, confirm"), SchedulerTest("novell-239079", "Notification priority"), SchedulerTest("notifs-for-unrunnable", "Don't schedule notifications for an unrunnable action"), SchedulerTest("route-remote-notify", "Route remote notify actions through correct cluster node"), SchedulerTest("notify-behind-stopping-remote", "Don't schedule notifications behind stopped remote"), ]), SchedulerTestGroup([ SchedulerTest("594", "OSDL #594 - Unrunnable actions scheduled in transition"), SchedulerTest("662", "OSDL #662 - Two resources start on one node when incarnation_node_max = 1"), SchedulerTest("696", "OSDL #696 - CRM starts stonith RA without monitor"), SchedulerTest("726", "OSDL #726 - Attempting to schedule rsc_posic041_monitor_5000 _after_ a stop"), SchedulerTest("735", "OSDL #735 - Correctly detect that rsc_hadev1 is stopped on hadev3"), SchedulerTest("764", "OSDL #764 - Missing monitor op for DoFencing:child_DoFencing:1"), SchedulerTest("797", "OSDL #797 - Assert triggered: task_id_i > max_call_id"), SchedulerTest("829", "OSDL #829"), SchedulerTest("994", "OSDL #994 - Stopping the last resource in a resource group causes the entire group to be restarted"), SchedulerTest("994-2", "OSDL #994 - with a dependent resource"), SchedulerTest("1360", "OSDL #1360 - Clone stickiness"), SchedulerTest("1484", "OSDL #1484 - on_fail=stop"), SchedulerTest("1494", "OSDL #1494 - Clone stability"), SchedulerTest("unrunnable-1", "Unrunnable"), SchedulerTest("unrunnable-2", "Unrunnable 2"), SchedulerTest("stonith-0", "Stonith loop - 1"), SchedulerTest("stonith-1", "Stonith loop - 2"), SchedulerTest("stonith-2", "Stonith loop - 3"), SchedulerTest("stonith-3", "Stonith startup"), SchedulerTest("stonith-4", "Stonith node state"), SchedulerTest("dc-fence-ordering", "DC needs fencing while other nodes are shutting down"), SchedulerTest("bug-1572-1", "Recovery of groups depending on promotable role"), SchedulerTest("bug-1572-2", "Recovery of groups depending on promotable role when promoted is not re-promoted"), SchedulerTest("bug-1685", "Depends-on-promoted ordering"), SchedulerTest("bug-1822", "Don't promote partially active groups"), SchedulerTest("bug-pm-11", "New resource added to a m/s group"), SchedulerTest("bug-pm-12", "Recover only the failed portion of a cloned group"), SchedulerTest("bug-n-387749", "Don't shuffle clone instances"), SchedulerTest("bug-n-385265", "Don't ignore the failure stickiness of group children - resource_idvscommon should stay stopped"), SchedulerTest("bug-n-385265-2", "Ensure groups are migrated instead of remaining partially active on the current node"), SchedulerTest("bug-lf-1920", "Correctly handle probes that find active resources"), SchedulerTest("bnc-515172", "Location constraint with multiple expressions"), SchedulerTest("colocate-primitive-with-clone", "Optional colocation with a clone"), SchedulerTest("use-after-free-merge", "Use-after-free in native_merge_weights"), SchedulerTest("bug-lf-2551", "STONITH ordering for stop"), SchedulerTest("bug-lf-2606", "Stonith implies demote"), SchedulerTest("bug-lf-2474", "Ensure resource op timeout takes precedence over op_defaults"), SchedulerTest("bug-suse-707150", "Prevent vm-01 from starting due to colocation/ordering"), SchedulerTest("bug-5014-A-start-B-start", "Verify when A starts B starts using symmetrical=false"), SchedulerTest("bug-5014-A-stop-B-started", "Verify when A stops B does not stop if it has already started using symmetric=false"), SchedulerTest("bug-5014-A-stopped-B-stopped", "Verify when A is stopped and B has not started, B does not start before A using symmetric=false"), SchedulerTest("bug-5014-CthenAthenB-C-stopped", "Verify when C then A is symmetrical=true, A then B is symmetric=false, and C is stopped that nothing starts"), SchedulerTest("bug-5014-CLONE-A-start-B-start", "Verify when A starts B starts using clone resources with symmetric=false"), SchedulerTest("bug-5014-CLONE-A-stop-B-started", "Verify when A stops B does not stop if it has already started using clone resources with symmetric=false"), SchedulerTest("bug-5014-GROUP-A-start-B-start", "Verify when A starts B starts when using group resources with symmetric=false"), SchedulerTest("bug-5014-GROUP-A-stopped-B-started", "Verify when A stops B does not stop if it has already started using group resources with symmetric=false"), SchedulerTest("bug-5014-GROUP-A-stopped-B-stopped", "Verify when A is stopped and B has not started, B does not start before A using group resources with symmetric=false"), SchedulerTest("bug-5014-ordered-set-symmetrical-false", "Verify ordered sets work with symmetrical=false"), SchedulerTest("bug-5014-ordered-set-symmetrical-true", "Verify ordered sets work with symmetrical=true"), SchedulerTest("clbz5007-promotable-colocation", "Verify use of colocation scores other than INFINITY and -INFINITY work on multi-state resources"), SchedulerTest("bug-5038", "Prevent restart of anonymous clones when clone-max decreases"), SchedulerTest("bug-5025-1", "Automatically clean up failcount after resource config change with reload"), SchedulerTest("bug-5025-2", "Make sure clear failcount action isn't set when config does not change"), SchedulerTest("bug-5025-3", "Automatically clean up failcount after resource config change with restart"), SchedulerTest("bug-5025-4", "Clear failcount when last failure is a start op and rsc attributes changed"), SchedulerTest("failcount", "Ensure failcounts are correctly expired"), SchedulerTest("failcount-block", "Ensure failcounts are not expired when on-fail=block is present"), SchedulerTest("per-op-failcount", "Ensure per-operation failcount is handled and not passed to fence agent"), SchedulerTest("on-fail-ignore", "Ensure on-fail=ignore works even beyond migration-threshold"), SchedulerTest("monitor-onfail-restart", "bug-5058 - Monitor failure with on-fail set to restart"), SchedulerTest("monitor-onfail-stop", "bug-5058 - Monitor failure wiht on-fail set to stop"), SchedulerTest("bug-5059", "No need to restart p_stateful1:*"), SchedulerTest("bug-5069-op-enabled", "Test on-fail=ignore with failure when monitor is enabled"), SchedulerTest("bug-5069-op-disabled", "Test on-fail-ignore with failure when monitor is disabled"), SchedulerTest("obsolete-lrm-resource", "cl#5115 - Do not use obsolete lrm_resource sections"), SchedulerTest("expire-non-blocked-failure", "Ignore failure-timeout only if the failed operation has on-fail=block"), SchedulerTest("asymmetrical-order-move", "Respect asymmetrical ordering when trying to move resources"), SchedulerTest("asymmetrical-order-restart", "Respect asymmetrical ordering when restarting dependent resource"), SchedulerTest("start-then-stop-with-unfence", "Avoid graph loop with start-then-stop constraint plus unfencing"), SchedulerTest("order-expired-failure", "Order failcount cleanup after remote fencing"), SchedulerTest("expired-stop-1", "Expired stop failure should not block resource"), SchedulerTest("ignore_stonith_rsc_order1", "cl#5056- Ignore order constraint between stonith and non-stonith rsc"), SchedulerTest("ignore_stonith_rsc_order2", "cl#5056- Ignore order constraint with group rsc containing mixed stonith and non-stonith"), SchedulerTest("ignore_stonith_rsc_order3", "cl#5056- Ignore order constraint, stonith clone and mixed group"), SchedulerTest("ignore_stonith_rsc_order4", "cl#5056- Ignore order constraint, stonith clone and clone with nested mixed group"), SchedulerTest("honor_stonith_rsc_order1", "cl#5056- Honor order constraint, stonith clone and pure stonith group(single rsc)"), SchedulerTest("honor_stonith_rsc_order2", "cl#5056- Honor order constraint, stonith clone and pure stonith group(multiple rsc)"), SchedulerTest("honor_stonith_rsc_order3", "cl#5056- Honor order constraint, stonith clones with nested pure stonith group"), SchedulerTest("honor_stonith_rsc_order4", "cl#5056- Honor order constraint, between two native stonith rscs"), SchedulerTest("multiply-active-stonith", "Multiply active stonith"), SchedulerTest("probe-timeout", "cl#5099 - Default probe timeout"), SchedulerTest("order-first-probes", "cl#5301 - respect order constraints when relevant resources are being probed"), SchedulerTest("concurrent-fencing", "Allow performing fencing operations in parallel"), SchedulerTest("priority-fencing-delay", "Delay fencing targeting the more significant node"), SchedulerTest("pending-node-no-uname", "Do not fence a pending node that doesn't have an uname in node state yet"), SchedulerTest("node-pending-timeout", "Fence a pending node that has reached `node-pending-timeout`"), ]), SchedulerTestGroup([ SchedulerTest("systemhealth1", "System Health () #1"), SchedulerTest("systemhealth2", "System Health () #2"), SchedulerTest("systemhealth3", "System Health () #3"), SchedulerTest("systemhealthn1", "System Health (None) #1"), SchedulerTest("systemhealthn2", "System Health (None) #2"), SchedulerTest("systemhealthn3", "System Health (None) #3"), SchedulerTest("systemhealthm1", "System Health (Migrate On Red) #1"), SchedulerTest("systemhealthm2", "System Health (Migrate On Red) #2"), SchedulerTest("systemhealthm3", "System Health (Migrate On Red) #3"), SchedulerTest("systemhealtho1", "System Health (Only Green) #1"), SchedulerTest("systemhealtho2", "System Health (Only Green) #2"), SchedulerTest("systemhealtho3", "System Health (Only Green) #3"), SchedulerTest("systemhealthp1", "System Health (Progessive) #1"), SchedulerTest("systemhealthp2", "System Health (Progessive) #2"), SchedulerTest("systemhealthp3", "System Health (Progessive) #3"), SchedulerTest("allow-unhealthy-nodes", "System Health (migrate-on-red + allow-unhealth-nodes)"), ]), SchedulerTestGroup([ SchedulerTest("utilization", "Placement Strategy - utilization"), SchedulerTest("minimal", "Placement Strategy - minimal"), SchedulerTest("balanced", "Placement Strategy - balanced"), ]), SchedulerTestGroup([ SchedulerTest("placement-stickiness", "Optimized Placement Strategy - stickiness"), SchedulerTest("placement-priority", "Optimized Placement Strategy - priority"), SchedulerTest("placement-location", "Optimized Placement Strategy - location"), SchedulerTest("placement-capacity", "Optimized Placement Strategy - capacity"), ]), SchedulerTestGroup([ SchedulerTest("utilization-order1", "Utilization Order - Simple"), SchedulerTest("utilization-order2", "Utilization Order - Complex"), SchedulerTest("utilization-order3", "Utilization Order - Migrate"), SchedulerTest("utilization-order4", "Utilization Order - Live Migration (bnc#695440)"), SchedulerTest("utilization-complex", "Utilization with complex relationships"), SchedulerTest("utilization-shuffle", "Don't displace prmExPostgreSQLDB2 on act2, Start prmExPostgreSQLDB1 on act3"), SchedulerTest("load-stopped-loop", "Avoid transition loop due to load_stopped (cl#5044)"), SchedulerTest("load-stopped-loop-2", "cl#5235 - Prevent graph loops that can be introduced by load_stopped -> migrate_to ordering"), ]), SchedulerTestGroup([ SchedulerTest("colocated-utilization-primitive-1", "Colocated Utilization - Primitive"), SchedulerTest("colocated-utilization-primitive-2", "Colocated Utilization - Choose the most capable node"), SchedulerTest("colocated-utilization-group", "Colocated Utilization - Group"), SchedulerTest("colocated-utilization-clone", "Colocated Utilization - Clone"), SchedulerTest("utilization-check-allowed-nodes", "Only check the capacities of the nodes that can run the resource"), ]), SchedulerTestGroup([ SchedulerTest("node-maintenance-1", "cl#5128 - Node maintenance"), SchedulerTest("node-maintenance-2", "cl#5128 - Node maintenance (coming out of maintenance mode)"), SchedulerTest("shutdown-maintenance-node", "Do not fence a maintenance node if it shuts down cleanly"), SchedulerTest("rsc-maintenance", "Per-resource maintenance"), ]), SchedulerTestGroup([ SchedulerTest("not-installed-agent", "The resource agent is missing"), SchedulerTest("not-installed-tools", "Something the resource agent needs is missing"), ]), SchedulerTestGroup([ SchedulerTest("stopped-monitor-00", "Stopped Monitor - initial start"), SchedulerTest("stopped-monitor-01", "Stopped Monitor - failed started"), SchedulerTest("stopped-monitor-02", "Stopped Monitor - started multi-up"), SchedulerTest("stopped-monitor-03", "Stopped Monitor - stop started"), SchedulerTest("stopped-monitor-04", "Stopped Monitor - failed stop"), SchedulerTest("stopped-monitor-05", "Stopped Monitor - start unmanaged"), SchedulerTest("stopped-monitor-06", "Stopped Monitor - unmanaged multi-up"), SchedulerTest("stopped-monitor-07", "Stopped Monitor - start unmanaged multi-up"), SchedulerTest("stopped-monitor-08", "Stopped Monitor - migrate"), SchedulerTest("stopped-monitor-09", "Stopped Monitor - unmanage started"), SchedulerTest("stopped-monitor-10", "Stopped Monitor - unmanaged started multi-up"), SchedulerTest("stopped-monitor-11", "Stopped Monitor - stop unmanaged started"), SchedulerTest("stopped-monitor-12", "Stopped Monitor - unmanaged started multi-up (target-role=Stopped)"), SchedulerTest("stopped-monitor-20", "Stopped Monitor - initial stop"), SchedulerTest("stopped-monitor-21", "Stopped Monitor - stopped single-up"), SchedulerTest("stopped-monitor-22", "Stopped Monitor - stopped multi-up"), SchedulerTest("stopped-monitor-23", "Stopped Monitor - start stopped"), SchedulerTest("stopped-monitor-24", "Stopped Monitor - unmanage stopped"), SchedulerTest("stopped-monitor-25", "Stopped Monitor - unmanaged stopped multi-up"), SchedulerTest("stopped-monitor-26", "Stopped Monitor - start unmanaged stopped"), SchedulerTest("stopped-monitor-27", "Stopped Monitor - unmanaged stopped multi-up (target-role=Started)"), SchedulerTest("stopped-monitor-30", "Stopped Monitor - new node started"), SchedulerTest("stopped-monitor-31", "Stopped Monitor - new node stopped"), ]), SchedulerTestGroup([ # This is a combo test to check: # - probe timeout defaults to the minimum-interval monitor's # - duplicate recurring operations are ignored # - if timeout spec is bad, the default timeout is used # - failure is blocked with on-fail=block even if ISO8601 interval is specified # - started/stopped role monitors are started/stopped on right nodes SchedulerTest("intervals", "Recurring monitor interval handling"), ]), SchedulerTestGroup([ SchedulerTest("ticket-primitive-1", "Ticket - Primitive (loss-policy=stop, initial)"), SchedulerTest("ticket-primitive-2", "Ticket - Primitive (loss-policy=stop, granted)"), SchedulerTest("ticket-primitive-3", "Ticket - Primitive (loss-policy-stop, revoked)"), SchedulerTest("ticket-primitive-4", "Ticket - Primitive (loss-policy=demote, initial)"), SchedulerTest("ticket-primitive-5", "Ticket - Primitive (loss-policy=demote, granted)"), SchedulerTest("ticket-primitive-6", "Ticket - Primitive (loss-policy=demote, revoked)"), SchedulerTest("ticket-primitive-7", "Ticket - Primitive (loss-policy=fence, initial)"), SchedulerTest("ticket-primitive-8", "Ticket - Primitive (loss-policy=fence, granted)"), SchedulerTest("ticket-primitive-9", "Ticket - Primitive (loss-policy=fence, revoked)"), SchedulerTest("ticket-primitive-10", "Ticket - Primitive (loss-policy=freeze, initial)"), SchedulerTest("ticket-primitive-11", "Ticket - Primitive (loss-policy=freeze, granted)"), SchedulerTest("ticket-primitive-12", "Ticket - Primitive (loss-policy=freeze, revoked)"), SchedulerTest("ticket-primitive-13", "Ticket - Primitive (loss-policy=stop, standby, granted)"), SchedulerTest("ticket-primitive-14", "Ticket - Primitive (loss-policy=stop, granted, standby)"), SchedulerTest("ticket-primitive-15", "Ticket - Primitive (loss-policy=stop, standby, revoked)"), SchedulerTest("ticket-primitive-16", "Ticket - Primitive (loss-policy=demote, standby, granted)"), SchedulerTest("ticket-primitive-17", "Ticket - Primitive (loss-policy=demote, granted, standby)"), SchedulerTest("ticket-primitive-18", "Ticket - Primitive (loss-policy=demote, standby, revoked)"), SchedulerTest("ticket-primitive-19", "Ticket - Primitive (loss-policy=fence, standby, granted)"), SchedulerTest("ticket-primitive-20", "Ticket - Primitive (loss-policy=fence, granted, standby)"), SchedulerTest("ticket-primitive-21", "Ticket - Primitive (loss-policy=fence, standby, revoked)"), SchedulerTest("ticket-primitive-22", "Ticket - Primitive (loss-policy=freeze, standby, granted)"), SchedulerTest("ticket-primitive-23", "Ticket - Primitive (loss-policy=freeze, granted, standby)"), SchedulerTest("ticket-primitive-24", "Ticket - Primitive (loss-policy=freeze, standby, revoked)"), ]), SchedulerTestGroup([ SchedulerTest("ticket-group-1", "Ticket - Group (loss-policy=stop, initial)"), SchedulerTest("ticket-group-2", "Ticket - Group (loss-policy=stop, granted)"), SchedulerTest("ticket-group-3", "Ticket - Group (loss-policy-stop, revoked)"), SchedulerTest("ticket-group-4", "Ticket - Group (loss-policy=demote, initial)"), SchedulerTest("ticket-group-5", "Ticket - Group (loss-policy=demote, granted)"), SchedulerTest("ticket-group-6", "Ticket - Group (loss-policy=demote, revoked)"), SchedulerTest("ticket-group-7", "Ticket - Group (loss-policy=fence, initial)"), SchedulerTest("ticket-group-8", "Ticket - Group (loss-policy=fence, granted)"), SchedulerTest("ticket-group-9", "Ticket - Group (loss-policy=fence, revoked)"), SchedulerTest("ticket-group-10", "Ticket - Group (loss-policy=freeze, initial)"), SchedulerTest("ticket-group-11", "Ticket - Group (loss-policy=freeze, granted)"), SchedulerTest("ticket-group-12", "Ticket - Group (loss-policy=freeze, revoked)"), SchedulerTest("ticket-group-13", "Ticket - Group (loss-policy=stop, standby, granted)"), SchedulerTest("ticket-group-14", "Ticket - Group (loss-policy=stop, granted, standby)"), SchedulerTest("ticket-group-15", "Ticket - Group (loss-policy=stop, standby, revoked)"), SchedulerTest("ticket-group-16", "Ticket - Group (loss-policy=demote, standby, granted)"), SchedulerTest("ticket-group-17", "Ticket - Group (loss-policy=demote, granted, standby)"), SchedulerTest("ticket-group-18", "Ticket - Group (loss-policy=demote, standby, revoked)"), SchedulerTest("ticket-group-19", "Ticket - Group (loss-policy=fence, standby, granted)"), SchedulerTest("ticket-group-20", "Ticket - Group (loss-policy=fence, granted, standby)"), SchedulerTest("ticket-group-21", "Ticket - Group (loss-policy=fence, standby, revoked)"), SchedulerTest("ticket-group-22", "Ticket - Group (loss-policy=freeze, standby, granted)"), SchedulerTest("ticket-group-23", "Ticket - Group (loss-policy=freeze, granted, standby)"), SchedulerTest("ticket-group-24", "Ticket - Group (loss-policy=freeze, standby, revoked)"), ]), SchedulerTestGroup([ SchedulerTest("ticket-clone-1", "Ticket - Clone (loss-policy=stop, initial)"), SchedulerTest("ticket-clone-2", "Ticket - Clone (loss-policy=stop, granted)"), SchedulerTest("ticket-clone-3", "Ticket - Clone (loss-policy-stop, revoked)"), SchedulerTest("ticket-clone-4", "Ticket - Clone (loss-policy=demote, initial)"), SchedulerTest("ticket-clone-5", "Ticket - Clone (loss-policy=demote, granted)"), SchedulerTest("ticket-clone-6", "Ticket - Clone (loss-policy=demote, revoked)"), SchedulerTest("ticket-clone-7", "Ticket - Clone (loss-policy=fence, initial)"), SchedulerTest("ticket-clone-8", "Ticket - Clone (loss-policy=fence, granted)"), SchedulerTest("ticket-clone-9", "Ticket - Clone (loss-policy=fence, revoked)"), SchedulerTest("ticket-clone-10", "Ticket - Clone (loss-policy=freeze, initial)"), SchedulerTest("ticket-clone-11", "Ticket - Clone (loss-policy=freeze, granted)"), SchedulerTest("ticket-clone-12", "Ticket - Clone (loss-policy=freeze, revoked)"), SchedulerTest("ticket-clone-13", "Ticket - Clone (loss-policy=stop, standby, granted)"), SchedulerTest("ticket-clone-14", "Ticket - Clone (loss-policy=stop, granted, standby)"), SchedulerTest("ticket-clone-15", "Ticket - Clone (loss-policy=stop, standby, revoked)"), SchedulerTest("ticket-clone-16", "Ticket - Clone (loss-policy=demote, standby, granted)"), SchedulerTest("ticket-clone-17", "Ticket - Clone (loss-policy=demote, granted, standby)"), SchedulerTest("ticket-clone-18", "Ticket - Clone (loss-policy=demote, standby, revoked)"), SchedulerTest("ticket-clone-19", "Ticket - Clone (loss-policy=fence, standby, granted)"), SchedulerTest("ticket-clone-20", "Ticket - Clone (loss-policy=fence, granted, standby)"), SchedulerTest("ticket-clone-21", "Ticket - Clone (loss-policy=fence, standby, revoked)"), SchedulerTest("ticket-clone-22", "Ticket - Clone (loss-policy=freeze, standby, granted)"), SchedulerTest("ticket-clone-23", "Ticket - Clone (loss-policy=freeze, granted, standby)"), SchedulerTest("ticket-clone-24", "Ticket - Clone (loss-policy=freeze, standby, revoked)"), ]), SchedulerTestGroup([ SchedulerTest("ticket-promoted-1", "Ticket - Promoted (loss-policy=stop, initial)"), SchedulerTest("ticket-promoted-2", "Ticket - Promoted (loss-policy=stop, granted)"), SchedulerTest("ticket-promoted-3", "Ticket - Promoted (loss-policy-stop, revoked)"), SchedulerTest("ticket-promoted-4", "Ticket - Promoted (loss-policy=demote, initial)"), SchedulerTest("ticket-promoted-5", "Ticket - Promoted (loss-policy=demote, granted)"), SchedulerTest("ticket-promoted-6", "Ticket - Promoted (loss-policy=demote, revoked)"), SchedulerTest("ticket-promoted-7", "Ticket - Promoted (loss-policy=fence, initial)"), SchedulerTest("ticket-promoted-8", "Ticket - Promoted (loss-policy=fence, granted)"), SchedulerTest("ticket-promoted-9", "Ticket - Promoted (loss-policy=fence, revoked)"), SchedulerTest("ticket-promoted-10", "Ticket - Promoted (loss-policy=freeze, initial)"), SchedulerTest("ticket-promoted-11", "Ticket - Promoted (loss-policy=freeze, granted)"), SchedulerTest("ticket-promoted-12", "Ticket - Promoted (loss-policy=freeze, revoked)"), SchedulerTest("ticket-promoted-13", "Ticket - Promoted (loss-policy=stop, standby, granted)"), SchedulerTest("ticket-promoted-14", "Ticket - Promoted (loss-policy=stop, granted, standby)"), SchedulerTest("ticket-promoted-15", "Ticket - Promoted (loss-policy=stop, standby, revoked)"), SchedulerTest("ticket-promoted-16", "Ticket - Promoted (loss-policy=demote, standby, granted)"), SchedulerTest("ticket-promoted-17", "Ticket - Promoted (loss-policy=demote, granted, standby)"), SchedulerTest("ticket-promoted-18", "Ticket - Promoted (loss-policy=demote, standby, revoked)"), SchedulerTest("ticket-promoted-19", "Ticket - Promoted (loss-policy=fence, standby, granted)"), SchedulerTest("ticket-promoted-20", "Ticket - Promoted (loss-policy=fence, granted, standby)"), SchedulerTest("ticket-promoted-21", "Ticket - Promoted (loss-policy=fence, standby, revoked)"), SchedulerTest("ticket-promoted-22", "Ticket - Promoted (loss-policy=freeze, standby, granted)"), SchedulerTest("ticket-promoted-23", "Ticket - Promoted (loss-policy=freeze, granted, standby)"), SchedulerTest("ticket-promoted-24", "Ticket - Promoted (loss-policy=freeze, standby, revoked)"), ]), SchedulerTestGroup([ SchedulerTest("ticket-rsc-sets-1", "Ticket - Resource sets (1 ticket, initial)"), SchedulerTest("ticket-rsc-sets-2", "Ticket - Resource sets (1 ticket, granted)"), SchedulerTest("ticket-rsc-sets-3", "Ticket - Resource sets (1 ticket, revoked)"), SchedulerTest("ticket-rsc-sets-4", "Ticket - Resource sets (2 tickets, initial)"), SchedulerTest("ticket-rsc-sets-5", "Ticket - Resource sets (2 tickets, granted)"), SchedulerTest("ticket-rsc-sets-6", "Ticket - Resource sets (2 tickets, granted)"), SchedulerTest("ticket-rsc-sets-7", "Ticket - Resource sets (2 tickets, revoked)"), SchedulerTest("ticket-rsc-sets-8", "Ticket - Resource sets (1 ticket, standby, granted)"), SchedulerTest("ticket-rsc-sets-9", "Ticket - Resource sets (1 ticket, granted, standby)"), SchedulerTest("ticket-rsc-sets-10", "Ticket - Resource sets (1 ticket, standby, revoked)"), SchedulerTest("ticket-rsc-sets-11", "Ticket - Resource sets (2 tickets, standby, granted)"), SchedulerTest("ticket-rsc-sets-12", "Ticket - Resource sets (2 tickets, standby, granted)"), SchedulerTest("ticket-rsc-sets-13", "Ticket - Resource sets (2 tickets, granted, standby)"), SchedulerTest("ticket-rsc-sets-14", "Ticket - Resource sets (2 tickets, standby, revoked)"), SchedulerTest("cluster-specific-params", "Cluster-specific instance attributes based on rules"), SchedulerTest("site-specific-params", "Site-specific instance attributes based on rules"), ]), SchedulerTestGroup([ SchedulerTest("template-1", "Template - 1"), SchedulerTest("template-2", "Template - 2"), SchedulerTest("template-3", "Template - 3 (merge operations)"), SchedulerTest("template-coloc-1", "Template - Colocation 1"), SchedulerTest("template-coloc-2", "Template - Colocation 2"), SchedulerTest("template-coloc-3", "Template - Colocation 3"), SchedulerTest("template-order-1", "Template - Order 1"), SchedulerTest("template-order-2", "Template - Order 2"), SchedulerTest("template-order-3", "Template - Order 3"), SchedulerTest("template-ticket", "Template - Ticket"), SchedulerTest("template-rsc-sets-1", "Template - Resource Sets 1"), SchedulerTest("template-rsc-sets-2", "Template - Resource Sets 2"), SchedulerTest("template-rsc-sets-3", "Template - Resource Sets 3"), SchedulerTest("template-rsc-sets-4", "Template - Resource Sets 4"), SchedulerTest("template-clone-primitive", "Cloned primitive from template"), SchedulerTest("template-clone-group", "Cloned group from template"), SchedulerTest("location-sets-templates", "Resource sets and templates - Location"), SchedulerTest("tags-coloc-order-1", "Tags - Colocation and Order (Simple)"), SchedulerTest("tags-coloc-order-2", "Tags - Colocation and Order (Resource Sets with Templates)"), SchedulerTest("tags-location", "Tags - Location"), SchedulerTest("tags-ticket", "Tags - Ticket"), ]), SchedulerTestGroup([ SchedulerTest("container-1", "Container - initial"), SchedulerTest("container-2", "Container - monitor failed"), SchedulerTest("container-3", "Container - stop failed"), SchedulerTest("container-4", "Container - reached migration-threshold"), SchedulerTest("container-group-1", "Container in group - initial"), SchedulerTest("container-group-2", "Container in group - monitor failed"), SchedulerTest("container-group-3", "Container in group - stop failed"), SchedulerTest("container-group-4", "Container in group - reached migration-threshold"), SchedulerTest("container-is-remote-node", "Place resource within container when container is remote-node"), SchedulerTest("bug-rh-1097457", "Kill user defined container/contents ordering"), SchedulerTest("bug-cl-5247", "Graph loop when recovering m/s resource in a container"), SchedulerTest("bundle-order-startup", "Bundle startup ordering"), SchedulerTest("bundle-order-partial-start", "Bundle startup ordering when some dependencies are already running"), SchedulerTest("bundle-order-partial-start-2", "Bundle startup ordering when some dependencies and the container are already running"), SchedulerTest("bundle-order-stop", "Bundle stop ordering"), SchedulerTest("bundle-order-partial-stop", "Bundle startup ordering when some dependencies are already stopped"), SchedulerTest("bundle-order-stop-on-remote", "Stop nested resource after bringing up the connection"), SchedulerTest("bundle-order-startup-clone", "Prevent startup because bundle isn't promoted"), SchedulerTest("bundle-order-startup-clone-2", "Bundle startup with clones"), SchedulerTest("bundle-order-stop-clone", "Stop bundle because clone is stopping"), SchedulerTest("bundle-interleave-start", "Interleave bundle starts"), SchedulerTest("bundle-interleave-promote", "Interleave bundle promotes"), SchedulerTest("bundle-nested-colocation", "Colocation of nested connection resources"), SchedulerTest("bundle-order-fencing", "Order pseudo bundle fencing after parent node fencing if both are happening"), SchedulerTest("bundle-probe-order-1", "order 1"), SchedulerTest("bundle-probe-order-2", "order 2"), SchedulerTest("bundle-probe-order-3", "order 3"), SchedulerTest("bundle-probe-remotes", "Ensure remotes get probed too"), SchedulerTest("bundle-replicas-change", "Change bundle from 1 replica to multiple"), SchedulerTest("bundle-connection-with-container", "Don't move a container due to connection preferences"), SchedulerTest("nested-remote-recovery", "Recover bundle's container hosted on remote node"), SchedulerTest("bundle-promoted-location-1", "Promotable bundle, positive location"), SchedulerTest("bundle-promoted-location-2", "Promotable bundle, negative location"), SchedulerTest("bundle-promoted-location-3", "Promotable bundle, positive location for promoted role"), SchedulerTest("bundle-promoted-location-4", "Promotable bundle, negative location for promoted role"), SchedulerTest("bundle-promoted-location-5", "Promotable bundle, positive location for unpromoted role"), SchedulerTest("bundle-promoted-location-6", "Promotable bundle, negative location for unpromoted role"), SchedulerTest("bundle-promoted-colocation-1", "Primary promoted bundle, dependent primitive (mandatory coloc)"), SchedulerTest("bundle-promoted-colocation-2", "Primary promoted bundle, dependent primitive (optional coloc)"), SchedulerTest("bundle-promoted-colocation-3", "Dependent promoted bundle, primary primitive (mandatory coloc)"), SchedulerTest("bundle-promoted-colocation-4", "Dependent promoted bundle, primary primitive (optional coloc)"), SchedulerTest("bundle-promoted-colocation-5", "Primary and dependent promoted bundle instances (mandatory coloc)"), SchedulerTest("bundle-promoted-colocation-6", "Primary and dependent promoted bundle instances (optional coloc)"), SchedulerTest("bundle-promoted-anticolocation-1", "Primary promoted bundle, dependent primitive (mandatory anti)"), SchedulerTest("bundle-promoted-anticolocation-2", "Primary promoted bundle, dependent primitive (optional anti)"), SchedulerTest("bundle-promoted-anticolocation-3", "Dependent promoted bundle, primary primitive (mandatory anti)"), SchedulerTest("bundle-promoted-anticolocation-4", "Dependent promoted bundle, primary primitive (optional anti)"), SchedulerTest("bundle-promoted-anticolocation-5", "Primary and dependent promoted bundle instances (mandatory anti)"), SchedulerTest("bundle-promoted-anticolocation-6", "Primary and dependent promoted bundle instances (optional anti)"), ]), SchedulerTestGroup([ SchedulerTest("whitebox-fail1", "Fail whitebox container rsc"), SchedulerTest("whitebox-fail2", "Fail cluster connection to guest node"), SchedulerTest("whitebox-fail3", "Failed containers should not run nested on remote nodes"), SchedulerTest("whitebox-start", "Start whitebox container with resources assigned to it"), SchedulerTest("whitebox-stop", "Stop whitebox container with resources assigned to it"), SchedulerTest("whitebox-move", "Move whitebox container with resources assigned to it"), SchedulerTest("whitebox-asymmetric", "Verify connection rsc opts-in based on container resource"), SchedulerTest("whitebox-ms-ordering", "Verify promote/demote can not occur before connection is established"), SchedulerTest("whitebox-ms-ordering-move", "Stop/Start cycle within a moving container"), SchedulerTest("whitebox-orphaned", "Properly shutdown orphaned whitebox container"), SchedulerTest("whitebox-orphan-ms", "Properly tear down orphan ms resources on remote-nodes"), SchedulerTest("whitebox-unexpectedly-running", "Recover container nodes the cluster did not start"), SchedulerTest("whitebox-migrate1", "Migrate both container and connection resource"), SchedulerTest("whitebox-imply-stop-on-fence", "imply stop action on container node rsc when host node is fenced"), SchedulerTest("whitebox-nested-group", "Verify guest remote-node works nested in a group"), SchedulerTest("guest-node-host-dies", "Verify guest node is recovered if host goes away"), SchedulerTest("guest-node-cleanup", "Order guest node connection recovery after container probe"), SchedulerTest("guest-host-not-fenceable", "Actions on guest node are unrunnable if host is unclean and cannot be fenced"), ]), SchedulerTestGroup([ SchedulerTest("remote-startup-probes", "Baremetal remote-node startup probes"), SchedulerTest("remote-startup", "Startup a newly discovered remote-nodes with no status"), SchedulerTest("remote-fence-unclean", "Fence unclean baremetal remote-node"), SchedulerTest("remote-fence-unclean2", "Fence baremetal remote-node after cluster node fails and connection can not be recovered"), SchedulerTest("remote-fence-unclean-3", "Probe failed remote nodes (triggers fencing)"), SchedulerTest("remote-move", "Move remote-node connection resource"), SchedulerTest("remote-disable", "Disable a baremetal remote-node"), SchedulerTest("remote-probe-disable", "Probe then stop a baremetal remote-node"), SchedulerTest("remote-orphaned", "Properly shutdown orphaned connection resource"), SchedulerTest("remote-orphaned2", "verify we can handle orphaned remote connections with active resources on the remote"), SchedulerTest("remote-recover", "Recover connection resource after cluster-node fails"), SchedulerTest("remote-stale-node-entry", "Make sure we properly handle leftover remote-node entries in the node section"), SchedulerTest("remote-partial-migrate", "Make sure partial migrations are handled before ops on the remote node"), SchedulerTest("remote-partial-migrate2", "Make sure partial migration target is prefered for remote connection"), SchedulerTest("remote-recover-fail", "Make sure start failure causes fencing if rsc are active on remote"), SchedulerTest("remote-start-fail", "Make sure a start failure does not result in fencing if no active resources are on remote"), SchedulerTest("remote-unclean2", "Make monitor failure always results in fencing, even if no rsc are active on remote"), SchedulerTest("remote-fence-before-reconnect", "Fence before clearing recurring monitor failure"), SchedulerTest("remote-recovery", "Recover remote connections before attempting demotion"), SchedulerTest("remote-recover-connection", "Optimistically recovery of only the connection"), SchedulerTest("remote-recover-all", "Fencing when the connection has no home"), SchedulerTest("remote-recover-no-resources", "Fencing when the connection has no home and no active resources"), SchedulerTest("remote-recover-unknown", "Fencing when the connection has no home and the remote has no operation history"), SchedulerTest("remote-reconnect-delay", "Waiting for remote reconnect interval to expire"), SchedulerTest("remote-connection-unrecoverable", "Remote connection host must be fenced, with connection unrecoverable"), SchedulerTest("remote-connection-shutdown", "Remote connection shutdown"), SchedulerTest("cancel-behind-moving-remote", "Route recurring monitor cancellations through original node of a moving remote connection"), ]), SchedulerTestGroup([ SchedulerTest("resource-discovery", "Exercises resource-discovery location constraint option"), SchedulerTest("rsc-discovery-per-node", "Disable resource discovery per node"), SchedulerTest("shutdown-lock", "Ensure shutdown lock works properly"), SchedulerTest("shutdown-lock-expiration", "Ensure shutdown lock expiration works properly"), ]), SchedulerTestGroup([ SchedulerTest("op-defaults", "Test op_defaults conditional expressions"), SchedulerTest("op-defaults-2", "Test op_defaults AND'ed conditional expressions"), SchedulerTest("op-defaults-3", "Test op_defaults precedence"), SchedulerTest("rsc-defaults", "Test rsc_defaults conditional expressions"), SchedulerTest("rsc-defaults-2", "Test rsc_defaults conditional expressions without type"), ]), SchedulerTestGroup([ SchedulerTest("stop-all-resources", "Test stop-all-resources=true "), ]), SchedulerTestGroup([ SchedulerTest("ocf_degraded-remap-ocf_ok", "Test degraded remapped to OK"), SchedulerTest("ocf_degraded_promoted-remap-ocf_ok", "Test degraded promoted remapped to OK"), ]), ] TESTS_64BIT = [ SchedulerTestGroup([ SchedulerTest("year-2038", "Check handling of timestamps beyond 2038-01-19 03:14:08 UTC"), ]), ] def is_executable(path): """ Check whether a file at a given path is executable. """ try: return os.stat(path)[stat.ST_MODE] & stat.S_IXUSR except OSError: return False def diff(file1, file2, **kwargs): """ Call diff on two files """ return subprocess.call([ "diff", "-u", "-N", "--ignore-all-space", "--ignore-blank-lines", file1, file2 ], **kwargs) def sort_file(filename): """ Sort a file alphabetically """ with io.open(filename, "rt") as f: lines = sorted(f) with io.open(filename, "wt") as f: f.writelines(lines) def remove_files(filenames): """ Remove a list of files """ for filename in filenames: try: os.remove(filename) except OSError: pass def normalize(filename): """ Remove text from a file that isn't important for comparison """ if not hasattr(normalize, "patterns"): normalize.patterns = [ re.compile(r'crm_feature_set="[^"]*"'), re.compile(r'batch-limit="[0-9]*"') ] if os.path.isfile(filename): with io.open(filename, "rt") as f: lines = f.readlines() with io.open(filename, "wt") as f: for line in lines: for pattern in normalize.patterns: line = pattern.sub("", line) f.write(line) def cat(filename, dest=sys.stdout): """ Copy a file to a destination file descriptor """ with io.open(filename, "rt") as f: shutil.copyfileobj(f, dest) class CtsScheduler(object): """ Regression tests for Pacemaker's scheduler """ def _parse_args(self, argv): """ Parse command-line arguments """ parser = argparse.ArgumentParser(description=DESC) parser.add_argument('-V', '--verbose', action='count', help='Display any differences from expected output') parser.add_argument('--run', metavar='TEST', help=('Run only single specified test (any further ' 'arguments will be passed to crm_simulate)')) parser.add_argument('--update', action='store_true', help='Update expected results with actual results') parser.add_argument('-b', '--binary', metavar='PATH', help='Specify path to crm_simulate') parser.add_argument('-i', '--io-dir', metavar='PATH', help='Specify path to regression test data directory') parser.add_argument('-o', '--out-dir', metavar='PATH', help='Specify where intermediate and output files should go') parser.add_argument('-v', '--valgrind', action='store_true', help='Run all commands under valgrind') parser.add_argument('--valgrind-dhat', action='store_true', help='Run all commands under valgrind with heap analyzer') parser.add_argument('--valgrind-skip-output', action='store_true', help='If running under valgrind, do not display output') parser.add_argument('--testcmd-options', metavar='OPTIONS', default='', help='Additional options for command under test') # argparse can't handle "everything after --run TEST", so grab that self.single_test_args = [] narg = 0 for arg in argv: narg = narg + 1 if arg == '--run': (argv, self.single_test_args) = (argv[:narg+1], argv[narg+1:]) break self.args = parser.parse_args(argv[1:]) def _error(self, s): - print(" * ERROR: %s" % s) + print(f" * ERROR: {s}") def _failed(self, s): - print(" * FAILED: %s" % s) + print(f" * FAILED: {s}") def _get_valgrind_cmd(self): """ Return command arguments needed (or not) to run valgrind """ if self.args.valgrind: os.environ['G_SLICE'] = "always-malloc" return [ "valgrind", "-q", "--gen-suppressions=all", "--time-stamp=yes", "--trace-children=no", "--show-reachable=no", "--leak-check=full", "--num-callers=20", - "--suppressions=%s/valgrind-pcmk.suppressions" % (self.test_home) + f"--suppressions={self.test_home}/valgrind-pcmk.suppressions" ] if self.args.valgrind_dhat: os.environ['G_SLICE'] = "always-malloc" return [ "valgrind", "--tool=exp-dhat", "--time-stamp=yes", "--trace-children=no", "--show-top-n=100", "--num-callers=4" ] return [] def _get_simulator_cmd(self): """ Locate the simulation binary """ if self.args.binary is None: self.args.binary = BuildOptions._BUILD_DIR + "/tools/crm_simulate" if not is_executable(self.args.binary): self.args.binary = BuildOptions.SBIN_DIR + "/crm_simulate" if not is_executable(self.args.binary): # @TODO it would be more pythonic to raise an exception self._error("Test binary " + self.args.binary + " not found") sys.exit(ExitStatus.NOT_INSTALLED) return [ self.args.binary ] + shlex.split(self.args.testcmd_options) def set_schema_env(self): """ Ensure schema directory environment variable is set, if possible """ try: return os.environ['PCMK_schema_directory'] except KeyError: for d in [ os.path.join(BuildOptions._BUILD_DIR, "xml"), BuildOptions.SCHEMA_DIR ]: if os.path.isdir(d): os.environ['PCMK_schema_directory'] = d return d return None def __init__(self, argv=sys.argv): # Ensure all command output is in portable locale for comparison os.environ['LC_ALL'] = "C" self._parse_args(argv) # Where this executable lives self.test_home = os.path.dirname(os.path.realpath(argv[0])) # Where test data resides if self.args.io_dir is None: self.args.io_dir = os.path.join(self.test_home, "scheduler") self.xml_input_dir = os.path.join(self.args.io_dir, "xml") self.expected_dir = os.path.join(self.args.io_dir, "exp") self.dot_expected_dir = os.path.join(self.args.io_dir, "dot") self.scores_dir = os.path.join(self.args.io_dir, "scores") self.summary_dir = os.path.join(self.args.io_dir, "summary") self.stderr_expected_dir = os.path.join(self.args.io_dir, "stderr") # Create a temporary directory to store diff file self.failed_dir = tempfile.mkdtemp(prefix='cts-scheduler_') # Where to store generated files if self.args.out_dir is None: self.args.out_dir = self.args.io_dir self.failed_filename = os.path.join(self.failed_dir, "test-output.diff") else: self.failed_filename = os.path.join(self.args.out_dir, "test-output.diff") os.environ['CIB_shadow_dir'] = self.args.out_dir self.failed_file = None self.outfile_out_dir = os.path.join(self.args.out_dir, "out") self.dot_out_dir = os.path.join(self.args.out_dir, "dot") self.scores_out_dir = os.path.join(self.args.out_dir, "scores") self.summary_out_dir = os.path.join(self.args.out_dir, "summary") self.stderr_out_dir = os.path.join(self.args.out_dir, "stderr") self.valgrind_out_dir = os.path.join(self.args.out_dir, "valgrind") # Single test mode (if requested) try: # User can give test base name or file name of a test input self.args.run = os.path.splitext(os.path.basename(self.args.run))[0] except (AttributeError, TypeError): pass # --run was not specified self.set_schema_env() # Arguments needed (or not) to run commands self.valgrind_args = self._get_valgrind_cmd() self.simulate_args = self._get_simulator_cmd() # Test counters self.num_failed = 0 self.num_tests = 0 # Ensure that the main output directory exists # We don't want to create it with os.makedirs below if not os.path.isdir(self.args.out_dir): self._error("Output directory missing; can't create output files") sys.exit(ExitStatus.CANTCREAT) # Create output subdirectories if they don't exist try: os.makedirs(self.outfile_out_dir, 0o755, True) os.makedirs(self.dot_out_dir, 0o755, True) os.makedirs(self.scores_out_dir, 0o755, True) os.makedirs(self.summary_out_dir, 0o755, True) os.makedirs(self.stderr_out_dir, 0o755, True) if self.valgrind_args: os.makedirs(self.valgrind_out_dir, 0o755, True) except OSError as ex: - self._error("Unable to create output subdirectory: %s" % ex) + self._error(f"Unable to create output subdirectory: {ex}") remove_files([ self.outfile_out_dir, self.dot_out_dir, self.scores_out_dir, self.summary_out_dir, self.stderr_out_dir, ]) sys.exit(ExitStatus.CANTCREAT) def _compare_files(self, filename1, filename2): """ Add any file differences to failed results """ if diff(filename1, filename2, stdout=subprocess.DEVNULL) != 0: diff(filename1, filename2, stdout=self.failed_file, stderr=subprocess.DEVNULL) self.failed_file.write("\n") return True return False def run_one(self, test_name, test_desc, test_args): """ Run one scheduler test """ - print(" Test %-41s %s" % ((test_name + ":"), test_desc)) + s = test_name + ":" + print(f" Test {s:41} {test_desc}") did_fail = False self.num_tests = self.num_tests + 1 # Test inputs - input_filename = os.path.join( - self.xml_input_dir, "%s.xml" % test_name) - expected_filename = os.path.join( - self.expected_dir, "%s.exp" % test_name) - dot_expected_filename = os.path.join( - self.dot_expected_dir, "%s.dot" % test_name) - scores_filename = os.path.join( - self.scores_dir, "%s.scores" % test_name) - summary_filename = os.path.join( - self.summary_dir, "%s.summary" % test_name) - stderr_expected_filename = os.path.join( - self.stderr_expected_dir, "%s.stderr" % test_name) + input_filename = os.path.join(self.xml_input_dir, f"{test_name}.xml") + expected_filename = os.path.join(self.expected_dir, f"{test_name}.exp") + dot_expected_filename = os.path.join(self.dot_expected_dir, f"{test_name}.dot") + scores_filename = os.path.join(self.scores_dir, f"{test_name}.scores") + summary_filename = os.path.join(self.summary_dir, f"{test_name}.summary") + stderr_expected_filename = os.path.join(self.stderr_expected_dir, f"{test_name}.stderr") # (Intermediate) test outputs - output_filename = os.path.join( - self.outfile_out_dir, "%s.out" % test_name) - dot_output_filename = os.path.join( - self.dot_out_dir, "%s.dot.pe" % test_name) - score_output_filename = os.path.join( - self.scores_out_dir, "%s.scores.pe" % test_name) - summary_output_filename = os.path.join( - self.summary_out_dir, "%s.summary.pe" % test_name) - stderr_output_filename = os.path.join( - self.stderr_out_dir, "%s.stderr.pe" % test_name) - valgrind_output_filename = os.path.join( - self.valgrind_out_dir, "%s.valgrind" % test_name) + output_filename = os.path.join(self.outfile_out_dir, f"{test_name}.out") + dot_output_filename = os.path.join(self.dot_out_dir, f"{test_name}.dot.pe") + score_output_filename = os.path.join(self.scores_out_dir, f"{test_name}.scores.pe") + summary_output_filename = os.path.join(self.summary_out_dir, f"{test_name}.summary.pe") + stderr_output_filename = os.path.join(self.stderr_out_dir, f"{test_name}.stderr.pe") + valgrind_output_filename = os.path.join(self.valgrind_out_dir, f"{test_name}.valgrind") # Common arguments for running test test_cmd = [] if self.valgrind_args: - test_cmd = self.valgrind_args + [ "--log-file=%s" % valgrind_output_filename ] + test_cmd = self.valgrind_args + [ f"--log-file={valgrind_output_filename}" ] test_cmd = test_cmd + self.simulate_args # @TODO It would be more pythonic to raise exceptions for errors, # then perhaps it would be nice to make a single-test class # Ensure necessary test inputs exist if not os.path.isfile(input_filename): self._error("No input") self.num_failed = self.num_failed + 1 return ExitStatus.NOINPUT if not self.args.update and not os.path.isfile(expected_filename): self._error("no stored output") return ExitStatus.NOINPUT # Run simulation to generate summary output test_cmd_full = test_cmd + [ '-x', input_filename, '-S' ] + test_args if self.args.run: # Single test mode print(" ".join(test_cmd_full)) with io.open(summary_output_filename, "wt") as f: subprocess.run(test_cmd_full, stdout=f, stderr=subprocess.STDOUT, env=os.environ, check=False) if self.args.run: cat(summary_output_filename) # Re-run simulation to generate dot, graph, and scores test_cmd_full = test_cmd + [ '-x', input_filename, '-D', dot_output_filename, '-G', output_filename, '-sSQ' ] + test_args with io.open(stderr_output_filename, "wt") as f_stderr, \ io.open(score_output_filename, "wt") as f_score: rc = subprocess.call(test_cmd_full, stdout=f_score, stderr=f_stderr, env=os.environ) # Check for test command failure if rc != ExitStatus.OK: - self._failed("Test returned: %d" % rc) + self._failed(f"Test returned: {rc}") did_fail = True print(" ".join(test_cmd_full)) # Check for valgrind errors if self.valgrind_args and not self.args.valgrind_skip_output: if os.stat(valgrind_output_filename).st_size > 0: self._failed("Valgrind reported errors") did_fail = True cat(valgrind_output_filename) remove_files([ valgrind_output_filename ]) # Check for core dump if os.path.isfile("core"): self._failed("Core-file detected: core." + test_name) did_fail = True - os.rename("core", "%s/core.%s" % (self.test_home, test_name)) + os.rename("core", f"{self.test_home}/core.{test_name}") # Check any stderr output if os.path.isfile(stderr_expected_filename): if self._compare_files(stderr_expected_filename, stderr_output_filename): self._failed("stderr changed") did_fail = True elif os.stat(stderr_output_filename).st_size > 0: self._failed("Output was written to stderr") did_fail = True cat(stderr_output_filename) remove_files([ stderr_output_filename ]) # Check whether output graph exists, and normalize it if (not os.path.isfile(output_filename) or os.stat(output_filename).st_size == 0): self._error("No graph produced") did_fail = True self.num_failed = self.num_failed + 1 remove_files([ output_filename ]) return ExitStatus.ERROR normalize(output_filename) # Check whether dot output exists, and sort it if (not os.path.isfile(dot_output_filename) or os.stat(dot_output_filename).st_size == 0): self._error("No dot-file summary produced") did_fail = True self.num_failed = self.num_failed + 1 remove_files([ dot_output_filename, output_filename ]) return ExitStatus.ERROR with io.open(dot_output_filename, "rt") as f: first_line = f.readline() # "digraph" line with opening brace lines = f.readlines() last_line = lines[-1] # closing brace del lines[-1] lines = sorted(set(lines)) # unique sort with io.open(dot_output_filename, "wt") as f: f.write(first_line) f.writelines(lines) f.write(last_line) # Check whether score output exists, and sort it if (not os.path.isfile(score_output_filename) or os.stat(score_output_filename).st_size == 0): self._error("No allocation scores produced") did_fail = True self.num_failed = self.num_failed + 1 remove_files([ score_output_filename, output_filename ]) return ExitStatus.ERROR else: sort_file(score_output_filename) if self.args.update: shutil.copyfile(output_filename, expected_filename) shutil.copyfile(dot_output_filename, dot_expected_filename) shutil.copyfile(score_output_filename, scores_filename) shutil.copyfile(summary_output_filename, summary_filename) print(" Updated expected outputs") if self._compare_files(summary_filename, summary_output_filename): self._failed("summary changed") did_fail = True if self._compare_files(dot_expected_filename, dot_output_filename): self._failed("dot-file summary changed") did_fail = True else: remove_files([ dot_output_filename ]) if self._compare_files(expected_filename, output_filename): self._failed("xml-file changed") did_fail = True if self._compare_files(scores_filename, score_output_filename): self._failed("scores-file changed") did_fail = True remove_files([ output_filename, dot_output_filename, score_output_filename, summary_output_filename]) if did_fail: self.num_failed = self.num_failed + 1 return ExitStatus.ERROR return ExitStatus.OK def run_all(self): """ Run all defined tests """ if platform.architecture()[0] == "64bit": TESTS.extend(TESTS_64BIT) for group in TESTS: for test in group.tests: self.run_one(test.name, test.desc, test.args) print() def _print_summary(self): """ Print a summary of parameters for this test run """ print("Test home is:\t" + self.test_home) print("Test binary is:\t" + self.args.binary) if 'PCMK_schema_directory' in os.environ: print("Schema home is:\t" + os.environ['PCMK_schema_directory']) if self.valgrind_args != []: print("Activating memory testing with valgrind") print() def _test_results(self): if self.num_failed == 0: shutil.rmtree(self.failed_dir) return ExitStatus.OK if os.path.isfile(self.failed_filename) and os.stat(self.failed_filename).st_size != 0: if self.args.verbose: - self._error("Results of %d failed tests (out of %d):" % - (self.num_failed, self.num_tests)) + self._error(f"Results of {self.num_failed} failed tests (out of {self.num_tests}):") cat(self.failed_filename) else: - self._error("Results of %d failed tests (out of %d) are in %s" % - (self.num_failed, self.num_tests, self.failed_filename)) + self._error(f"Results of {self.num_failed} failed tests (out of {self.num_tests}) " + f"are in {self.failed_filename}") self._error("Use -V to display them after running the tests") else: - self._error("%d (of %d) tests failed (no diff results)" % - (self.num_failed, self.num_tests)) + self._error(f"{self.num_failed} (of {self.num_tests}) tests failed (no diff results)") if os.path.isfile(self.failed_filename): shutil.rmtree(self.failed_dir) return ExitStatus.ERROR def find_test(self, name): if platform.architecture()[0] == "64bit": TESTS.extend(TESTS_64BIT) for group in TESTS: for test in group.tests: if test.name == name: return test return None def run(self): """ Run test(s) as specified """ # Check for pre-existing core so we don't think it's from us if os.path.exists("core"): self._failed("Can't run with core already present in " + self.test_home) return ExitStatus.OSFILE self._print_summary() # Zero out the error log self.failed_file = io.open(self.failed_filename, "wt") if self.args.run is None: print("Performing the following tests from " + self.args.io_dir) print() self.run_all() print() self.failed_file.close() rc = self._test_results() else: # Find the test we were asked to run test = self.find_test(self.args.run) if test is None: - print("No test named %s" % self.args.run) + print(f"No test named {self.args.run}") return ExitStatus.INVALID_PARAM # If no arguments were given on the command line, default to the ones # contained in the test if self.single_test_args: args = self.single_test_args else: args = test.args rc = self.run_one(test.name, test.desc, args) self.failed_file.close() if self.num_failed > 0: print("\nFailures:\nThese have also been written to: " + self.failed_filename + "\n") cat(self.failed_filename) shutil.rmtree(self.failed_dir) return rc if __name__ == "__main__": sys.exit(CtsScheduler().run()) # vim: set filetype=python expandtab tabstop=4 softtabstop=4 shiftwidth=4 textwidth=120: