diff --git a/cts/cts-fencing.in b/cts/cts-fencing.in
index 3f1a22f2d1..c2ed29afda 100644
--- a/cts/cts-fencing.in
+++ b/cts/cts-fencing.in
@@ -1,1096 +1,1114 @@
 #!@PYTHON@
 """ Regression tests for Pacemaker's fencer
 """
 
 __copyright__ = "Copyright 2012-2023 the Pacemaker project contributors"
 __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
 
 import argparse
 import os
 import sys
 import subprocess
 import tempfile
 
 # These imports allow running from a source checkout after running `make`.
 # Note that while this doesn't necessarily mean it will successfully run tests,
 # but being able to see --help output can be useful.
 if os.path.exists("@abs_top_srcdir@/python"):
     sys.path.insert(0, "@abs_top_srcdir@/python")
 
 if os.path.exists("@abs_top_builddir@/python") and "@abs_top_builddir@" != "@abs_top_srcdir@":
     sys.path.insert(0, "@abs_top_builddir@/python")
 
 from pacemaker.buildoptions import BuildOptions
 from pacemaker.exitstatus import ExitStatus
 from pacemaker._cts.corosync import Corosync, localname
 from pacemaker._cts.errors import ExitCodeError, OutputFoundError, OutputNotFoundError, XmlValidationError
 from pacemaker._cts.process import killall, exit_if_proc_running
 from pacemaker._cts.test import Test, Tests
 
 TEST_DIR = sys.path[0]
 
 def update_path():
     """ Set the PATH environment variable appropriately for the tests """
 
     new_path = os.environ['PATH']
     if os.path.exists("%s/cts-fencing.in" % TEST_DIR):
         print("Running tests from the source tree: %s (%s)" % (BuildOptions._BUILD_DIR, TEST_DIR))
         # For pacemaker-fenced and cts-fence-helper
         new_path = "%s/daemons/fenced:%s" % (BuildOptions._BUILD_DIR, new_path)
         new_path = "%s/tools:%s" % (BuildOptions._BUILD_DIR, new_path) # For stonith_admin
         new_path = "%s/cts/support:%s" % (BuildOptions._BUILD_DIR, new_path) # For cts-support
 
     else:
         print("Running tests from the install tree: %s (not %s)" % (BuildOptions.DAEMON_DIR, TEST_DIR))
         # For pacemaker-fenced, cts-fence-helper, and cts-support
         new_path = "%s:%s" % (BuildOptions.DAEMON_DIR, new_path)
 
     print('Using PATH="%s"' % new_path)
     os.environ['PATH'] = new_path
 
 
 class FenceTest(Test):
     """ Executor for a single test """
 
     def __init__(self, name, description, **kwargs):
         Test.__init__(self, name, description, **kwargs)
 
         if kwargs.get("with_cpg", False):
             self._enable_corosync = True
             self._daemon_options = ["-c"]
         else:
             self._enable_corosync = False
             self._daemon_options = ["-s"]
 
         self._daemon_location = "pacemaker-fenced"
 
     def _kill_daemons(self):
         killall(["pacemakerd", "pacemaker-fenced"])
 
     def _start_daemons(self):
         if self.verbose:
             self._daemon_options += ["-V"]
             print("Starting %s with %s" % (self._daemon_location, self._daemon_options))
 
         cmd = ["pacemaker-fenced", "-l", self.logpath] + self._daemon_options
         self._daemon_process = subprocess.Popen(cmd)
 
 class FenceTests(Tests):
     """ Collection of all fencing regression tests """
 
     def __init__(self, **kwargs):
         Tests.__init__(self, **kwargs)
 
         self._corosync = Corosync(self.verbose, self.logdir, "cts-fencing")
 
     def new_test(self, name, description, with_cpg=False):
         """ Create a named test """
 
         test = FenceTest(name, description, verbose=self.verbose, with_cpg=with_cpg,
                          timeout=self.timeout, force_wait=self.force_wait,
                          logdir=self.logdir)
         self._tests.append(test)
         return test
 
     def run_cpg_only(self):
         """ Run all corosync-enabled tests """
 
         for test in self._tests:
             if test._enable_corosync:
                 test.run()
 
     def run_no_cpg(self):
         """ Run all standalone tests """
 
         for test in self._tests:
             if not test._enable_corosync:
                 test.run()
 
     def build_api_sanity_tests(self):
         """ Register tests to verify basic API usage """
 
         verbose_arg = ""
         if self.verbose:
             verbose_arg = "-V"
 
         test = self.new_test("standalone_low_level_api_test", "Sanity test client api in standalone mode.")
         test.add_cmd("cts-fence-helper", "-t %s" % (verbose_arg), validate=False)
 
         test = self.new_test("cpg_low_level_api_test", "Sanity test client api using mainloop and cpg.", True)
         test.add_cmd("cts-fence-helper", "-m %s" % (verbose_arg), validate=False)
 
     def build_custom_timeout_tests(self):
         """ Register tests to verify custom timeout usage """
 
         # custom timeout without topology
         test = self.new_test("cpg_custom_timeout_1",
                              "Verify per device timeouts work as expected without using topology.", True)
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R false1 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=node1 node2 node3"')
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R true1  -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node3" -o "pcmk_off_timeout=1"')
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R false2 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=node3" -o "pcmk_off_timeout=4"')
         test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5")
         # timeout is 5+1+4 = 10
         test.add_log_pattern("Total timeout set to 12")
 
         # custom timeout _WITH_ topology
         test = self.new_test("cpg_custom_timeout_2",
                              "Verify per device timeouts work as expected _WITH_ topology.", True)
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R false1 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=node1 node2 node3"')
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R true1  -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node3" -o "pcmk_off_timeout=1"')
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R false2 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=node3" -o "pcmk_off_timeout=4000"')
         test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false1")
         test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true1")
         test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 3 -v false2")
         test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5")
         # timeout is 5+1+4000 = 4006
         test.add_log_pattern("Total timeout set to 4807")
 
     def build_fence_merge_tests(self):
         """ Register tests to verify when fence operations should be merged """
 
         ### Simple test that overlapping fencing operations get merged
         test = self.new_test("cpg_custom_merge_single",
                              "Verify overlapping identical fencing operations are merged, no fencing levels used.", True)
         test.add_cmd("stonith_admin", "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3\"")
         test.add_cmd("stonith_admin", "--output-as=xml -R true1  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3\" ")
         test.add_cmd("stonith_admin", "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3\"")
         test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10")
         test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 10")
         ### one merger will happen
         test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
         ### the pattern below signifies that both the original and duplicate operation completed
         test.add_log_pattern("Operation 'off' targeting node3 by ")
         test.add_log_pattern("Operation 'off' targeting node3 by ")
 
         ### Test that multiple mergers occur
         test = self.new_test("cpg_custom_merge_multiple",
                              "Verify multiple overlapping identical fencing operations are merged", True)
         test.add_cmd("stonith_admin", "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3\"")
         test.add_cmd("stonith_admin",
                      "--output-as=xml -R true1  -a fence_dummy -o \"mode=pass\" -o \"delay=2\" -o \"pcmk_host_list=node3\" ")
         test.add_cmd("stonith_admin", "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3\"")
         test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10")
         test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10")
         test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10")
         test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10")
         test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 10")
         ### 4 mergers should occur
         test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
         test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
         test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
         test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
         ### the pattern below signifies that both the original and duplicate operation completed
         test.add_log_pattern("Operation 'off' targeting node3 by ")
         test.add_log_pattern("Operation 'off' targeting node3 by ")
         test.add_log_pattern("Operation 'off' targeting node3 by ")
         test.add_log_pattern("Operation 'off' targeting node3 by ")
         test.add_log_pattern("Operation 'off' targeting node3 by ")
 
         ### Test that multiple mergers occur with topologies used
         test = self.new_test("cpg_custom_merge_with_topology",
                              "Verify multiple overlapping identical fencing operations are merged with fencing levels.",
                              True)
         test.add_cmd("stonith_admin", "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3\"")
         test.add_cmd("stonith_admin", "--output-as=xml -R true1  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3\" ")
         test.add_cmd("stonith_admin", "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3\"")
         test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false1")
         test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false2")
         test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true1")
         test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10")
         test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10")
         test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10")
         test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10")
         test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 10")
         ### 4 mergers should occur
         test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
         test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
         test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
         test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
         ### the pattern below signifies that both the original and duplicate operation completed
         test.add_log_pattern("Operation 'off' targeting node3 by ")
         test.add_log_pattern("Operation 'off' targeting node3 by ")
         test.add_log_pattern("Operation 'off' targeting node3 by ")
         test.add_log_pattern("Operation 'off' targeting node3 by ")
         test.add_log_pattern("Operation 'off' targeting node3 by ")
 
     def build_fence_no_merge_tests(self):
         """ Register tests to verify when fence operations should not be merged """
 
         test = self.new_test("cpg_custom_no_merge",
                              "Verify differing fencing operations are not merged", True)
         test.add_cmd("stonith_admin", "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3 node2\"")
         test.add_cmd("stonith_admin", "--output-as=xml -R true1  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3 node2\" ")
         test.add_cmd("stonith_admin", "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3 node2\"")
         test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false1")
         test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false2")
         test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true1")
         test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node2 -t 10")
         test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 10")
         test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client",
                              negative=True)
 
     def build_standalone_tests(self):
         """ Register a grab bag of tests that can be executed in standalone or corosync mode """
 
         test_types = [
             {
                 "prefix" : "standalone",
                 "use_cpg" : False,
             },
             {
                 "prefix" : "cpg",
                 "use_cpg" : True,
             },
         ]
 
         # test what happens when all devices timeout
         for test_type in test_types:
             test = self.new_test("%s_fence_multi_device_failure" % test_type["prefix"],
                                  "Verify that all devices timeout, a fencing failure is returned.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R false2  -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R false3 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"")
             if test_type["use_cpg"]:
                 test.add_cmd_expected_fail("stonith_admin", "--output-as=xml -F node3 -t 2", ExitStatus.TIMEOUT)
                 test.add_log_pattern("Total timeout set to 7")
             else:
                 test.add_cmd_expected_fail("stonith_admin", "--output-as=xml -F node3 -t 2", ExitStatus.ERROR)
 
             test.add_log_pattern("targeting node3 using false1 returned ")
             test.add_log_pattern("targeting node3 using false2 returned ")
             test.add_log_pattern("targeting node3 using false3 returned ")
 
         # test what happens when multiple devices can fence a node, but the first device fails.
         for test_type in test_types:
             test = self.new_test("%s_fence_device_failure_rollover" % test_type["prefix"],
                                  "Verify that when one fence device fails for a node, the others are tried.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true1  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5")
 
             if test_type["use_cpg"]:
                 test.add_log_pattern("Total timeout set to 18")
 
         # test what happens when we try to use a missing fence-agent.
         for test_type in test_types:
             test = self.new_test("%s_fence_missing_agent" % test_type["prefix"],
                                  "Verify proper error-handling when using a non-existent fence-agent.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true1 -a fence_missing -o \"mode=pass\" -o \"pcmk_host_list=node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node2\"")
 
             test.add_cmd_expected_fail("stonith_admin",
                                        "--output-as=xml -F node3 -t 5",
                                        ExitStatus.NOSUCH)
             test.add_cmd("stonith_admin", "--output-as=xml -F node2 -t 5")
 
         # simple topology test for one device
         for test_type in test_types:
             if not test_type["use_cpg"]:
                 continue
 
             test = self.new_test("%s_topology_simple" % test_type["prefix"],
                                  "Verify all fencing devices at a level are used.", test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v true")
             test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5")
 
             test.add_log_pattern("Total timeout set to 6")
             test.add_log_pattern("targeting node3 using true returned 0")
 
 
         # add topology, delete topology, verify fencing still works
         for test_type in test_types:
             if not test_type["use_cpg"]:
                 continue
 
             test = self.new_test("%s_topology_add_remove" % test_type["prefix"],
                                  "Verify fencing occurrs after all topology levels are removed",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v true")
             test.add_cmd("stonith_admin", "--output-as=xml -d node3 -i 1")
             test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5")
 
             test.add_log_pattern("Total timeout set to 6")
             test.add_log_pattern("targeting node3 using true returned 0")
 
         # test what happens when the first fencing level has multiple devices.
         for test_type in test_types:
             if not test_type["use_cpg"]:
                 continue
 
             test = self.new_test("%s_topology_device_fails" % test_type["prefix"],
                                  "Verify if one device in a level fails, the other is tried.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R false  -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true")
             test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 20")
 
             test.add_log_pattern("Total timeout set to 48")
             test.add_log_pattern("targeting node3 using false returned 1")
             test.add_log_pattern("targeting node3 using true returned 0")
 
         # test what happens when the first fencing level fails.
         for test_type in test_types:
             if not test_type["use_cpg"]:
                 continue
 
             test = self.new_test("%s_topology_multi_level_fails" % test_type["prefix"],
                                  "Verify if one level fails, the next leve is tried.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true1  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true2  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true3  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true4  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"")
 
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false1")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v true1")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true2")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v false2")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 3 -v true3")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 3 -v true4")
 
             test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 3")
 
             test.add_log_pattern("Total timeout set to 21")
             test.add_log_pattern("targeting node3 using false1 returned 1")
             test.add_log_pattern("targeting node3 using false2 returned 1")
             test.add_log_pattern("targeting node3 using true3 returned 0")
             test.add_log_pattern("targeting node3 using true4 returned 0")
 
 
         # test what happens when the first fencing level had devices that no one has registered
         for test_type in test_types:
             if not test_type["use_cpg"]:
                 continue
 
             test = self.new_test("%s_topology_missing_devices" % test_type["prefix"],
                                  "Verify topology can continue with missing devices.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true2  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true3  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true4  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"")
 
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false1")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v true1")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true2")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v false2")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 3 -v true3")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 3 -v true4")
 
             test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5")
 
         # Test what happens if multiple fencing levels are defined, and then the first one is removed.
         for test_type in test_types:
             if not test_type["use_cpg"]:
                 continue
 
             test = self.new_test("%s_topology_level_removal" % test_type["prefix"],
                                  "Verify level removal works.", test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true1  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true2  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true3  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true4  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"")
 
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false1")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v true1")
 
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true2")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v false2")
 
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 3 -v true3")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 3 -v true4")
 
             # Now remove level 2, verify none of the devices in level two are hit.
             test.add_cmd("stonith_admin", "--output-as=xml -d node3 -i 2")
 
             test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 20")
 
             test.add_log_pattern("Total timeout set to 96")
             test.add_log_pattern("targeting node3 using false1 returned 1")
             test.add_log_pattern("targeting node3 using false2 returned ",
                                  negative=True)
             test.add_log_pattern("targeting node3 using true3 returned 0")
             test.add_log_pattern("targeting node3 using true4 returned 0")
 
         # Test targeting a topology level by node name pattern.
         for test_type in test_types:
             if not test_type["use_cpg"]:
                 continue
 
             test = self.new_test("%s_topology_level_pattern" % test_type["prefix"],
                                  "Verify targeting topology by node name pattern works.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          """--output-as=xml -R true -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node1 node2 node3" """)
             test.add_cmd("stonith_admin", """--output-as=xml -r '@node.*' -i 1 -v true""")
             test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5")
             test.add_log_pattern("targeting node3 using true returned 0")
 
         # test allowing commas and semicolons as delimiters in pcmk_host_list
         for test_type in test_types:
             test = self.new_test("%s_host_list_delimiters" % test_type["prefix"],
                                  "Verify commas and semicolons can be used as pcmk_host_list delimiters",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          """--output-as=xml -R true1 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node1,node2,node3" """)
             test.add_cmd("stonith_admin",
                          """--output-as=xml -R true2 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=pcmk1;pcmk2;pcmk3" """)
             test.add_cmd("stonith_admin", "stonith_admin --output-as=xml -F node2 -t 5")
             test.add_cmd("stonith_admin", "stonith_admin --output-as=xml -F pcmk3 -t 5")
             test.add_log_pattern("targeting node2 using true1 returned 0")
             test.add_log_pattern("targeting pcmk3 using true2 returned 0")
 
         # test the stonith builds the correct list of devices that can fence a node.
         for test_type in test_types:
             test = self.new_test("%s_list_devices" % test_type["prefix"],
                                  "Verify list of devices that can fence a node is correct",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true1  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true3 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd_check_stdout("stonith_admin", "--output-as=xml -l node1 -V", "true2", "true1")
             test.add_cmd_check_stdout("stonith_admin", "--output-as=xml -l node1 -V", "true3", "true1")
 
         # simple test of device monitor
         for test_type in test_types:
             test = self.new_test("%s_monitor" % test_type["prefix"],
                                  "Verify device is reachable", test_type["use_cpg"])
             test.add_cmd("stonith_admin", "--output-as=xml -R true1  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3\"")
             test.add_cmd("stonith_admin", "--output-as=xml -R false1  -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3\"")
 
             test.add_cmd("stonith_admin", "--output-as=xml -Q true1")
             test.add_cmd("stonith_admin", "--output-as=xml -Q false1")
             test.add_cmd_expected_fail("stonith_admin",
                                        "--output-as=xml -Q true2",
                                        ExitStatus.NOSUCH)
 
         # Verify monitor occurs for duration of timeout period on failure
         for test_type in test_types:
             test = self.new_test("%s_monitor_timeout" % test_type["prefix"],
                                  "Verify monitor uses duration of timeout period given.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          '--output-as=xml -R true1 -a fence_dummy -o "mode=fail" -o "monitor_mode=fail" -o "pcmk_host_list=node3"')
             test.add_cmd_expected_fail("stonith_admin", "--output-as=xml -Q true1 -t 5", ExitStatus.ERROR)
             test.add_log_pattern("Attempt 2 to execute")
 
         # Verify monitor occurs for duration of timeout period on failure, but stops at max retries
         for test_type in test_types:
             test = self.new_test("%s_monitor_timeout_max_retries" % test_type["prefix"],
                                  "Verify monitor retries until max retry value or timeout is hit.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          '--output-as=xml -R true1  -a fence_dummy -o "mode=fail" -o "monitor_mode=fail" -o "pcmk_host_list=node3"')
             test.add_cmd_expected_fail("stonith_admin", "--output-as=xml -Q true1 -t 15", ExitStatus.ERROR)
             test.add_log_pattern("Attempted to execute agent fence_dummy (list) the maximum number of times")
 
         # simple register test
         for test_type in test_types:
             test = self.new_test("%s_register" % test_type["prefix"],
                                  "Verify devices can be registered and un-registered",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin", "--output-as=xml -R true1  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3\"")
 
             test.add_cmd("stonith_admin", "--output-as=xml -Q true1")
 
             test.add_cmd("stonith_admin", "--output-as=xml -D true1")
 
             test.add_cmd_expected_fail("stonith_admin",
                                        "--output-as=xml -Q true1",
                                        ExitStatus.NOSUCH)
 
         # simple reboot test
         for test_type in test_types:
             test = self.new_test("%s_reboot" % test_type["prefix"],
                                  "Verify devices can be rebooted",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin", "--output-as=xml -R true1  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3\"")
 
             test.add_cmd("stonith_admin", "--output-as=xml -B node3 -t 5")
 
             test.add_cmd("stonith_admin", "--output-as=xml -D true1")
 
             test.add_cmd_expected_fail("stonith_admin",
                                        "--output-as=xml -Q true1",
                                        ExitStatus.NOSUCH)
 
         # test fencing history.
         for test_type in test_types:
             if not test_type["use_cpg"]:
                 continue
             test = self.new_test("%s_fence_history" % test_type["prefix"],
                                  "Verify last fencing operation is returned.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin", "--output-as=xml -R true1  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3\"")
 
             test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5 -V")
 
             test.add_cmd_check_stdout("stonith_admin", "--output-as=xml -H node3", 'action="off" target="node3" .* status="success"')
 
         # simple test of dynamic list query
         for test_type in test_types:
             test = self.new_test("%s_dynamic_list_query" % test_type["prefix"],
                                  "Verify dynamic list of fencing devices can be retrieved.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin", "--output-as=xml -R true1 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1")
             test.add_cmd("stonith_admin", "--output-as=xml -R true2 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1")
             test.add_cmd("stonith_admin", "--output-as=xml -R true3 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1")
 
             test.add_cmd_check_stdout("stonith_admin", "--output-as=xml -l fake_port_1", 'count="3"')
 
 
         # fence using dynamic list query
         for test_type in test_types:
             test = self.new_test("%s_fence_dynamic_list_query" % test_type["prefix"],
                                  "Verify dynamic list of fencing devices can be retrieved.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin", "--output-as=xml -R true1 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1")
             test.add_cmd("stonith_admin", "--output-as=xml -R true2 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1")
             test.add_cmd("stonith_admin", "--output-as=xml -R true3 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1")
 
             test.add_cmd("stonith_admin", "--output-as=xml -F fake_port_1 -t 5 -V")
 
         # simple test of  query using status action
         for test_type in test_types:
             test = self.new_test("%s_status_query" % test_type["prefix"],
                                  "Verify dynamic list of fencing devices can be retrieved.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin", "--output-as=xml -R true1  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_check=status\"")
             test.add_cmd("stonith_admin", "--output-as=xml -R true2  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_check=status\"")
             test.add_cmd("stonith_admin", "--output-as=xml -R true3  -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_check=status\"")
 
             test.add_cmd_check_stdout("stonith_admin", "--output-as=xml -l fake_port_1", 'count="3"')
 
         # test what happens when no reboot action is advertised
         for test_type in test_types:
             test = self.new_test("%s_no_reboot_support" % test_type["prefix"],
                                  "Verify reboot action defaults to off when no reboot action is advertised by agent.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true1 -a fence_dummy_no_reboot -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin", "--output-as=xml -B node1 -t 5 -V")
             test.add_log_pattern("does not support reboot")
             test.add_log_pattern("using true1 returned 0")
 
         # make sure reboot is used when reboot action is advertised
         for test_type in test_types:
             test = self.new_test("%s_with_reboot_support" % test_type["prefix"],
                                  "Verify reboot action can be used when metadata advertises it.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
             test.add_cmd("stonith_admin", "--output-as=xml -B node1 -t 5 -V")
             test.add_log_pattern("does not advertise support for 'reboot', performing 'off'",
                                  negative=True)
             test.add_log_pattern("using true1 returned 0")
 
-        # make sure requested fencing delay is applied only for the first device in the first level
-        # make sure static delay from pcmk_delay_base is added
+        # make sure all fencing delays are applied correctly and taken into account by fencing timeouts with topology
         for test_type in test_types:
             if not test_type["use_cpg"]:
                 continue
 
-            test = self.new_test("%s_topology_delay" % test_type["prefix"],
-                                 "Verify requested fencing delay is applied only for the first device in the first level and pcmk_delay_base is added.",
+            test = self.new_test("%s_topology_delays" % test_type["prefix"],
+                                 "Verify all fencing delays are applied correctly and taken into account by fencing timeouts with topology.",
                                  test_type["use_cpg"])
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\" -o \"pcmk_delay_base=1\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\" -o \"pcmk_delay_base=1\"")
+            # Resulting "random" delay will always be 1 since (rand() % (delay_max - delay_base)) is always 0 here.
             test.add_cmd("stonith_admin",
-                         "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
+                         "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\" -o \"pcmk_delay_base=1\" -o \"pcmk_delay_max=2\"")
             test.add_cmd("stonith_admin",
                          "--output-as=xml -R true3 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"")
 
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v true1")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false1")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true2")
             test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true3")
 
             test.add_cmd("stonith_admin", "--output-as=xml -F node3 --delay 1")
 
+            # Total fencing timeout takes all fencing delays into account.
+            test.add_log_pattern("Total timeout set to 582")
+
+            # Fencing timeout for the first device takes the requested fencing delay into account.
+            # Fencing timeout also takes pcmk_delay_base into account.
+            test.add_log_pattern(r"Requesting that .* perform 'off' action targeting node3 using true1 .*146s.*",
+                                 regex=True)
+            # Requested fencing delay is applied only for the first device in the first level.
+            # Static delay from pcmk_delay_base is added.
             test.add_log_pattern("Delaying 'off' action targeting node3 using true1 for 2s | timeout=120s requested_delay=1s base=1s max=1s")
+
+            # Fencing timeout no longer takes the requested fencing delay into account for further devices.
+            test.add_log_pattern(r"Requesting that .* perform 'off' action targeting node3 using false1 .*145s.*",
+                                 regex=True)
+            # Requested fencing delay is no longer applied for further devices.
             test.add_log_pattern("Delaying 'off' action targeting node3 using false1 for 1s | timeout=120s requested_delay=0s base=1s max=1s")
-            test.add_log_pattern("Delaying 'off' action targeting node3 using true2",
-                                 negative=True)
+
+            # Fencing timeout takes pcmk_delay_max into account.
+            test.add_log_pattern(r"Requesting that .* perform 'off' action targeting node3 using true2 .*146s.*",
+                                 regex=True)
+            test.add_log_pattern("Delaying 'off' action targeting node3 using true2 for 1s | timeout=120s requested_delay=0s base=1s max=2s")
+
             test.add_log_pattern("Delaying 'off' action targeting node3 using true3",
                                  negative=True)
 
     def build_nodeid_tests(self):
         """ Register tests that use a corosync node id """
 
         our_uname = localname()
 
         ### verify nodeid is supplied when nodeid is in the metadata parameters
         test = self.new_test("cpg_supply_nodeid",
                              "Verify nodeid is given when fence agent has nodeid as parameter", True)
 
         test.add_cmd("stonith_admin",
                      "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s\"" % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -F %s -t 3" % (our_uname))
         test.add_log_pattern("as nodeid with fence action 'off' targeting %s" % (our_uname))
 
         ### verify nodeid is _NOT_ supplied when nodeid is not in the metadata parameters
         test = self.new_test("cpg_do_not_supply_nodeid",
                              "Verify nodeid is _NOT_ given when fence agent does not have nodeid as parameter",
                              True)
 
         # use a host name that won't be in corosync.conf
         test.add_cmd("stonith_admin",
                      "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=regr-test\"")
         test.add_cmd("stonith_admin", "--output-as=xml -F regr-test -t 3")
         test.add_log_pattern("as nodeid with fence action 'off' targeting regr-test",
                              negative=True)
 
         ### verify nodeid use doesn't explode standalone mode
         test = self.new_test("standalone_do_not_supply_nodeid",
                              "Verify nodeid in metadata parameter list doesn't kill standalone mode",
                              False)
 
         test.add_cmd("stonith_admin",
                      "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s\"" % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -F %s -t 3" % (our_uname))
         test.add_log_pattern("as nodeid with fence action 'off' targeting %s" % our_uname,
                              negative=True)
 
     def build_unfence_tests(self):
         """ Register tests that verify unfencing """
 
         our_uname = localname()
 
         ### verify unfencing using automatic unfencing
         test = self.new_test("cpg_unfence_required_1",
                              "Verify require unfencing on all devices when automatic=true in agent's metadata",
                              True)
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R true1 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s"' % (our_uname))
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R true2 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s"' % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -U %s -t 3" % (our_uname))
         # both devices should be executed
         test.add_log_pattern("using true1 returned 0")
         test.add_log_pattern("using true2 returned 0")
 
         ### verify unfencing using automatic unfencing fails if any of the required agents fail
         test = self.new_test("cpg_unfence_required_2",
                              "Verify require unfencing on all devices when automatic=true in agent's metadata",
                              True)
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R true1 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s"' % (our_uname))
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R true2 -a fence_dummy_auto_unfence -o "mode=fail" -o "pcmk_host_list=%s"' % (our_uname))
         test.add_cmd_expected_fail("stonith_admin", "--output-as=xml -U %s -t 6" % (our_uname), ExitStatus.ERROR)
 
         ### verify unfencing using automatic devices with topology
         test = self.new_test("cpg_unfence_required_3",
                              "Verify require unfencing on all devices even when at different topology levels",
                              True)
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R true1 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s node3"' % (our_uname))
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R true2 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s node3"' % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 1 -v true1" % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 2 -v true2" % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -U %s -t 3" % (our_uname))
         test.add_log_pattern("using true1 returned 0")
         test.add_log_pattern("using true2 returned 0")
 
         ### verify unfencing using automatic devices with topology
         test = self.new_test("cpg_unfence_required_4",
                              "Verify all required devices are executed even with topology levels fail.",
                              True)
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R true1 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s node3"' % (our_uname))
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R true2 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s node3"' % (our_uname))
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R true3 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s node3"' % (our_uname))
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R true4 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s node3"' % (our_uname))
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R false1 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=%s node3"' % (our_uname))
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R false2 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=%s node3"' % (our_uname))
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R false3 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=%s node3"' % (our_uname))
         test.add_cmd('stonith_admin',
                      '--output-as=xml -R false4 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=%s node3"' % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 1 -v true1" % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 1 -v false1" % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 2 -v false2" % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 2 -v true2" % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 2 -v false3" % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 2 -v true3" % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 3 -v false4" % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 4 -v true4" % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -U %s -t 3" % (our_uname))
         test.add_log_pattern("using true1 returned 0")
         test.add_log_pattern("using true2 returned 0")
         test.add_log_pattern("using true3 returned 0")
         test.add_log_pattern("using true4 returned 0")
 
     def build_unfence_on_target_tests(self):
         """ Register tests that verify unfencing that runs on the target """
 
         our_uname = localname()
 
         ### verify unfencing using on_target device
         test = self.new_test("cpg_unfence_on_target_1",
                              "Verify unfencing with on_target = true", True)
         test.add_cmd("stonith_admin",
                      "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s\"" % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -U %s -t 3" % (our_uname))
         test.add_log_pattern("(on) to be executed on target")
 
         ### verify failure of unfencing using on_target device
         test = self.new_test("cpg_unfence_on_target_2",
                              "Verify failure unfencing with on_target = true",
                              True)
         test.add_cmd("stonith_admin",
                      "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s node_fake_1234\"" % (our_uname))
         test.add_cmd_expected_fail("stonith_admin",
                                    "--output-as=xml -U node_fake_1234 -t 3",
                                    ExitStatus.NOSUCH)
         test.add_log_pattern("(on) to be executed on target")
 
         ### verify unfencing using on_target device with topology
         test = self.new_test("cpg_unfence_on_target_3",
                              "Verify unfencing with on_target = true using topology",
                              True)
 
         test.add_cmd("stonith_admin",
                      "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s node3\"" % (our_uname))
         test.add_cmd("stonith_admin",
                      "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s node3\"" % (our_uname))
 
         test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 1 -v true1" % (our_uname))
         test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 2 -v true2" % (our_uname))
 
         test.add_cmd("stonith_admin", "--output-as=xml -U %s -t 3" % (our_uname))
         test.add_log_pattern("(on) to be executed on target")
 
         ### verify unfencing using on_target device with topology fails when target node doesn't exist
         test = self.new_test("cpg_unfence_on_target_4",
                              "Verify unfencing failure with on_target = true using topology",
                              True)
 
         test.add_cmd("stonith_admin",
                      "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s node_fake\"" % (our_uname))
         test.add_cmd("stonith_admin",
                      "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s node_fake\"" % (our_uname))
 
         test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 1 -v true1")
         test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 2 -v true2")
 
         test.add_cmd_expected_fail("stonith_admin",
                                    "--output-as=xml -U node_fake -t 3",
                                    ExitStatus.NOSUCH)
         test.add_log_pattern("(on) to be executed on target")
 
     def build_remap_tests(self):
         """ Register tests that verify remapping of reboots to off-on """
 
         test = self.new_test("cpg_remap_simple",
                              "Verify sequential topology reboot is remapped to all-off-then-all-on", True)
         test.add_cmd("stonith_admin",
                      """--output-as=xml -R true1 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node_fake" """
                      """-o "pcmk_off_timeout=1" -o "pcmk_reboot_timeout=10" """)
         test.add_cmd("stonith_admin",
                      """--output-as=xml -R true2 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node_fake" """
                      """-o "pcmk_off_timeout=2" -o "pcmk_reboot_timeout=20" """)
         test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 1 -v true1 -v true2")
         test.add_cmd("stonith_admin", "--output-as=xml -B node_fake -t 5")
         test.add_log_pattern("Remapping multiple-device reboot targeting node_fake")
         # timeout should be sum of off timeouts (1+2=3), not reboot timeouts (10+20=30)
         test.add_log_pattern("Total timeout set to 3 for peer's fencing targeting node_fake")
         test.add_log_pattern("perform 'off' action targeting node_fake using true1")
         test.add_log_pattern("perform 'off' action targeting node_fake using true2")
         test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'")
         # fence_dummy sets "on" as an on_target action
         test.add_log_pattern("Ignoring true1 'on' failure (no capable peers) targeting node_fake")
         test.add_log_pattern("Ignoring true2 'on' failure (no capable peers) targeting node_fake")
         test.add_log_pattern("Undoing remap of reboot targeting node_fake")
 
         test = self.new_test("cpg_remap_simple_off",
                              "Verify sequential topology reboot skips 'on' if "
                              "pcmk_reboot_action=off or agent doesn't support "
                              "'on'", True)
         test.add_cmd("stonith_admin",
                      "--output-as=xml -R true1 -a fence_dummy -o mode=pass "
                      "-o pcmk_host_list=node_fake -o pcmk_off_timeout=1 "
                      "-o pcmk_reboot_timeout=10 -o pcmk_reboot_action=off")
         test.add_cmd("stonith_admin",
                      "--output-as=xml -R true2 -a fence_dummy_no_on "
                      "-o mode=pass -o pcmk_host_list=node_fake "
                      "-o pcmk_off_timeout=2 -o pcmk_reboot_timeout=20")
         test.add_cmd("stonith_admin",
                      "--output-as=xml -r node_fake -i 1 -v true1 -v true2")
         test.add_cmd("stonith_admin", "--output-as=xml -B node_fake -t 5")
         test.add_log_pattern("Remapping multiple-device reboot targeting node_fake")
         # timeout should be sum of off timeouts (1+2=3), not reboot timeouts (10+20=30)
         test.add_log_pattern("Total timeout set to 3 for peer's fencing targeting node_fake")
         test.add_log_pattern("perform 'off' action targeting node_fake using true1")
         test.add_log_pattern("perform 'off' action targeting node_fake using true2")
         test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'")
         # "on" should be skipped
         test.add_log_pattern("Not turning node_fake back on using "
                          "true1 because the device is configured "
                          "to stay off")
         test.add_log_pattern("Not turning node_fake back on using true2"
                          " because the agent doesn't support 'on'")
         test.add_log_pattern("Undoing remap of reboot targeting node_fake")
 
         test = self.new_test("cpg_remap_automatic",
                              "Verify remapped topology reboot skips automatic 'on'", True)
         test.add_cmd("stonith_admin",
                      """--output-as=xml -R true1 -a fence_dummy_auto_unfence """
                      """-o "mode=pass" -o "pcmk_host_list=node_fake" """)
         test.add_cmd("stonith_admin",
                      """--output-as=xml -R true2 -a fence_dummy_auto_unfence """
                      """-o "mode=pass" -o "pcmk_host_list=node_fake" """)
         test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 1 -v true1 -v true2")
         test.add_cmd("stonith_admin", "--output-as=xml -B node_fake -t 5")
         test.add_log_pattern("Remapping multiple-device reboot targeting node_fake")
         test.add_log_pattern("perform 'off' action targeting node_fake using true1")
         test.add_log_pattern("perform 'off' action targeting node_fake using true2")
         test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'")
         test.add_log_pattern("Undoing remap of reboot targeting node_fake")
         test.add_log_pattern("perform 'on' action targeting node_fake using",
                              negative=True)
         test.add_log_pattern("'on' failure",
                              negative=True)
 
         test = self.new_test("cpg_remap_complex_1",
                              "Verify remapped topology reboot in second level works if non-remapped first level fails",
                              True)
         test.add_cmd("stonith_admin", """--output-as=xml -R false1 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=node_fake" """)
         test.add_cmd("stonith_admin", """--output-as=xml -R true1 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node_fake" """)
         test.add_cmd("stonith_admin", """--output-as=xml -R true2 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node_fake" """)
         test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 1 -v false1")
         test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 2 -v true1 -v true2")
         test.add_cmd("stonith_admin", "--output-as=xml -B node_fake -t 5")
         test.add_log_pattern("perform 'reboot' action targeting node_fake using false1")
         test.add_log_pattern("Remapping multiple-device reboot targeting node_fake")
         test.add_log_pattern("perform 'off' action targeting node_fake using true1")
         test.add_log_pattern("perform 'off' action targeting node_fake using true2")
         test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'")
         test.add_log_pattern("Ignoring true1 'on' failure (no capable peers) targeting node_fake")
         test.add_log_pattern("Ignoring true2 'on' failure (no capable peers) targeting node_fake")
         test.add_log_pattern("Undoing remap of reboot targeting node_fake")
 
         test = self.new_test("cpg_remap_complex_2",
                              "Verify remapped topology reboot failure in second level proceeds to third level",
                              True)
         test.add_cmd("stonith_admin", """--output-as=xml -R false1 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=node_fake" """)
         test.add_cmd("stonith_admin", """--output-as=xml -R false2 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=node_fake" """)
         test.add_cmd("stonith_admin", """--output-as=xml -R true1 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node_fake" """)
         test.add_cmd("stonith_admin", """--output-as=xml -R true2 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node_fake" """)
         test.add_cmd("stonith_admin", """--output-as=xml -R true3 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node_fake" """)
         test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 1 -v false1")
         test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 2 -v true1 -v false2 -v true3")
         test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 3 -v true2")
         test.add_cmd("stonith_admin", "--output-as=xml -B node_fake -t 5")
         test.add_log_pattern("perform 'reboot' action targeting node_fake using false1")
         test.add_log_pattern("Remapping multiple-device reboot targeting node_fake")
         test.add_log_pattern("perform 'off' action targeting node_fake using true1")
         test.add_log_pattern("perform 'off' action targeting node_fake using false2")
         test.add_log_pattern("Attempted to execute agent fence_dummy (off) the maximum number of times")
         test.add_log_pattern("Undoing remap of reboot targeting node_fake")
         test.add_log_pattern("perform 'reboot' action targeting node_fake using true2")
         test.add_log_pattern("node_fake with true3",
                              negative=True)
 
     def build_query_tests(self):
         """ run stonith_admin --metadata for the fence_dummy agent and check command output """
 
         test = self.new_test("get_metadata",
                              "Run stonith_admin --metadata for the fence_dummy agent", True)
         test.add_cmd_check_stdout("stonith_admin", "--output-as=xml -a fence_dummy --metadata", '<shortdesc lang')
 
     def build_metadata_tests(self):
         """ run fence-agents coming with pacemaker with -o metadata and check for valid xml """
 
         test = self.new_test("check_metadata_dummy",
                              "Run fence_dummy -o metadata and check for valid xml", False)
         test.add_cmd("fence_dummy", "-o metadata", check_rng=False, check_stderr=False)
         # fence_dummy prints on stderr to check that tools just listen on stdout
 
         test = self.new_test("check_metadata_watchdog",
                              "Run fence_watchdog -o metadata and check for valid xml", False)
         test.add_cmd("fence_watchdog", "-o metadata", check_rng=False)
 
     def build_validate_tests(self):
         """ run stonith_admin --validate for the fence_dummy agent and check command output """
 
         test = self.new_test("validate_dummy",
                              "Run stonith_admin --validate-all and check output", False)
         test.add_cmd_expected_fail("stonith_admin", "-a fence_dummy --validate --output-as=xml")
         test.add_cmd("stonith_admin", """-a fence_dummy --validate -o "delay=5" --output-as=xml""", check_rng=False)
         test.add_cmd_expected_fail("stonith_admin", """-a fence_dummy --validate -o "delay=15" --output-as=xml""")
 
     def setup_environment(self, use_corosync):
         """ Prepare the host before executing any tests """
 
         if use_corosync:
             self._corosync.start(kill_first=True)
 
         subprocess.call(["cts-support", "install"])
 
     def cleanup_environment(self, use_corosync):
         """ Clean up the host after executing desired tests """
 
         if use_corosync:
             self._corosync.stop()
 
         subprocess.call(["cts-support", "uninstall"])
 
 
 def build_options():
     parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
                                      description="Run pacemaker-fenced regression tests",
                                      epilog="Example: Run only the test 'start_stop'\n"
                                             "\t " + sys.argv[0] + " --run-only start_stop\n\n"
                                             "Example: Run only the tests with the string 'systemd' present in them\n"
                                             "\t " + sys.argv[0] + " --run-only-pattern systemd")
     parser.add_argument("-c", "--cpg-only", action="store_true",
                         help="Only run tests that require corosync")
     parser.add_argument("-l", "--list-tests", action="store_true",
                         help="Print out all registered tests")
     parser.add_argument("-n", "--no-cpg", action="store_true",
                         help="Only run tests that do not require corosync")
     parser.add_argument("-p", "--run-only-pattern", metavar='PATTERN',
                         help="Run only tests matching the given pattern")
     parser.add_argument("-r", "--run-only", metavar='TEST',
                         help="Run a specific test")
     parser.add_argument("-t", "--timeout", type=float, default=2,
                         help="Up to how many seconds each test case waits for the daemon to "
                              "be initialized.  Defaults to 2.  The value 0 means no limit.")
     parser.add_argument("-w", "--force-wait", action="store_true",
                         help="Each test case waits the default/specified --timeout for the "
                              "daemon without tracking the log")
     parser.add_argument("-V", "--verbose", action="store_true",
                         help="Verbose output")
 
     args = parser.parse_args()
     return args
 
 
 def main():
     """ Run fencing regression tests as specified by arguments """
 
     update_path()
 
     # Ensure all command output is in portable locale for comparison
     os.environ['LC_ALL'] = "C"
 
     opts = build_options()
 
     exit_if_proc_running("pacemaker-fenced")
 
     use_corosync = not opts.no_cpg
     if use_corosync:
         exit_if_proc_running("corosync")
 
     # Create a temporary directory for log files (the directory and its
     # contents will automatically be erased when done)
     with tempfile.TemporaryDirectory(prefix="cts-fencing-") as logdir:
         tests = FenceTests(verbose=opts.verbose, timeout=opts.timeout,
                            force_wait=opts.force_wait, logdir=logdir)
 
         tests.build_standalone_tests()
         tests.build_custom_timeout_tests()
         tests.build_api_sanity_tests()
         tests.build_fence_merge_tests()
         tests.build_fence_no_merge_tests()
         tests.build_unfence_tests()
         tests.build_unfence_on_target_tests()
         tests.build_nodeid_tests()
         tests.build_remap_tests()
         tests.build_query_tests()
         tests.build_metadata_tests()
         tests.build_validate_tests()
 
         if opts.list_tests:
             tests.print_list()
             sys.exit(ExitStatus.OK)
 
         print("Starting ...")
 
         try:
             tests.setup_environment(use_corosync)
         except TimeoutError:
             print("corosync did not start in time, exiting")
             sys.exit(ExitStatus.TIMEOUT)
 
         if opts.run_only_pattern:
             tests.run_tests_matching(opts.run_only_pattern)
             tests.print_results()
         elif opts.run_only:
             tests.run_single(opts.run_only)
             tests.print_results()
         elif opts.no_cpg:
             tests.run_no_cpg()
             tests.print_results()
         elif opts.cpg_only:
             tests.run_cpg_only()
             tests.print_results()
         else:
             tests.run_tests()
             tests.print_results()
 
         tests.cleanup_environment(use_corosync)
 
     tests.exit()
 
 
 if __name__ == "__main__":
     main()
diff --git a/cts/lab/CTStests.py b/cts/lab/CTStests.py
index 7a85be55e6..61766ce188 100644
--- a/cts/lab/CTStests.py
+++ b/cts/lab/CTStests.py
@@ -1,3178 +1,3178 @@
 """ Test-specific classes for Pacemaker's Cluster Test Suite (CTS)
 """
 
 __copyright__ = "Copyright 2000-2023 the Pacemaker project contributors"
 __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
 
 #
 #        SPECIAL NOTE:
 #
 #        Tests may NOT implement any cluster-manager-specific code in them.
 #        EXTEND the ClusterManager object to provide the base capabilities
 #        the test needs if you need to do something that the current CM classes
 #        do not.  Otherwise you screw up the whole point of the object structure
 #        in CTS.
 #
 #                Thank you.
 #
 
 import os
 import re
 import time
 import subprocess
 import tempfile
 
 from stat import *
 from cts.CTSaudits import *
 
 from pacemaker import BuildOptions
 from pacemaker._cts.CTS import NodeStatus
 from pacemaker._cts.environment import EnvFactory
 from pacemaker._cts.logging import LogFactory
 from pacemaker._cts.patterns import PatternSelector
 from pacemaker._cts.remote import RemoteFactory
 from pacemaker._cts.watcher import LogWatcher
 
 AllTestClasses = [ ]
 
 
 class CTSTest(object):
     '''
     A Cluster test.
     We implement the basic set of properties and behaviors for a generic
     cluster test.
 
     Cluster tests track their own statistics.
     We keep each of the kinds of counts we track as separate {name,value}
     pairs.
     '''
 
     def __init__(self, cm):
         #self.name="the unnamed test"
         self.Stats = {"calls":0
         ,        "success":0
         ,        "failure":0
         ,        "skipped":0
         ,        "auditfail":0}
 
 #        if not issubclass(cm.__class__, ClusterManager):
 #            raise ValueError("Must be a ClusterManager object")
         self.CM = cm
         self.Env = EnvFactory().getInstance()
         self.rsh = RemoteFactory().getInstance()
         self.logger = LogFactory()
         self.templates = PatternSelector(cm["Name"])
         self.Audits = []
         self.timeout = 120
         self.passed = 1
         self.is_loop = 0
         self.is_unsafe = 0
         self.is_experimental = 0
         self.is_container = 0
         self.is_valgrind = 0
         self.benchmark = 0  # which tests to benchmark
         self.timer = {}  # timers
 
     def log(self, args):
         self.logger.log(args)
 
     def debug(self, args):
         self.logger.debug(args)
 
     def has_key(self, key):
         return key in self.Stats
 
     def __setitem__(self, key, value):
         self.Stats[key] = value
 
     def __getitem__(self, key):
         if str(key) == "0":
             raise ValueError("Bad call to 'foo in X', should reference 'foo in X.Stats' instead")
 
         if key in self.Stats:
             return self.Stats[key]
         return None
 
     def log_mark(self, msg):
         self.debug("MARK: test %s %s %d" % (self.name,msg,time.time()))
         return
 
     def get_timer(self,key = "test"):
         try: return self.timer[key]
         except: return 0
 
     def set_timer(self,key = "test"):
         self.timer[key] = time.time()
         return self.timer[key]
 
     def log_timer(self,key = "test"):
         elapsed = 0
         if key in self.timer:
             elapsed = time.time() - self.timer[key]
             s = key == "test" and self.name or "%s:%s" % (self.name,key)
             self.debug("%s runtime: %.2f" % (s, elapsed))
             del self.timer[key]
         return elapsed
 
     def incr(self, name):
         '''Increment (or initialize) the value associated with the given name'''
         if not name in self.Stats:
             self.Stats[name] = 0
         self.Stats[name] = self.Stats[name]+1
 
         # Reset the test passed boolean
         if name == "calls":
             self.passed = 1
 
     def failure(self, reason="none"):
         '''Increment the failure count'''
         self.passed = 0
         self.incr("failure")
         self.logger.log(("Test %s" % self.name).ljust(35) + " FAILED: %s" % reason)
         return None
 
     def success(self):
         '''Increment the success count'''
         self.incr("success")
         return 1
 
     def skipped(self):
         '''Increment the skipped count'''
         self.incr("skipped")
         return 1
 
     def __call__(self, node):
         '''Perform the given test'''
         raise ValueError("Abstract Class member (__call__)")
         self.incr("calls")
         return self.failure()
 
     def audit(self):
         passed = 1
         if len(self.Audits) > 0:
             for audit in self.Audits:
                 if not audit():
                     self.logger.log("Internal %s Audit %s FAILED." % (self.name, audit.name()))
                     self.incr("auditfail")
                     passed = 0
         return passed
 
     def setup(self, node):
         '''Setup the given test'''
         return self.success()
 
     def teardown(self, node):
         '''Tear down the given test'''
         return self.success()
 
     def create_watch(self, patterns, timeout, name=None):
         if not name:
             name = self.name
         return LogWatcher(self.Env["LogFileName"], patterns, self.Env["nodes"], self.Env["LogWatcher"], name, timeout)
 
     def local_badnews(self, prefix, watch, local_ignore=[]):
         errcount = 0
         if not prefix:
             prefix = "LocalBadNews:"
 
         ignorelist = []
         ignorelist.append(" CTS: ")
         ignorelist.append(prefix)
         ignorelist.extend(local_ignore)
 
         while errcount < 100:
             match = watch.look(0)
             if match:
                add_err = 1
                for ignore in ignorelist:
                    if add_err == 1 and re.search(ignore, match):
                        add_err = 0
                if add_err == 1:
                    self.logger.log(prefix + " " + match)
                    errcount = errcount + 1
             else:
               break
         else:
             self.logger.log("Too many errors!")
 
         watch.end()
         return errcount
 
     def is_applicable(self):
         return self.is_applicable_common()
 
     def is_applicable_common(self):
         '''Return True if we are applicable in the current test configuration'''
         #raise ValueError("Abstract Class member (is_applicable)")
 
         if self.is_loop and not self.Env["loop-tests"]:
             return False
         elif self.is_unsafe and not self.Env["unsafe-tests"]:
             return False
         elif self.is_valgrind and not self.Env["valgrind-tests"]:
             return False
         elif self.is_experimental and not self.Env["experimental-tests"]:
             return False
         elif self.is_container and not self.Env["container-tests"]:
             return False
         elif self.Env["benchmark"] and self.benchmark == 0:
             return False
 
         return True
 
     def find_ocfs2_resources(self, node):
         self.r_o2cb = None
         self.r_ocfs2 = []
 
         (_, lines) = self.rsh(node, "crm_resource -c", verbose=1)
         for line in lines:
             if re.search("^Resource", line):
                 r = AuditResource(self.CM, line)
                 if r.rtype == "o2cb" and r.parent != "NA":
                     self.debug("Found o2cb: %s" % self.r_o2cb)
                     self.r_o2cb = r.parent
             if re.search("^Constraint", line):
                 c = AuditConstraint(self.CM, line)
                 if c.type == "rsc_colocation" and c.target == self.r_o2cb:
                     self.r_ocfs2.append(c.rsc)
 
         self.debug("Found ocfs2 filesystems: %s" % repr(self.r_ocfs2))
         return len(self.r_ocfs2)
 
     def canrunnow(self, node):
         '''Return TRUE if we can meaningfully run right now'''
         return 1
 
     def errorstoignore(self):
         '''Return list of errors which are 'normal' and should be ignored'''
         return []
 
 
 class StopTest(CTSTest):
     '''Stop (deactivate) the cluster manager on a node'''
     def __init__(self, cm):
         CTSTest.__init__(self, cm)
         self.name = "Stop"
 
     def __call__(self, node):
         '''Perform the 'stop' test. '''
         self.incr("calls")
         if self.CM.ShouldBeStatus[node] != "up":
             return self.skipped()
 
         patterns = []
         # Technically we should always be able to notice ourselves stopping
         patterns.append(self.templates["Pat:We_stopped"] % node)
 
         # Any active node needs to notice this one left
         # (note that this won't work if we have multiple partitions)
         for other in self.Env["nodes"]:
             if self.CM.ShouldBeStatus[other] == "up" and other != node:
                 patterns.append(self.templates["Pat:They_stopped"] %(other, self.CM.key_for_node(node)))
                 #self.debug("Checking %s will notice %s left"%(other, node))
 
         watch = self.create_watch(patterns, self.Env["DeadTime"])
         watch.set_watch()
 
         if node == self.CM.OurNode:
             self.incr("us")
         else:
             if self.CM.upcount() <= 1:
                 self.incr("all")
             else:
                 self.incr("them")
 
         self.CM.StopaCM(node)
         watch_result = watch.look_for_all()
 
         failreason = None
         UnmatchedList = "||"
         if watch.unmatched:
             (_, output) = self.rsh(node, "/bin/ps axf", verbose=1)
             for line in output:
                 self.debug(line)
 
             (_, output) = self.rsh(node, "/usr/sbin/dlm_tool dump 2>/dev/null", verbose=1)
             for line in output:
                 self.debug(line)
 
             for regex in watch.unmatched:
                 self.logger.log ("ERROR: Shutdown pattern not found: %s" % (regex))
                 UnmatchedList +=  regex + "||";
                 failreason = "Missing shutdown pattern"
 
         self.CM.cluster_stable(self.Env["DeadTime"])
 
         if not watch.unmatched or self.CM.upcount() == 0:
             return self.success()
 
         if len(watch.unmatched) >= self.CM.upcount():
             return self.failure("no match against (%s)" % UnmatchedList)
 
         if failreason == None:
             return self.success()
         else:
             return self.failure(failreason)
 #
 # We don't register StopTest because it's better when called by
 # another test...
 #
 
 
 class StartTest(CTSTest):
     '''Start (activate) the cluster manager on a node'''
     def __init__(self, cm, debug=None):
         CTSTest.__init__(self,cm)
         self.name = "start"
         self.debug = debug
 
     def __call__(self, node):
         '''Perform the 'start' test. '''
         self.incr("calls")
 
         if self.CM.upcount() == 0:
             self.incr("us")
         else:
             self.incr("them")
 
         if self.CM.ShouldBeStatus[node] != "down":
             return self.skipped()
         elif self.CM.StartaCM(node):
             return self.success()
         else:
             return self.failure("Startup %s on node %s failed"
                                 % (self.Env["Name"], node))
 
 #
 # We don't register StartTest because it's better when called by
 # another test...
 #
 
 
 class FlipTest(CTSTest):
     '''If it's running, stop it.  If it's stopped start it.
        Overthrow the status quo...
     '''
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "Flip"
         self.start = StartTest(cm)
         self.stop = StopTest(cm)
 
     def __call__(self, node):
         '''Perform the 'Flip' test. '''
         self.incr("calls")
         if self.CM.ShouldBeStatus[node] == "up":
             self.incr("stopped")
             ret = self.stop(node)
             type = "up->down"
             # Give the cluster time to recognize it's gone...
             time.sleep(self.Env["StableTime"])
         elif self.CM.ShouldBeStatus[node] == "down":
             self.incr("started")
             ret = self.start(node)
             type = "down->up"
         else:
             return self.skipped()
 
         self.incr(type)
         if ret:
             return self.success()
         else:
             return self.failure("%s failure" % type)
 
 #        Register FlipTest as a good test to run
 AllTestClasses.append(FlipTest)
 
 
 class RestartTest(CTSTest):
     '''Stop and restart a node'''
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "Restart"
         self.start = StartTest(cm)
         self.stop = StopTest(cm)
         self.benchmark = 1
 
     def __call__(self, node):
         '''Perform the 'restart' test. '''
         self.incr("calls")
 
         self.incr("node:" + node)
 
         ret1 = 1
         if self.CM.StataCM(node):
             self.incr("WasStopped")
             if not self.start(node):
                 return self.failure("start (setup) failure: "+node)
 
         self.set_timer()
         if not self.stop(node):
             return self.failure("stop failure: "+node)
         if not self.start(node):
             return self.failure("start failure: "+node)
         return self.success()
 
 #        Register RestartTest as a good test to run
 AllTestClasses.append(RestartTest)
 
 
 class StonithdTest(CTSTest):
     def __init__(self, cm):
         CTSTest.__init__(self, cm)
         self.name = "Stonithd"
         self.startall = SimulStartLite(cm)
         self.benchmark = 1
 
     def __call__(self, node):
         self.incr("calls")
         if len(self.Env["nodes"]) < 2:
             return self.skipped()
 
         ret = self.startall(None)
         if not ret:
             return self.failure("Setup failed")
 
         is_dc = self.CM.is_node_dc(node)
 
         watchpats = []
         watchpats.append(self.templates["Pat:Fencing_ok"] % node)
         watchpats.append(self.templates["Pat:NodeFenced"] % node)
 
         if not self.Env["at-boot"]:
             self.debug("Expecting %s to stay down" % node)
             self.CM.ShouldBeStatus[node] = "down"
         else:
             self.debug("Expecting %s to come up again %d" % (node, self.Env["at-boot"]))
             watchpats.append("%s.* S_STARTING -> S_PENDING" % node)
             watchpats.append("%s.* S_PENDING -> S_NOT_DC" % node)
 
         watch = self.create_watch(watchpats, 30 + self.Env["DeadTime"] + self.Env["StableTime"] + self.Env["StartTime"])
         watch.set_watch()
 
         origin = self.Env.random_gen.choice(self.Env["nodes"])
 
         (rc, _) = self.rsh(origin, "stonith_admin --reboot %s -VVVVVV" % node)
 
         if rc == 124: # CRM_EX_TIMEOUT
             # Look for the patterns, usually this means the required
             # device was running on the node to be fenced - or that
             # the required devices were in the process of being loaded
             # and/or moved
             #
             # Effectively the node committed suicide so there will be
             # no confirmation, but pacemaker should be watching and
             # fence the node again
 
             self.logger.log("Fencing command on %s to fence %s timed out" % (origin, node))
 
         elif origin != node and rc != 0:
             self.debug("Waiting for the cluster to recover")
             self.CM.cluster_stable()
 
             self.debug("Waiting for fenced node to come back up")
             self.CM.ns.wait_for_all_nodes(self.Env["nodes"], 600)
 
             self.logger.log("Fencing command on %s failed to fence %s (rc=%d)" % (origin, node, rc))
 
         elif origin == node and rc != 255:
             # 255 == broken pipe, ie. the node was fenced as expected
             self.logger.log("Locally originated fencing returned %d" % rc)
 
         self.set_timer("fence")
         matched = watch.look_for_all()
         self.log_timer("fence")
         self.set_timer("reform")
         if watch.unmatched:
             self.logger.log("Patterns not found: " + repr(watch.unmatched))
 
         self.debug("Waiting for the cluster to recover")
         self.CM.cluster_stable()
 
         self.debug("Waiting for fenced node to come back up")
         self.CM.ns.wait_for_all_nodes(self.Env["nodes"], 600)
 
         self.debug("Waiting for the cluster to re-stabilize with all nodes")
         is_stable = self.CM.cluster_stable(self.Env["StartTime"])
 
         if not matched:
             return self.failure("Didn't find all expected patterns")
         elif not is_stable:
             return self.failure("Cluster did not become stable")
 
         self.log_timer("reform")
         return self.success()
 
     def errorstoignore(self):
         return [
             self.templates["Pat:Fencing_start"] % ".*",
             self.templates["Pat:Fencing_ok"] % ".*",
             self.templates["Pat:Fencing_active"],
             r"error.*: Operation 'reboot' targeting .* by .* for stonith_admin.*: Timer expired",
         ]
 
     def is_applicable(self):
         if not self.is_applicable_common():
             return False
 
         if "DoFencing" in list(self.Env.keys()):
             return self.Env["DoFencing"]
 
         return True
 
 AllTestClasses.append(StonithdTest)
 
 
 class StartOnebyOne(CTSTest):
     '''Start all the nodes ~ one by one'''
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "StartOnebyOne"
         self.stopall = SimulStopLite(cm)
         self.start = StartTest(cm)
         self.ns = NodeStatus(cm.Env)
 
     def __call__(self, dummy):
         '''Perform the 'StartOnebyOne' test. '''
         self.incr("calls")
 
         #        We ignore the "node" parameter...
 
         #        Shut down all the nodes...
         ret = self.stopall(None)
         if not ret:
             return self.failure("Test setup failed")
 
         failed = []
         self.set_timer()
         for node in self.Env["nodes"]:
             if not self.start(node):
                 failed.append(node)
 
         if len(failed) > 0:
             return self.failure("Some node failed to start: " + repr(failed))
 
         return self.success()
 
 #        Register StartOnebyOne as a good test to run
 AllTestClasses.append(StartOnebyOne)
 
 
 class SimulStart(CTSTest):
     '''Start all the nodes ~ simultaneously'''
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "SimulStart"
         self.stopall = SimulStopLite(cm)
         self.startall = SimulStartLite(cm)
 
     def __call__(self, dummy):
         '''Perform the 'SimulStart' test. '''
         self.incr("calls")
 
         #        We ignore the "node" parameter...
 
         #        Shut down all the nodes...
         ret = self.stopall(None)
         if not ret:
             return self.failure("Setup failed")
 
         if not self.startall(None):
             return self.failure("Startall failed")
 
         return self.success()
 
 #        Register SimulStart as a good test to run
 AllTestClasses.append(SimulStart)
 
 
 class SimulStop(CTSTest):
     '''Stop all the nodes ~ simultaneously'''
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "SimulStop"
         self.startall = SimulStartLite(cm)
         self.stopall = SimulStopLite(cm)
 
     def __call__(self, dummy):
         '''Perform the 'SimulStop' test. '''
         self.incr("calls")
 
         #     We ignore the "node" parameter...
 
         #     Start up all the nodes...
         ret = self.startall(None)
         if not ret:
             return self.failure("Setup failed")
 
         if not self.stopall(None):
             return self.failure("Stopall failed")
 
         return self.success()
 
 #     Register SimulStop as a good test to run
 AllTestClasses.append(SimulStop)
 
 
 class StopOnebyOne(CTSTest):
     '''Stop all the nodes in order'''
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "StopOnebyOne"
         self.startall = SimulStartLite(cm)
         self.stop = StopTest(cm)
 
     def __call__(self, dummy):
         '''Perform the 'StopOnebyOne' test. '''
         self.incr("calls")
 
         #     We ignore the "node" parameter...
 
         #     Start up all the nodes...
         ret = self.startall(None)
         if not ret:
             return self.failure("Setup failed")
 
         failed = []
         self.set_timer()
         for node in self.Env["nodes"]:
             if not self.stop(node):
                 failed.append(node)
 
         if len(failed) > 0:
             return self.failure("Some node failed to stop: " + repr(failed))
 
         return self.success()
 
 #     Register StopOnebyOne as a good test to run
 AllTestClasses.append(StopOnebyOne)
 
 
 class RestartOnebyOne(CTSTest):
     '''Restart all the nodes in order'''
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "RestartOnebyOne"
         self.startall = SimulStartLite(cm)
 
     def __call__(self, dummy):
         '''Perform the 'RestartOnebyOne' test. '''
         self.incr("calls")
 
         #     We ignore the "node" parameter...
 
         #     Start up all the nodes...
         ret = self.startall(None)
         if not ret:
             return self.failure("Setup failed")
 
         did_fail = []
         self.set_timer()
         self.restart = RestartTest(self.CM)
         for node in self.Env["nodes"]:
             if not self.restart(node):
                 did_fail.append(node)
 
         if did_fail:
             return self.failure("Could not restart %d nodes: %s"
                                 % (len(did_fail), repr(did_fail)))
         return self.success()
 
 #     Register StopOnebyOne as a good test to run
 AllTestClasses.append(RestartOnebyOne)
 
 
 class PartialStart(CTSTest):
     '''Start a node - but tell it to stop before it finishes starting up'''
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "PartialStart"
         self.startall = SimulStartLite(cm)
         self.stopall = SimulStopLite(cm)
         self.stop = StopTest(cm)
         #self.is_unsafe = 1
 
     def __call__(self, node):
         '''Perform the 'PartialStart' test. '''
         self.incr("calls")
 
         ret = self.stopall(None)
         if not ret:
             return self.failure("Setup failed")
 
         watchpats = []
         watchpats.append("pacemaker-controld.*Connecting to .* cluster infrastructure")
         watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
         watch.set_watch()
 
         self.CM.StartaCMnoBlock(node)
         ret = watch.look_for_all()
         if not ret:
             self.logger.log("Patterns not found: " + repr(watch.unmatched))
             return self.failure("Setup of %s failed" % node)
 
         ret = self.stop(node)
         if not ret:
             return self.failure("%s did not stop in time" % node)
 
         return self.success()
 
     def errorstoignore(self):
         '''Return list of errors which should be ignored'''
 
         # We might do some fencing in the 2-node case if we make it up far enough
         return [
             r"Executing reboot fencing operation",
-            r"Requesting fencing \([^)]+\) of node ",
+            r"Requesting fencing \([^)]+\) targeting node ",
         ]
 
 #     Register StopOnebyOne as a good test to run
 AllTestClasses.append(PartialStart)
 
 
 class StandbyTest(CTSTest):
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "Standby"
         self.benchmark = 1
 
         self.start = StartTest(cm)
         self.startall = SimulStartLite(cm)
 
     # make sure the node is active
     # set the node to standby mode
     # check resources, none resource should be running on the node
     # set the node to active mode
     # check resouces, resources should have been migrated back (SHOULD THEY?)
 
     def __call__(self, node):
 
         self.incr("calls")
         ret = self.startall(None)
         if not ret:
             return self.failure("Start all nodes failed")
 
         self.debug("Make sure node %s is active" % node)
         if self.CM.StandbyStatus(node) != "off":
             if not self.CM.SetStandbyMode(node, "off"):
                 return self.failure("can't set node %s to active mode" % node)
 
         self.CM.cluster_stable()
 
         status = self.CM.StandbyStatus(node)
         if status != "off":
             return self.failure("standby status of %s is [%s] but we expect [off]" % (node, status))
 
         self.debug("Getting resources running on node %s" % node)
         rsc_on_node = self.CM.active_resources(node)
 
         watchpats = []
         watchpats.append(r"State transition .* -> S_POLICY_ENGINE")
         watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
         watch.set_watch()
 
         self.debug("Setting node %s to standby mode" % node)
         if not self.CM.SetStandbyMode(node, "on"):
             return self.failure("can't set node %s to standby mode" % node)
 
         self.set_timer("on")
 
         ret = watch.look_for_all()
         if not ret:
             self.logger.log("Patterns not found: " + repr(watch.unmatched))
             self.CM.SetStandbyMode(node, "off")
             return self.failure("cluster didn't react to standby change on %s" % node)
 
         self.CM.cluster_stable()
 
         status = self.CM.StandbyStatus(node)
         if status != "on":
             return self.failure("standby status of %s is [%s] but we expect [on]" % (node, status))
         self.log_timer("on")
 
         self.debug("Checking resources")
         bad_run = self.CM.active_resources(node)
         if len(bad_run) > 0:
             rc = self.failure("%s set to standby, %s is still running on it" % (node, repr(bad_run)))
             self.debug("Setting node %s to active mode" % node)
             self.CM.SetStandbyMode(node, "off")
             return rc
 
         self.debug("Setting node %s to active mode" % node)
         if not self.CM.SetStandbyMode(node, "off"):
             return self.failure("can't set node %s to active mode" % node)
 
         self.set_timer("off")
         self.CM.cluster_stable()
 
         status = self.CM.StandbyStatus(node)
         if status != "off":
             return self.failure("standby status of %s is [%s] but we expect [off]" % (node, status))
         self.log_timer("off")
 
         return self.success()
 
 AllTestClasses.append(StandbyTest)
 
 
 class ValgrindTest(CTSTest):
     '''Check for memory leaks'''
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "Valgrind"
         self.stopall = SimulStopLite(cm)
         self.startall = SimulStartLite(cm)
         self.is_valgrind = 1
         self.is_loop = 1
 
     def setup(self, node):
         self.incr("calls")
 
         ret = self.stopall(None)
         if not ret:
             return self.failure("Stop all nodes failed")
 
         # @TODO Edit /etc/sysconfig/pacemaker on all nodes to enable valgrind,
         # and clear any valgrind logs from previous runs. For now, we rely on
         # the user to do this manually.
 
         ret = self.startall(None)
         if not ret:
             return self.failure("Start all nodes failed")
 
         return self.success()
 
     def teardown(self, node):
         # Return all nodes to normal
         # @TODO Edit /etc/sysconfig/pacemaker on all nodes to disable valgrind
         ret = self.stopall(None)
         if not ret:
             return self.failure("Stop all nodes failed")
 
         return self.success()
 
     def find_leaks(self):
         # Check for leaks
         # (no longer used but kept in case feature is restored)
         leaked = []
         self.stop = StopTest(self.CM)
 
         for node in self.Env["nodes"]:
             rc = self.stop(node)
             if not rc:
                 self.failure("Couldn't shut down %s" % node)
 
             (rc, _) = self.rsh(node, "grep -e indirectly.*lost:.*[1-9] -e definitely.*lost:.*[1-9] -e (ERROR|error).*SUMMARY:.*[1-9].*errors %s" % self.logger.logPat)
             if rc != 1:
                 leaked.append(node)
                 self.failure("Valgrind errors detected on %s" % node)
                 (_, output) = self.rsh(node, "grep -e lost: -e SUMMARY: %s" % self.logger.logPat, verbose=1)
                 for line in output:
                     self.logger.log(line)
                 (_, output) = self.rsh(node, "cat %s" % self.logger.logPat, verbose=1)
                 for line in output:
                     self.debug(line)
 
         self.rsh(node, "rm -f %s" % self.logger.logPat, verbose=1)
         return leaked
 
     def __call__(self, node):
         #leaked = self.find_leaks()
         #if len(leaked) > 0:
         #    return self.failure("Nodes %s leaked" % repr(leaked))
 
         return self.success()
 
     def errorstoignore(self):
         '''Return list of errors which should be ignored'''
         return [
             r"pacemaker-based.*: \*\*\*\*\*\*\*\*\*\*\*\*\*",
             r"pacemaker-based.*: .* avoid confusing Valgrind",
             r"HA_VALGRIND_ENABLED",
         ]
 
 
 class StandbyLoopTest(ValgrindTest):
     '''Check for memory leaks by putting a node in and out of standby for an hour'''
     # @TODO This is not a useful test for memory leaks
     def __init__(self, cm):
         ValgrindTest.__init__(self,cm)
         self.name = "StandbyLoop"
 
     def __call__(self, node):
 
         lpc = 0
         delay = 2
         failed = 0
         done = time.time() + self.Env["loop-minutes"] * 60
         while time.time() <= done and not failed:
             lpc = lpc + 1
 
             time.sleep(delay)
             if not self.CM.SetStandbyMode(node, "on"):
                 self.failure("can't set node %s to standby mode" % node)
                 failed = lpc
 
             time.sleep(delay)
             if not self.CM.SetStandbyMode(node, "off"):
                 self.failure("can't set node %s to active mode" % node)
                 failed = lpc
 
         leaked = self.find_leaks()
         if failed:
             return self.failure("Iteration %d failed" % failed)
         elif len(leaked) > 0:
             return self.failure("Nodes %s leaked" % repr(leaked))
 
         return self.success()
 
 #AllTestClasses.append(StandbyLoopTest)
 
 
 class BandwidthTest(CTSTest):
 #        Tests should not be cluster-manager-specific
 #        If you need to find out cluster manager configuration to do this, then
 #        it should be added to the generic cluster manager API.
     '''Test the bandwidth which the cluster uses'''
     def __init__(self, cm):
         CTSTest.__init__(self, cm)
         self.name = "Bandwidth"
         self.start = StartTest(cm)
         self.__setitem__("min",0)
         self.__setitem__("max",0)
         self.__setitem__("totalbandwidth",0)
         (handle, self.tempfile) = tempfile.mkstemp(".cts")
         os.close(handle)
         self.startall = SimulStartLite(cm)
 
     def __call__(self, node):
         '''Perform the Bandwidth test'''
         self.incr("calls")
 
         if self.CM.upcount() < 1:
             return self.skipped()
 
         Path = self.CM.InternalCommConfig()
         if "ip" not in Path["mediatype"]:
              return self.skipped()
 
         port = Path["port"][0]
         port = int(port)
 
         ret = self.startall(None)
         if not ret:
             return self.failure("Test setup failed")
         time.sleep(5)  # We get extra messages right after startup.
 
         fstmpfile = "/var/run/band_estimate"
         dumpcmd = "tcpdump -p -n -c 102 -i any udp port %d > %s 2>&1" \
         %                (port, fstmpfile)
 
         (rc, _) = self.rsh(node, dumpcmd)
         if rc == 0:
             farfile = "root@%s:%s" % (node, fstmpfile)
             self.rsh.copy(farfile, self.tempfile)
             Bandwidth = self.countbandwidth(self.tempfile)
             if not Bandwidth:
                 self.logger.log("Could not compute bandwidth.")
                 return self.success()
             intband = int(Bandwidth + 0.5)
             self.logger.log("...bandwidth: %d bits/sec" % intband)
             self.Stats["totalbandwidth"] = self.Stats["totalbandwidth"] + Bandwidth
             if self.Stats["min"] == 0:
                 self.Stats["min"] = Bandwidth
             if Bandwidth > self.Stats["max"]:
                 self.Stats["max"] = Bandwidth
             if Bandwidth < self.Stats["min"]:
                 self.Stats["min"] = Bandwidth
             self.rsh(node, "rm -f %s" % fstmpfile)
             os.unlink(self.tempfile)
             return self.success()
         else:
             return self.failure("no response from tcpdump command [%d]!" % rc)
 
     def countbandwidth(self, file):
         fp = open(file, "r")
         fp.seek(0)
         count = 0
         sum = 0
         while 1:
             line = fp.readline()
             if not line:
                 return None
             if re.search("udp",line) or re.search("UDP,", line):
                 count = count + 1
                 linesplit = line.split(" ")
                 for j in range(len(linesplit)-1):
                     if linesplit[j] == "udp": break
                     if linesplit[j] == "length:": break
 
                 try:
                     sum = sum + int(linesplit[j+1])
                 except ValueError:
                     self.logger.log("Invalid tcpdump line: %s" % line)
                     return None
                 T1 = linesplit[0]
                 timesplit = T1.split(":")
                 time2split = timesplit[2].split(".")
                 time1 = (int(timesplit[0])*60+int(timesplit[1]))*60+int(time2split[0])+int(time2split[1])*0.000001
                 break
 
         while count < 100:
             line = fp.readline()
             if not line:
                 return None
             if re.search("udp",line) or re.search("UDP,", line):
                 count = count+1
                 linessplit = line.split(" ")
                 for j in range(len(linessplit)-1):
                     if linessplit[j] == "udp": break
                     if linessplit[j] == "length:": break
                 try:
                     sum = int(linessplit[j+1]) + sum
                 except ValueError:
                     self.logger.log("Invalid tcpdump line: %s" % line)
                     return None
 
         T2 = linessplit[0]
         timesplit = T2.split(":")
         time2split = timesplit[2].split(".")
         time2 = (int(timesplit[0])*60+int(timesplit[1]))*60+int(time2split[0])+int(time2split[1])*0.000001
         time = time2-time1
         if (time <= 0):
             return 0
         return int((sum*8)/time)
 
     def is_applicable(self):
         '''BandwidthTest never applicable'''
         return False
 
 AllTestClasses.append(BandwidthTest)
 
 
 ###################################################################
 class MaintenanceMode(CTSTest):
 ###################################################################
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "MaintenanceMode"
         self.start = StartTest(cm)
         self.startall = SimulStartLite(cm)
         self.max = 30
         #self.is_unsafe = 1
         self.benchmark = 1
         self.action = "asyncmon"
         self.interval = 0
         self.rid = "maintenanceDummy"
 
     def toggleMaintenanceMode(self, node, action):
         pats = []
         pats.append(self.templates["Pat:DC_IDLE"])
 
         # fail the resource right after turning Maintenance mode on
         # verify it is not recovered until maintenance mode is turned off
         if action == "On":
             pats.append(self.templates["Pat:RscOpFail"] % (self.action, self.rid))
         else:
             pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))
             pats.append(self.templates["Pat:RscOpOK"] % ("start", self.rid))
 
         watch = self.create_watch(pats, 60)
         watch.set_watch()
 
         self.debug("Turning maintenance mode %s" % action)
         self.rsh(node, self.templates["MaintenanceMode%s" % (action)])
         if (action == "On"):
             self.rsh(node, "crm_resource -V -F -r %s -H %s &>/dev/null" % (self.rid, node))
 
         self.set_timer("recover%s" % (action))
         watch.look_for_all()
         self.log_timer("recover%s" % (action))
         if watch.unmatched:
             self.debug("Failed to find patterns when turning maintenance mode %s" % action)
             return repr(watch.unmatched)
 
         return ""
 
     def insertMaintenanceDummy(self, node):
         pats = []
         pats.append(("%s.*" % node) + (self.templates["Pat:RscOpOK"] % ("start", self.rid)))
 
         watch = self.create_watch(pats, 60)
         watch.set_watch()
 
         self.CM.AddDummyRsc(node, self.rid)
 
         self.set_timer("addDummy")
         watch.look_for_all()
         self.log_timer("addDummy")
 
         if watch.unmatched:
             self.debug("Failed to find patterns when adding maintenance dummy resource")
             return repr(watch.unmatched)
         return ""
 
     def removeMaintenanceDummy(self, node):
         pats = []
         pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))
 
         watch = self.create_watch(pats, 60)
         watch.set_watch()
         self.CM.RemoveDummyRsc(node, self.rid)
 
         self.set_timer("removeDummy")
         watch.look_for_all()
         self.log_timer("removeDummy")
 
         if watch.unmatched:
             self.debug("Failed to find patterns when removing maintenance dummy resource")
             return repr(watch.unmatched)
         return ""
 
     def managedRscList(self, node):
         rscList = []
         (_, lines) = self.rsh(node, "crm_resource -c", verbose=1)
         for line in lines:
             if re.search("^Resource", line):
                 tmp = AuditResource(self.CM, line)
                 if tmp.managed():
                     rscList.append(tmp.id)
 
         return rscList
 
     def verifyResources(self, node, rscList, managed):
         managedList = list(rscList)
         managed_str = "managed"
         if not managed:
             managed_str = "unmanaged"
 
         (_, lines) = self.rsh(node, "crm_resource -c", verbose=1)
         for line in lines:
             if re.search("^Resource", line):
                 tmp = AuditResource(self.CM, line)
                 if managed and not tmp.managed():
                     continue
                 elif not managed and tmp.managed():
                     continue
                 elif managedList.count(tmp.id):
                     managedList.remove(tmp.id)
 
         if len(managedList) == 0:
             self.debug("Found all %s resources on %s" % (managed_str, node))
             return True
 
         self.logger.log("Could not find all %s resources on %s. %s" % (managed_str, node, managedList))
         return False
 
     def __call__(self, node):
         '''Perform the 'MaintenanceMode' test. '''
         self.incr("calls")
         verify_managed = False
         verify_unmanaged = False
         failPat = ""
 
         ret = self.startall(None)
         if not ret:
             return self.failure("Setup failed")
 
         # get a list of all the managed resources. We use this list
         # after enabling maintenance mode to verify all managed resources
         # become un-managed.  After maintenance mode is turned off, we use
         # this list to verify all the resources become managed again.
         managedResources = self.managedRscList(node)
         if len(managedResources) == 0:
             self.logger.log("No managed resources on %s" % node)
             return self.skipped()
 
         # insert a fake resource we can fail during maintenance mode
         # so we can verify recovery does not take place until after maintenance
         # mode is disabled.
         failPat = failPat + self.insertMaintenanceDummy(node)
 
         # toggle maintenance mode ON, then fail dummy resource.
         failPat = failPat + self.toggleMaintenanceMode(node, "On")
 
         # verify all the resources are now unmanaged
         if self.verifyResources(node, managedResources, False):
             verify_unmanaged = True
 
         # Toggle maintenance mode  OFF, verify dummy is recovered.
         failPat = failPat + self.toggleMaintenanceMode(node, "Off")
 
         # verify all the resources are now managed again
         if self.verifyResources(node, managedResources, True):
             verify_managed = True
 
         # Remove our maintenance dummy resource.
         failPat = failPat + self.removeMaintenanceDummy(node)
 
         self.CM.cluster_stable()
 
         if failPat != "":
             return self.failure("Unmatched patterns: %s" % (failPat))
         elif verify_unmanaged is False:
             return self.failure("Failed to verify resources became unmanaged during maintenance mode")
         elif verify_managed is False:
             return self.failure("Failed to verify resources switched back to managed after disabling maintenance mode")
 
         return self.success()
 
     def errorstoignore(self):
         '''Return list of errors which should be ignored'''
         return [
             r"Updating failcount for %s" % self.rid,
             r"schedulerd.*: Recover\s+%s\s+\(.*\)" % self.rid,
             r"Unknown operation: fail",
             self.templates["Pat:RscOpOK"] % (self.action, self.rid),
             r"(ERROR|error).*: Action %s_%s_%d .* initiated outside of a transition" % (self.rid, self.action, self.interval),
         ]
 
 AllTestClasses.append(MaintenanceMode)
 
 
 class ResourceRecover(CTSTest):
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "ResourceRecover"
         self.start = StartTest(cm)
         self.startall = SimulStartLite(cm)
         self.max = 30
         self.rid = None
         self.rid_alt = None
         #self.is_unsafe = 1
         self.benchmark = 1
 
         # these are the values used for the new LRM API call
         self.action = "asyncmon"
         self.interval = 0
 
     def __call__(self, node):
         '''Perform the 'ResourceRecover' test. '''
         self.incr("calls")
 
         ret = self.startall(None)
         if not ret:
             return self.failure("Setup failed")
 
         # List all resources active on the node (skip test if none)
         resourcelist = self.CM.active_resources(node)
         if len(resourcelist) == 0:
             self.logger.log("No active resources on %s" % node)
             return self.skipped()
 
         # Choose one resource at random
         rsc = self.choose_resource(node, resourcelist)
         if rsc is None:
             return self.failure("Could not get details of resource '%s'" % self.rid)
         if rsc.id == rsc.clone_id:
             self.debug("Failing " + rsc.id)
         else:
             self.debug("Failing " + rsc.id + " (also known as " + rsc.clone_id + ")")
 
         # Log patterns to watch for (failure, plus restart if managed)
         pats = []
         pats.append(self.templates["Pat:CloneOpFail"] % (self.action, rsc.id, rsc.clone_id))
         if rsc.managed():
             pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))
             if rsc.unique():
                 pats.append(self.templates["Pat:RscOpOK"] % ("start", self.rid))
             else:
                 # Anonymous clones may get restarted with a different clone number
                 pats.append(self.templates["Pat:RscOpOK"] % ("start", ".*"))
 
         # Fail resource. (Ideally, we'd fail it twice, to ensure the fail count
         # is incrementing properly, but it might restart on a different node.
         # We'd have to temporarily ban it from all other nodes and ensure the
         # migration-threshold hasn't been reached.)
         if self.fail_resource(rsc, node, pats) is None:
             return None # self.failure() already called
 
         return self.success()
 
     def choose_resource(self, node, resourcelist):
         """ Choose a random resource to target """
 
         self.rid = self.Env.random_gen.choice(resourcelist)
         self.rid_alt = self.rid
         (_, lines) = self.rsh(node, "crm_resource -c", verbose=1)
         for line in lines:
             if line.startswith("Resource: "):
                 rsc = AuditResource(self.CM, line)
                 if rsc.id == self.rid:
                     # Handle anonymous clones that get renamed
                     self.rid = rsc.clone_id
                     return rsc
         return None
 
     def get_failcount(self, node):
         """ Check the fail count of targeted resource on given node """
 
         (rc, lines) = self.rsh(node,
                                "crm_failcount --quiet --query --resource %s "
                                "--operation %s --interval %d "
                                "--node %s" % (self.rid, self.action,
                                self.interval, node), verbose=1)
         if rc != 0 or len(lines) != 1:
             self.logger.log("crm_failcount on %s failed (%d): %s" % (node, rc,
                             " // ".join(map(str.strip, lines))))
             return -1
         try:
             failcount = int(lines[0])
         except (IndexError, ValueError):
             self.logger.log("crm_failcount output on %s unparseable: %s" % (node,
                             ' '.join(lines)))
             return -1
         return failcount
 
     def fail_resource(self, rsc, node, pats):
         """ Fail the targeted resource, and verify as expected """
 
         orig_failcount = self.get_failcount(node)
 
         watch = self.create_watch(pats, 60)
         watch.set_watch()
 
         self.rsh(node, "crm_resource -V -F -r %s -H %s &>/dev/null" % (self.rid, node))
 
         self.set_timer("recover")
         watch.look_for_all()
         self.log_timer("recover")
 
         self.CM.cluster_stable()
         recovered = self.CM.ResourceLocation(self.rid)
 
         if watch.unmatched:
             return self.failure("Patterns not found: %s" % repr(watch.unmatched))
 
         elif rsc.unique() and len(recovered) > 1:
             return self.failure("%s is now active on more than one node: %s"%(self.rid, repr(recovered)))
 
         elif len(recovered) > 0:
             self.debug("%s is running on: %s" % (self.rid, repr(recovered)))
 
         elif rsc.managed():
             return self.failure("%s was not recovered and is inactive" % self.rid)
 
         new_failcount = self.get_failcount(node)
         if new_failcount != (orig_failcount + 1):
             return self.failure("%s fail count is %d not %d" % (self.rid,
                                 new_failcount, orig_failcount + 1))
 
         return 0 # Anything but None is success
 
     def errorstoignore(self):
         '''Return list of errors which should be ignored'''
         return [
             r"Updating failcount for %s" % self.rid,
             r"schedulerd.*: Recover\s+(%s|%s)\s+\(.*\)" % (self.rid, self.rid_alt),
             r"Unknown operation: fail",
             self.templates["Pat:RscOpOK"] % (self.action, self.rid),
             r"(ERROR|error).*: Action %s_%s_%d .* initiated outside of a transition" % (self.rid, self.action, self.interval),
         ]
 
 AllTestClasses.append(ResourceRecover)
 
 
 class ComponentFail(CTSTest):
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "ComponentFail"
         self.startall = SimulStartLite(cm)
         self.complist = cm.Components()
         self.patterns = []
         self.okerrpatterns = []
         self.is_unsafe = 1
 
     def __call__(self, node):
         '''Perform the 'ComponentFail' test. '''
         self.incr("calls")
         self.patterns = []
         self.okerrpatterns = []
 
         # start all nodes
         ret = self.startall(None)
         if not ret:
             return self.failure("Setup failed")
 
         if not self.CM.cluster_stable(self.Env["StableTime"]):
             return self.failure("Setup failed - unstable")
 
         node_is_dc = self.CM.is_node_dc(node, None)
 
         # select a component to kill
         chosen = self.Env.random_gen.choice(self.complist)
         while chosen.dc_only and node_is_dc == 0:
             chosen = self.Env.random_gen.choice(self.complist)
 
         self.debug("...component %s (dc=%d)" % (chosen.name, node_is_dc))
         self.incr(chosen.name)
 
         if chosen.name != "corosync":
             self.patterns.append(self.templates["Pat:ChildKilled"] %(node, chosen.name))
             self.patterns.append(self.templates["Pat:ChildRespawn"] %(node, chosen.name))
 
         self.patterns.extend(chosen.pats)
         if node_is_dc:
           self.patterns.extend(chosen.dc_pats)
 
         # @TODO this should be a flag in the Component
         if chosen.name in [ "corosync", "pacemaker-based", "pacemaker-fenced" ]:
             # Ignore actions for fence devices if fencer will respawn
             # (their registration will be lost, and probes will fail)
             self.okerrpatterns = [ self.templates["Pat:Fencing_active"] ]
             (_, lines) = self.rsh(node, "crm_resource -c", verbose=1)
             for line in lines:
                 if re.search("^Resource", line):
                     r = AuditResource(self.CM, line)
                     if r.rclass == "stonith":
                         self.okerrpatterns.append(self.templates["Pat:Fencing_recover"] % r.id)
                         self.okerrpatterns.append(self.templates["Pat:Fencing_probe"] % r.id)
 
         # supply a copy so self.patterns doesn't end up empty
         tmpPats = []
         tmpPats.extend(self.patterns)
         self.patterns.extend(chosen.badnews_ignore)
 
         # Look for STONITH ops, depending on Env["at-boot"] we might need to change the nodes status
         stonithPats = []
         stonithPats.append(self.templates["Pat:Fencing_ok"] % node)
         stonith = self.create_watch(stonithPats, 0)
         stonith.set_watch()
 
         # set the watch for stable
         watch = self.create_watch(
             tmpPats, self.Env["DeadTime"] + self.Env["StableTime"] + self.Env["StartTime"])
         watch.set_watch()
 
         # kill the component
         chosen.kill(node)
 
         self.debug("Waiting for the cluster to recover")
         self.CM.cluster_stable()
 
         self.debug("Waiting for any fenced node to come back up")
         self.CM.ns.wait_for_all_nodes(self.Env["nodes"], 600)
 
         self.debug("Waiting for the cluster to re-stabilize with all nodes")
         self.CM.cluster_stable(self.Env["StartTime"])
 
         self.debug("Checking if %s was shot" % node)
         shot = stonith.look(60)
         if shot:
             self.debug("Found: " + repr(shot))
             self.okerrpatterns.append(self.templates["Pat:Fencing_start"] % node)
 
             if not self.Env["at-boot"]:
                 self.CM.ShouldBeStatus[node] = "down"
 
             # If fencing occurred, chances are many (if not all) the expected logs
             # will not be sent - or will be lost when the node reboots
             return self.success()
 
         # check for logs indicating a graceful recovery
         matched = watch.look_for_all(allow_multiple_matches=True)
         if watch.unmatched:
             self.logger.log("Patterns not found: " + repr(watch.unmatched))
 
         self.debug("Waiting for the cluster to re-stabilize with all nodes")
         is_stable = self.CM.cluster_stable(self.Env["StartTime"])
 
         if not matched:
             return self.failure("Didn't find all expected %s patterns" % chosen.name)
         elif not is_stable:
             return self.failure("Cluster did not become stable after killing %s" % chosen.name)
 
         return self.success()
 
     def errorstoignore(self):
         '''Return list of errors which should be ignored'''
     # Note that okerrpatterns refers to the last time we ran this test
     # The good news is that this works fine for us...
         self.okerrpatterns.extend(self.patterns)
         return self.okerrpatterns
 
 AllTestClasses.append(ComponentFail)
 
 
 class SplitBrainTest(CTSTest):
     '''It is used to test split-brain. when the path between the two nodes break
        check the two nodes both take over the resource'''
     def __init__(self,cm):
         CTSTest.__init__(self,cm)
         self.name = "SplitBrain"
         self.start = StartTest(cm)
         self.startall = SimulStartLite(cm)
         self.is_experimental = 1
 
     def isolate_partition(self, partition):
         other_nodes = []
         other_nodes.extend(self.Env["nodes"])
 
         for node in partition:
             try:
                 other_nodes.remove(node)
             except ValueError:
                 self.logger.log("Node "+node+" not in " + repr(self.Env["nodes"]) + " from " +repr(partition))
 
         if len(other_nodes) == 0:
             return 1
 
         self.debug("Creating partition: " + repr(partition))
         self.debug("Everyone else: " + repr(other_nodes))
 
         for node in partition:
             if not self.CM.isolate_node(node, other_nodes):
                 self.logger.log("Could not isolate %s" % node)
                 return 0
 
         return 1
 
     def heal_partition(self, partition):
         other_nodes = []
         other_nodes.extend(self.Env["nodes"])
 
         for node in partition:
             try:
                 other_nodes.remove(node)
             except ValueError:
                 self.logger.log("Node "+node+" not in " + repr(self.Env["nodes"]))
 
         if len(other_nodes) == 0:
             return 1
 
         self.debug("Healing partition: " + repr(partition))
         self.debug("Everyone else: " + repr(other_nodes))
 
         for node in partition:
             self.CM.unisolate_node(node, other_nodes)
 
     def __call__(self, node):
         '''Perform split-brain test'''
         self.incr("calls")
         self.passed = 1
         partitions = {}
 
         ret = self.startall(None)
         if not ret:
             return self.failure("Setup failed")
 
         while 1:
             # Retry until we get multiple partitions
             partitions = {}
             p_max = len(self.Env["nodes"])
             for node in self.Env["nodes"]:
                 p = self.Env.random_gen.randint(1, p_max)
                 if not p in partitions:
                     partitions[p] = []
                 partitions[p].append(node)
             p_max = len(list(partitions.keys()))
             if p_max > 1:
                 break
             # else, try again
 
         self.debug("Created %d partitions" % p_max)
         for key in list(partitions.keys()):
             self.debug("Partition["+str(key)+"]:\t"+repr(partitions[key]))
 
         # Disabling STONITH to reduce test complexity for now
         self.rsh(node, "crm_attribute -V -n stonith-enabled -v false")
 
         for key in list(partitions.keys()):
             self.isolate_partition(partitions[key])
 
         count = 30
         while count > 0:
             if len(self.CM.find_partitions()) != p_max:
                 time.sleep(10)
             else:
                 break
         else:
             self.failure("Expected partitions were not created")
 
         # Target number of partitions formed - wait for stability
         if not self.CM.cluster_stable():
             self.failure("Partitioned cluster not stable")
 
         # Now audit the cluster state
         self.CM.partitions_expected = p_max
         if not self.audit():
             self.failure("Audits failed")
         self.CM.partitions_expected = 1
 
         # And heal them again
         for key in list(partitions.keys()):
             self.heal_partition(partitions[key])
 
         # Wait for a single partition to form
         count = 30
         while count > 0:
             if len(self.CM.find_partitions()) != 1:
                 time.sleep(10)
                 count -= 1
             else:
                 break
         else:
             self.failure("Cluster did not reform")
 
         # Wait for it to have the right number of members
         count = 30
         while count > 0:
             members = []
 
             partitions = self.CM.find_partitions()
             if len(partitions) > 0:
                 members = partitions[0].split()
 
             if len(members) != len(self.Env["nodes"]):
                 time.sleep(10)
                 count -= 1
             else:
                 break
         else:
             self.failure("Cluster did not completely reform")
 
         # Wait up to 20 minutes - the delay is more preferable than
         # trying to continue with in a messed up state
         if not self.CM.cluster_stable(1200):
             self.failure("Reformed cluster not stable")
             if self.Env["continue"]:
                 answer = "Y"
             else:
                 try:
                     answer = input('Continue? [nY]')
                 except EOFError as e:
                     answer = "n" 
             if answer and answer == "n":
                 raise ValueError("Reformed cluster not stable")
 
         # Turn fencing back on
         if self.Env["DoFencing"]:
             self.rsh(node, "crm_attribute -V -D -n stonith-enabled")
 
         self.CM.cluster_stable()
 
         if self.passed:
             return self.success()
         return self.failure("See previous errors")
 
     def errorstoignore(self):
         '''Return list of errors which are 'normal' and should be ignored'''
         return [
             r"Another DC detected:",
             r"(ERROR|error).*: .*Application of an update diff failed",
             r"pacemaker-controld.*:.*not in our membership list",
             r"CRIT:.*node.*returning after partition",
         ]
 
     def is_applicable(self):
         if not self.is_applicable_common():
             return False
         return len(self.Env["nodes"]) > 2
 
 AllTestClasses.append(SplitBrainTest)
 
 
 class Reattach(CTSTest):
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "Reattach"
         self.startall = SimulStartLite(cm)
         self.restart1 = RestartTest(cm)
         self.stopall = SimulStopLite(cm)
         self.is_unsafe = 0 # Handled by canrunnow()
 
     def _is_managed(self, node):
         (_, is_managed) = self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -q -G -d true", verbose=1)
         is_managed = is_managed[0].strip()
         return is_managed == "true"
 
     def _set_unmanaged(self, node):
         self.debug("Disable resource management")
         self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -v false")
 
     def _set_managed(self, node):
         self.debug("Re-enable resource management")
         self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -D")
 
     def setup(self, node):
         attempt = 0
         if not self.startall(None):
             return None
 
         # Make sure we are really _really_ stable and that all
         # resources, including those that depend on transient node
         # attributes, are started
         while not self.CM.cluster_stable(double_check=True):
             if attempt < 5:
                 attempt += 1
                 self.debug("Not stable yet, re-testing")
             else:
                 self.logger.log("Cluster is not stable")
                 return None
 
         return 1
 
     def teardown(self, node):
 
         # Make sure 'node' is up
         start = StartTest(self.CM)
         start(node)
 
         if not self._is_managed(node):
             self.logger.log("Attempting to re-enable resource management on %s" % node)
             self._set_managed(node)
             self.CM.cluster_stable()
             if not self._is_managed(node):
                 self.logger.log("Could not re-enable resource management")
                 return 0
 
         return 1
 
     def canrunnow(self, node):
         '''Return TRUE if we can meaningfully run right now'''
         if self.find_ocfs2_resources(node):
             self.logger.log("Detach/Reattach scenarios are not possible with OCFS2 services present")
             return 0
         return 1
 
     def __call__(self, node):
         self.incr("calls")
 
         pats = []
         # Conveniently, the scheduler will display this message when disabling
         # management, even if fencing is not enabled, so we can rely on it.
         managed = self.create_watch(["No fencing will be done"], 60)
         managed.set_watch()
 
         self._set_unmanaged(node)
 
         if not managed.look_for_all():
             self.logger.log("Patterns not found: " + repr(managed.unmatched))
             return self.failure("Resource management not disabled")
 
         pats = []
         pats.append(self.templates["Pat:RscOpOK"] % ("start", ".*"))
         pats.append(self.templates["Pat:RscOpOK"] % ("stop", ".*"))
         pats.append(self.templates["Pat:RscOpOK"] % ("promote", ".*"))
         pats.append(self.templates["Pat:RscOpOK"] % ("demote", ".*"))
         pats.append(self.templates["Pat:RscOpOK"] % ("migrate", ".*"))
 
         watch = self.create_watch(pats, 60, "ShutdownActivity")
         watch.set_watch()
 
         self.debug("Shutting down the cluster")
         ret = self.stopall(None)
         if not ret:
             self._set_managed(node)
             return self.failure("Couldn't shut down the cluster")
 
         self.debug("Bringing the cluster back up")
         ret = self.startall(None)
         time.sleep(5) # allow ping to update the CIB
         if not ret:
             self._set_managed(node)
             return self.failure("Couldn't restart the cluster")
 
         if self.local_badnews("ResourceActivity:", watch):
             self._set_managed(node)
             return self.failure("Resources stopped or started during cluster restart")
 
         watch = self.create_watch(pats, 60, "StartupActivity")
         watch.set_watch()
 
         # Re-enable resource management (and verify it happened).
         self._set_managed(node)
         self.CM.cluster_stable()
         if not self._is_managed(node):
             return self.failure("Could not re-enable resource management")
 
         # Ignore actions for STONITH resources
         ignore = []
         (_, lines) = self.rsh(node, "crm_resource -c", verbose=1)
         for line in lines:
             if re.search("^Resource", line):
                 r = AuditResource(self.CM, line)
                 if r.rclass == "stonith":
 
                     self.debug("Ignoring start actions for %s" % r.id)
                     ignore.append(self.templates["Pat:RscOpOK"] % ("start", r.id))
 
         if self.local_badnews("ResourceActivity:", watch, ignore):
             return self.failure("Resources stopped or started after resource management was re-enabled")
 
         return ret
 
     def errorstoignore(self):
         '''Return list of errors which should be ignored'''
         return [
             r"resource( was|s were) active at shutdown",
         ]
 
     def is_applicable(self):
         return True
 
 AllTestClasses.append(Reattach)
 
 
 class SpecialTest1(CTSTest):
     '''Set up a custom test to cause quorum failure issues for Andrew'''
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "SpecialTest1"
         self.startall = SimulStartLite(cm)
         self.restart1 = RestartTest(cm)
         self.stopall = SimulStopLite(cm)
 
     def __call__(self, node):
         '''Perform the 'SpecialTest1' test for Andrew. '''
         self.incr("calls")
 
         #        Shut down all the nodes...
         ret = self.stopall(None)
         if not ret:
             return self.failure("Could not stop all nodes")
 
         # Test config recovery when the other nodes come up
         self.rsh(node, "rm -f " + BuildOptions.CIB_DIR + "/cib*")
 
         #        Start the selected node
         ret = self.restart1(node)
         if not ret:
             return self.failure("Could not start "+node)
 
         #        Start all remaining nodes
         ret = self.startall(None)
         if not ret:
             return self.failure("Could not start the remaining nodes")
 
         return self.success()
 
     def errorstoignore(self):
         '''Return list of errors which should be ignored'''
         # Errors that occur as a result of the CIB being wiped
         return [
             r"error.*: v1 patchset error, patch failed to apply: Application of an update diff failed",
             r"error.*: Resource start-up disabled since no STONITH resources have been defined",
             r"error.*: Either configure some or disable STONITH with the stonith-enabled option",
             r"error.*: NOTE: Clusters with shared data need STONITH to ensure data integrity",
         ]
 
 AllTestClasses.append(SpecialTest1)
 
 
 class HAETest(CTSTest):
     '''Set up a custom test to cause quorum failure issues for Andrew'''
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "HAETest"
         self.stopall = SimulStopLite(cm)
         self.startall = SimulStartLite(cm)
         self.is_loop = 1
 
     def setup(self, node):
         #  Start all remaining nodes
         ret = self.startall(None)
         if not ret:
             return self.failure("Couldn't start all nodes")
         return self.success()
 
     def teardown(self, node):
         # Stop everything
         ret = self.stopall(None)
         if not ret:
             return self.failure("Couldn't stop all nodes")
         return self.success()
 
     def wait_on_state(self, node, resource, expected_clones, attempts=240):
         while attempts > 0:
             active = 0
             (rc, lines) = self.rsh(node, "crm_resource -r %s -W -Q" % resource, verbose=1)
 
             # Hack until crm_resource does the right thing
             if rc == 0 and lines:
                 active = len(lines)
 
             if len(lines) == expected_clones:
                 return 1
 
             elif rc == 1:
                 self.debug("Resource %s is still inactive" % resource)
 
             elif rc == 234:
                 self.logger.log("Unknown resource %s" % resource)
                 return 0
 
             elif rc == 246:
                 self.logger.log("Cluster is inactive")
                 return 0
 
             elif rc != 0:
                 self.logger.log("Call to crm_resource failed, rc=%d" % rc)
                 return 0
 
             else:
                 self.debug("Resource %s is active on %d times instead of %d" % (resource, active, expected_clones))
 
             attempts -= 1
             time.sleep(1)
 
         return 0
 
     def find_dlm(self, node):
         self.r_dlm = None
 
         (_, lines) = self.rsh(node, "crm_resource -c", verbose=1)
         for line in lines:
             if re.search("^Resource", line):
                 r = AuditResource(self.CM, line)
                 if r.rtype == "controld" and r.parent != "NA":
                     self.debug("Found dlm: %s" % self.r_dlm)
                     self.r_dlm = r.parent
                     return 1
         return 0
 
     def find_hae_resources(self, node):
         self.r_dlm = None
         self.r_o2cb = None
         self.r_ocfs2 = []
 
         if self.find_dlm(node):
             self.find_ocfs2_resources(node)
 
     def is_applicable(self):
         if not self.is_applicable_common():
             return False
         if self.Env["Schema"] == "hae":
             return True
         return None
 
 
 class HAERoleTest(HAETest):
     def __init__(self, cm):
         '''Lars' mount/unmount test for the HA extension. '''
         HAETest.__init__(self,cm)
         self.name = "HAERoleTest"
 
     def change_state(self, node, resource, target):
         (rc, _) = self.rsh(node, "crm_resource -V -r %s -p target-role -v %s  --meta" % (resource, target))
         return rc
 
     def __call__(self, node):
         self.incr("calls")
         lpc = 0
         failed = 0
         delay = 2
         done = time.time() + self.Env["loop-minutes"]*60
         self.find_hae_resources(node)
 
         clone_max = len(self.Env["nodes"])
         while time.time() <= done and not failed:
             lpc = lpc + 1
 
             self.change_state(node, self.r_dlm, "Stopped")
             if not self.wait_on_state(node, self.r_dlm, 0):
                 self.failure("%s did not go down correctly" % self.r_dlm)
                 failed = lpc
 
             self.change_state(node, self.r_dlm, "Started")
             if not self.wait_on_state(node, self.r_dlm, clone_max):
                 self.failure("%s did not come up correctly" % self.r_dlm)
                 failed = lpc
 
             if not self.wait_on_state(node, self.r_o2cb, clone_max):
                 self.failure("%s did not come up correctly" % self.r_o2cb)
                 failed = lpc
 
             for fs in self.r_ocfs2:
                 if not self.wait_on_state(node, fs, clone_max):
                     self.failure("%s did not come up correctly" % fs)
                     failed = lpc
 
         if failed:
             return self.failure("iteration %d failed" % failed)
         return self.success()
 
 AllTestClasses.append(HAERoleTest)
 
 
 class HAEStandbyTest(HAETest):
     '''Set up a custom test to cause quorum failure issues for Andrew'''
     def __init__(self, cm):
         HAETest.__init__(self,cm)
         self.name = "HAEStandbyTest"
 
     def change_state(self, node, resource, target):
         (rc, _) = self.rsh(node, "crm_standby -V -l reboot -v %s" % (target))
         return rc
 
     def __call__(self, node):
         self.incr("calls")
 
         lpc = 0
         failed = 0
         done = time.time() + self.Env["loop-minutes"]*60
         self.find_hae_resources(node)
 
         clone_max = len(self.Env["nodes"])
         while time.time() <= done and not failed:
             lpc = lpc + 1
 
             self.change_state(node, self.r_dlm, "true")
             if not self.wait_on_state(node, self.r_dlm, clone_max-1):
                 self.failure("%s did not go down correctly" % self.r_dlm)
                 failed = lpc
 
             self.change_state(node, self.r_dlm, "false")
             if not self.wait_on_state(node, self.r_dlm, clone_max):
                 self.failure("%s did not come up correctly" % self.r_dlm)
                 failed = lpc
 
             if not self.wait_on_state(node, self.r_o2cb, clone_max):
                 self.failure("%s did not come up correctly" % self.r_o2cb)
                 failed = lpc
 
             for fs in self.r_ocfs2:
                 if not self.wait_on_state(node, fs, clone_max):
                     self.failure("%s did not come up correctly" % fs)
                     failed = lpc
 
         if failed:
             return self.failure("iteration %d failed" % failed)
         return self.success()
 
 AllTestClasses.append(HAEStandbyTest)
 
 
 class NearQuorumPointTest(CTSTest):
     '''
     This test brings larger clusters near the quorum point (50%).
     In addition, it will test doing starts and stops at the same time.
 
     Here is how I think it should work:
     - loop over the nodes and decide randomly which will be up and which
       will be down  Use a 50% probability for each of up/down.
     - figure out what to do to get into that state from the current state
     - in parallel, bring up those going up  and bring those going down.
     '''
 
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "NearQuorumPoint"
 
     def __call__(self, dummy):
         '''Perform the 'NearQuorumPoint' test. '''
         self.incr("calls")
         startset = []
         stopset = []
 
         stonith = self.CM.prepare_fencing_watcher("NearQuorumPoint")
         #decide what to do with each node
         for node in self.Env["nodes"]:
             action = self.Env.random_gen.choice(["start","stop"])
             #action = self.Env.random_gen.choice(["start","stop","no change"])
             if action == "start" :
                 startset.append(node)
             elif action == "stop" :
                 stopset.append(node)
 
         self.debug("start nodes:" + repr(startset))
         self.debug("stop nodes:" + repr(stopset))
 
         #add search patterns
         watchpats = [ ]
         for node in stopset:
             if self.CM.ShouldBeStatus[node] == "up":
                 watchpats.append(self.templates["Pat:We_stopped"] % node)
 
         for node in startset:
             if self.CM.ShouldBeStatus[node] == "down":
                 #watchpats.append(self.templates["Pat:NonDC_started"] % node)
                 watchpats.append(self.templates["Pat:Local_started"] % node)
             else:
                 for stopping in stopset:
                     if self.CM.ShouldBeStatus[stopping] == "up":
                         watchpats.append(self.templates["Pat:They_stopped"] % (node, self.CM.key_for_node(stopping)))
 
         if len(watchpats) == 0:
             return self.skipped()
 
         if len(startset) != 0:
             watchpats.append(self.templates["Pat:DC_IDLE"])
 
         watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
 
         watch.set_watch()
 
         #begin actions
         for node in stopset:
             if self.CM.ShouldBeStatus[node] == "up":
                 self.CM.StopaCMnoBlock(node)
 
         for node in startset:
             if self.CM.ShouldBeStatus[node] == "down":
                 self.CM.StartaCMnoBlock(node)
 
         #get the result
         if watch.look_for_all():
             self.CM.cluster_stable()
             self.CM.fencing_cleanup("NearQuorumPoint", stonith)
             return self.success()
 
         self.logger.log("Warn: Patterns not found: " + repr(watch.unmatched))
 
         #get the "bad" nodes
         upnodes = []
         for node in stopset:
             if self.CM.StataCM(node) == 1:
                 upnodes.append(node)
 
         downnodes = []
         for node in startset:
             if self.CM.StataCM(node) == 0:
                 downnodes.append(node)
 
         self.CM.fencing_cleanup("NearQuorumPoint", stonith)
         if upnodes == [] and downnodes == []:
             self.CM.cluster_stable()
 
             # Make sure they're completely down with no residule
             for node in stopset:
                 self.rsh(node, self.templates["StopCmd"])
 
             return self.success()
 
         if len(upnodes) > 0:
             self.logger.log("Warn: Unstoppable nodes: " + repr(upnodes))
 
         if len(downnodes) > 0:
             self.logger.log("Warn: Unstartable nodes: " + repr(downnodes))
 
         return self.failure()
 
     def is_applicable(self):
         return True
 
 AllTestClasses.append(NearQuorumPointTest)
 
 
 class RollingUpgradeTest(CTSTest):
     '''Perform a rolling upgrade of the cluster'''
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "RollingUpgrade"
         self.start = StartTest(cm)
         self.stop = StopTest(cm)
         self.stopall = SimulStopLite(cm)
         self.startall = SimulStartLite(cm)
 
     def setup(self, node):
         #  Start all remaining nodes
         ret = self.stopall(None)
         if not ret:
             return self.failure("Couldn't stop all nodes")
 
         for node in self.Env["nodes"]:
             if not self.downgrade(node, None):
                 return self.failure("Couldn't downgrade %s" % node)
 
         ret = self.startall(None)
         if not ret:
             return self.failure("Couldn't start all nodes")
         return self.success()
 
     def teardown(self, node):
         # Stop everything
         ret = self.stopall(None)
         if not ret:
             return self.failure("Couldn't stop all nodes")
 
         for node in self.Env["nodes"]:
             if not self.upgrade(node, None):
                 return self.failure("Couldn't upgrade %s" % node)
 
         return self.success()
 
     def install(self, node, version, start=1, flags="--force"):
 
         target_dir = "/tmp/rpm-%s" % version
         src_dir = "%s/%s" % (self.Env["rpm-dir"], version)
 
         self.logger.log("Installing %s on %s with %s" % (version, node, flags))
         if not self.stop(node):
             return self.failure("stop failure: "+node)
 
         self.rsh(node, "mkdir -p %s" % target_dir)
         self.rsh(node, "rm -f %s/*.rpm" % target_dir)
         (_, lines) = self.rsh(node, "ls -1 %s/*.rpm" % src_dir, verbose=1)
         for line in lines:
             line = line[:-1]
             rc = self.rsh.copy("%s" % (line), "%s:%s/" % (node, target_dir))
         self.rsh(node, "rpm -Uvh %s %s/*.rpm" % (flags, target_dir))
 
         if start and not self.start(node):
             return self.failure("start failure: "+node)
 
         return self.success()
 
     def upgrade(self, node, start=1):
         return self.install(node, self.Env["current-version"], start)
 
     def downgrade(self, node, start=1):
         return self.install(node, self.Env["previous-version"], start, "--force --nodeps")
 
     def __call__(self, node):
         '''Perform the 'Rolling Upgrade' test. '''
         self.incr("calls")
 
         for node in self.Env["nodes"]:
             if self.upgrade(node):
                 return self.failure("Couldn't upgrade %s" % node)
 
             self.CM.cluster_stable()
 
         return self.success()
 
     def is_applicable(self):
         if not self.is_applicable_common():
             return None
 
         if not "rpm-dir" in list(self.Env.keys()):
             return None
         if not "current-version" in list(self.Env.keys()):
             return None
         if not "previous-version" in list(self.Env.keys()):
             return None
 
         return 1
 
 #        Register RestartTest as a good test to run
 AllTestClasses.append(RollingUpgradeTest)
 
 
 class BSC_AddResource(CTSTest):
     '''Add a resource to the cluster'''
     def __init__(self, cm):
         CTSTest.__init__(self, cm)
         self.name = "AddResource"
         self.resource_offset = 0
         self.cib_cmd = """cibadmin -C -o %s -X '%s' """
 
     def __call__(self, node):
         self.incr("calls")
         self.resource_offset =         self.resource_offset  + 1
 
         r_id = "bsc-rsc-%s-%d" % (node, self.resource_offset)
         start_pat = "pacemaker-controld.*%s_start_0.*confirmed.*ok"
 
         patterns = []
         patterns.append(start_pat % r_id)
 
         watch = self.create_watch(patterns, self.Env["DeadTime"])
         watch.set_watch()
 
         ip = self.NextIP()
         if not self.make_ip_resource(node, r_id, "ocf", "IPaddr", ip):
             return self.failure("Make resource %s failed" % r_id)
 
         failed = 0
         watch_result = watch.look_for_all()
         if watch.unmatched:
             for regex in watch.unmatched:
                 self.logger.log ("Warn: Pattern not found: %s" % (regex))
                 failed = 1
 
         if failed:
             return self.failure("Resource pattern(s) not found")
 
         if not self.CM.cluster_stable(self.Env["DeadTime"]):
             return self.failure("Unstable cluster")
 
         return self.success()
 
     def NextIP(self):
         ip = self.Env["IPBase"]
         if ":" in ip:
             fields = ip.rpartition(":")
             fields[2] = str(hex(int(fields[2], 16)+1))
             print(str(hex(int(f[2], 16)+1)))
         else:
             fields = ip.rpartition('.')
             fields[2] = str(int(fields[2])+1)
 
         ip = fields[0] + fields[1] + fields[3];
         self.Env["IPBase"] = ip
         return ip.strip()
 
     def make_ip_resource(self, node, id, rclass, type, ip):
         self.logger.log("Creating %s:%s:%s (%s) on %s" % (rclass,type,id,ip,node))
         rsc_xml="""
 <primitive id="%s" class="%s" type="%s"  provider="heartbeat">
     <instance_attributes id="%s"><attributes>
         <nvpair id="%s" name="ip" value="%s"/>
     </attributes></instance_attributes>
 </primitive>""" % (id, rclass, type, id, id, ip)
 
         node_constraint = """
       <rsc_location id="run_%s" rsc="%s">
         <rule id="pref_run_%s" score="100">
           <expression id="%s_loc_expr" attribute="#uname" operation="eq" value="%s"/>
         </rule>
       </rsc_location>""" % (id, id, id, id, node)
 
         rc = 0
         (rc, _) = self.rsh(node, self.cib_cmd % ("constraints", node_constraint), verbose=1)
         if rc != 0:
             self.logger.log("Constraint creation failed: %d" % rc)
             return None
 
         (rc, _) = self.rsh(node, self.cib_cmd % ("resources", rsc_xml), verbose=1)
         if rc != 0:
             self.logger.log("Resource creation failed: %d" % rc)
             return None
 
         return 1
 
     def is_applicable(self):
         if self.Env["DoBSC"]:
             return True
         return None
 
 AllTestClasses.append(BSC_AddResource)
 
 
 class SimulStopLite(CTSTest):
     '''Stop any active nodes ~ simultaneously'''
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "SimulStopLite"
 
     def __call__(self, dummy):
         '''Perform the 'SimulStopLite' setup work. '''
         self.incr("calls")
 
         self.debug("Setup: " + self.name)
 
         #     We ignore the "node" parameter...
         watchpats = [ ]
 
         for node in self.Env["nodes"]:
             if self.CM.ShouldBeStatus[node] == "up":
                 self.incr("WasStarted")
                 watchpats.append(self.templates["Pat:We_stopped"] % node)
 
         if len(watchpats) == 0:
             return self.success()
 
         #     Stop all the nodes - at about the same time...
         watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
 
         watch.set_watch()
         self.set_timer()
         for node in self.Env["nodes"]:
             if self.CM.ShouldBeStatus[node] == "up":
                 self.CM.StopaCMnoBlock(node)
         if watch.look_for_all():
             # Make sure they're completely down with no residule
             for node in self.Env["nodes"]:
                 self.rsh(node, self.templates["StopCmd"])
 
             return self.success()
 
         did_fail = 0
         up_nodes = []
         for node in self.Env["nodes"]:
             if self.CM.StataCM(node) == 1:
                 did_fail = 1
                 up_nodes.append(node)
 
         if did_fail:
             return self.failure("Active nodes exist: " + repr(up_nodes))
 
         self.logger.log("Warn: All nodes stopped but CTS didn't detect: "
                     + repr(watch.unmatched))
 
         return self.failure("Missing log message: "+repr(watch.unmatched))
 
     def is_applicable(self):
         '''SimulStopLite is a setup test and never applicable'''
         return False
 
 
 class SimulStartLite(CTSTest):
     '''Start any stopped nodes ~ simultaneously'''
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "SimulStartLite"
 
     def __call__(self, dummy):
         '''Perform the 'SimulStartList' setup work. '''
         self.incr("calls")
         self.debug("Setup: " + self.name)
 
         #        We ignore the "node" parameter...
         node_list = []
         for node in self.Env["nodes"]:
             if self.CM.ShouldBeStatus[node] == "down":
                 self.incr("WasStopped")
                 node_list.append(node)
 
         self.set_timer()
         while len(node_list) > 0:
             # Repeat until all nodes come up
             watchpats = [ ]
 
             uppat = self.templates["Pat:NonDC_started"]
             if self.CM.upcount() == 0:
                 uppat = self.templates["Pat:Local_started"]
 
             watchpats.append(self.templates["Pat:DC_IDLE"])
             for node in node_list:
                 watchpats.append(uppat % node)
                 watchpats.append(self.templates["Pat:InfraUp"] % node)
                 watchpats.append(self.templates["Pat:PacemakerUp"] % node)
 
             #   Start all the nodes - at about the same time...
             watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
             watch.set_watch()
 
             stonith = self.CM.prepare_fencing_watcher(self.name)
 
             for node in node_list:
                 self.CM.StartaCMnoBlock(node)
 
             watch.look_for_all()
 
             node_list = self.CM.fencing_cleanup(self.name, stonith)
 
             if node_list == None:
                 return self.failure("Cluster did not stabilize")
 
             # Remove node_list messages from watch.unmatched
             for node in node_list:
                 self.logger.debug("Dealing with stonith operations for %s" % repr(node_list))
                 if watch.unmatched:
                     try:
                         watch.unmatched.remove(uppat % node)
                     except:
                         self.debug("Already matched: %s" % (uppat % node))
                     try:                        
                         watch.unmatched.remove(self.templates["Pat:InfraUp"] % node)
                     except:
                         self.debug("Already matched: %s" % (self.templates["Pat:InfraUp"] % node))
                     try:
                         watch.unmatched.remove(self.templates["Pat:PacemakerUp"] % node)
                     except:
                         self.debug("Already matched: %s" % (self.templates["Pat:PacemakerUp"] % node))
 
             if watch.unmatched:
                 for regex in watch.unmatched:
                     self.logger.log ("Warn: Startup pattern not found: %s" %(regex))
 
             if not self.CM.cluster_stable():
                 return self.failure("Cluster did not stabilize")
 
         did_fail = 0
         unstable = []
         for node in self.Env["nodes"]:
             if self.CM.StataCM(node) == 0:
                 did_fail = 1
                 unstable.append(node)
 
         if did_fail:
             return self.failure("Unstarted nodes exist: " + repr(unstable))
 
         unstable = []
         for node in self.Env["nodes"]:
             if not self.CM.node_stable(node):
                 did_fail = 1
                 unstable.append(node)
 
         if did_fail:
             return self.failure("Unstable cluster nodes exist: " + repr(unstable))
 
         return self.success()
 
     def is_applicable(self):
         '''SimulStartLite is a setup test and never applicable'''
         return False
 
 
 def TestList(cm, audits):
     result = []
     for testclass in AllTestClasses:
         bound_test = testclass(cm)
         if bound_test.is_applicable():
             bound_test.Audits = audits
             result.append(bound_test)
     return result
 
 
 class RemoteLXC(CTSTest):
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = "RemoteLXC"
         self.start = StartTest(cm)
         self.startall = SimulStartLite(cm)
         self.num_containers = 2
         self.is_container = 1
         self.failed = 0
         self.fail_string = ""
 
     def start_lxc_simple(self, node):
 
         # restore any artifacts laying around from a previous test.
         self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")
 
         # generate the containers, put them in the config, add some resources to them
         pats = [ ]
         watch = self.create_watch(pats, 120)
         watch.set_watch()
         pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc1"))
         pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc2"))
         pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc-ms"))
         pats.append(self.templates["Pat:RscOpOK"] % ("promote", "lxc-ms"))
 
         self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -g -a -m -s -c %d &>/dev/null" % self.num_containers)
         self.set_timer("remoteSimpleInit")
         watch.look_for_all()
         self.log_timer("remoteSimpleInit")
         if watch.unmatched:
             self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
             self.failed = 1
 
     def cleanup_lxc_simple(self, node):
 
         pats = [ ]
         # if the test failed, attempt to clean up the cib and libvirt environment
         # as best as possible 
         if self.failed == 1:
             # restore libvirt and cib
             self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")
             return
 
         watch = self.create_watch(pats, 120)
         watch.set_watch()
 
         pats.append(self.templates["Pat:RscOpOK"] % ("stop", "container1"))
         pats.append(self.templates["Pat:RscOpOK"] % ("stop", "container2"))
 
         self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -p &>/dev/null")
         self.set_timer("remoteSimpleCleanup")
         watch.look_for_all()
         self.log_timer("remoteSimpleCleanup")
 
         if watch.unmatched:
             self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
             self.failed = 1
 
         # cleanup libvirt
         self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")
 
     def __call__(self, node):
         '''Perform the 'RemoteLXC' test. '''
         self.incr("calls")
 
         ret = self.startall(None)
         if not ret:
             return self.failure("Setup failed, start all nodes failed.")
 
         (rc, _) = self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -v &>/dev/null")
         if rc == 1:
             self.log("Environment test for lxc support failed.")
             return self.skipped()
 
         self.start_lxc_simple(node)
         self.cleanup_lxc_simple(node)
 
         self.debug("Waiting for the cluster to recover")
         self.CM.cluster_stable()
 
         if self.failed == 1:
             return self.failure(self.fail_string)
 
         return self.success()
 
     def errorstoignore(self):
         '''Return list of errors which should be ignored'''
         return [
             r"Updating failcount for ping",
             r"schedulerd.*: Recover\s+(ping|lxc-ms|container)\s+\(.*\)",
             # The orphaned lxc-ms resource causes an expected transition error
             # that is a result of the scheduler not having knowledge that the
             # promotable resource used to be a clone. As a result, it looks like that 
             # resource is running in multiple locations when it shouldn't... But in
             # this instance we know why this error is occurring and that it is expected.
             r"Calculated [Tt]ransition .*pe-error",
             r"Resource lxc-ms .* is active on 2 nodes attempting recovery",
             r"Unknown operation: fail",
             r"VirtualDomain.*ERROR: Unable to determine emulator",
         ]
 
 AllTestClasses.append(RemoteLXC)
 
 
 class RemoteDriver(CTSTest):
 
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
         self.name = self.__class__.__name__
         self.start = StartTest(cm)
         self.startall = SimulStartLite(cm)
         self.stop = StopTest(cm)
         self.remote_rsc = "remote-rsc"
         self.cib_cmd = """cibadmin -C -o %s -X '%s' """
         self.reset()
 
     def reset(self):
         self.pcmk_started = 0
         self.failed = False
         self.fail_string = ""
         self.remote_node_added = 0
         self.remote_rsc_added = 0
         self.remote_use_reconnect_interval = self.Env.random_gen.choice([True,False])
 
     def fail(self, msg):
         """ Mark test as failed. """
 
         self.failed = True
 
         # Always log the failure.
         self.logger.log(msg)
 
         # Use first failure as test status, as it's likely to be most useful.
         if not self.fail_string:
             self.fail_string = msg
 
     def get_othernode(self, node):
         for othernode in self.Env["nodes"]:
             if othernode == node:
                 # we don't want to try and use the cib that we just shutdown.
                 # find a cluster node that is not our soon to be remote-node.
                 continue
             else:
                 return othernode
 
     def del_rsc(self, node, rsc):
         othernode = self.get_othernode(node)
         (rc, _) = self.rsh(othernode, "crm_resource -D -r %s -t primitive" % (rsc))
         if rc != 0:
             self.fail("Removal of resource '%s' failed" % rsc)
 
     def add_rsc(self, node, rsc_xml):
         othernode = self.get_othernode(node)
         (rc, _) = self.rsh(othernode, self.cib_cmd % ("resources", rsc_xml))
         if rc != 0:
             self.fail("resource creation failed")
 
     def add_primitive_rsc(self, node):
         rsc_xml = """
 <primitive class="ocf" id="%(node)s" provider="heartbeat" type="Dummy">
   <meta_attributes id="%(node)s-meta_attributes"/>
   <operations>
     <op id="%(node)s-monitor-interval-20s" interval="20s" name="monitor"/>
   </operations>
 </primitive>""" % { "node": self.remote_rsc }
         self.add_rsc(node, rsc_xml)
         if not self.failed:
             self.remote_rsc_added = 1
 
     def add_connection_rsc(self, node):
         rsc_xml = """
 <primitive class="ocf" id="%(node)s" provider="pacemaker" type="remote">
   <instance_attributes id="%(node)s-instance_attributes">
     <nvpair id="%(node)s-instance_attributes-server" name="server" value="%(server)s"/>
 """ % { "node": self.remote_node, "server": node }
 
         if self.remote_use_reconnect_interval:
             # Set reconnect interval on resource
             rsc_xml = rsc_xml + """
     <nvpair id="%s-instance_attributes-reconnect_interval" name="reconnect_interval" value="60s"/>
 """ % (self.remote_node)
 
         rsc_xml = rsc_xml + """
   </instance_attributes>
   <operations>
     <op id="%(node)s-start"       name="start"   interval="0"   timeout="120s"/>
     <op id="%(node)s-monitor-20s" name="monitor" interval="20s" timeout="45s"/>
   </operations>
 </primitive>
 """ % { "node": self.remote_node }
 
         self.add_rsc(node, rsc_xml)
         if not self.failed:
             self.remote_node_added = 1
 
     def disable_services(self, node):
         self.corosync_enabled = self.Env.service_is_enabled(node, "corosync")
         if self.corosync_enabled:
             self.Env.disable_service(node, "corosync")
 
         self.pacemaker_enabled = self.Env.service_is_enabled(node, "pacemaker")
         if self.pacemaker_enabled:
             self.Env.disable_service(node, "pacemaker")
 
     def restore_services(self, node):
         if self.corosync_enabled:
             self.Env.enable_service(node, "corosync")
 
         if self.pacemaker_enabled:
             self.Env.enable_service(node, "pacemaker")
 
     def stop_pcmk_remote(self, node):
         # disable pcmk remote
         for i in range(10):
             (rc, _) = self.rsh(node, "service pacemaker_remote stop")
             if rc != 0:
                 time.sleep(6)
             else:
                 break
 
     def start_pcmk_remote(self, node):
         for i in range(10):
             (rc, _) = self.rsh(node, "service pacemaker_remote start")
             if rc != 0:
                 time.sleep(6)
             else:
                 self.pcmk_started = 1
                 break
 
     def freeze_pcmk_remote(self, node):
         """ Simulate a Pacemaker Remote daemon failure. """
 
         # We freeze the process.
         self.rsh(node, "killall -STOP pacemaker-remoted")
 
     def resume_pcmk_remote(self, node):
         # We resume the process.
         self.rsh(node, "killall -CONT pacemaker-remoted")
 
     def start_metal(self, node):
         # Cluster nodes are reused as remote nodes in remote tests. If cluster
         # services were enabled at boot, in case the remote node got fenced, the
         # cluster node would join instead of the expected remote one. Meanwhile
         # pacemaker_remote would not be able to start. Depending on the chances,
         # the situations might not be able to be orchestrated gracefully any more.
         #
         # Temporarily disable any enabled cluster serivces.
         self.disable_services(node)
 
         pcmk_started = 0
 
         # make sure the resource doesn't already exist for some reason
         self.rsh(node, "crm_resource -D -r %s -t primitive" % (self.remote_rsc))
         self.rsh(node, "crm_resource -D -r %s -t primitive" % (self.remote_node))
 
         if not self.stop(node):
             self.fail("Failed to shutdown cluster node %s" % node)
             return
 
         self.start_pcmk_remote(node)
 
         if self.pcmk_started == 0:
             self.fail("Failed to start pacemaker_remote on node %s" % node)
             return
 
         # Convert node to baremetal now that it has shutdown the cluster stack
         pats = [ ]
         watch = self.create_watch(pats, 120)
         watch.set_watch()
         pats.append(self.templates["Pat:RscOpOK"] % ("start", self.remote_node))
         pats.append(self.templates["Pat:DC_IDLE"])
 
         self.add_connection_rsc(node)
 
         self.set_timer("remoteMetalInit")
         watch.look_for_all()
         self.log_timer("remoteMetalInit")
         if watch.unmatched:
             self.fail("Unmatched patterns: %s" % watch.unmatched)
 
     def migrate_connection(self, node):
         if self.failed:
             return
 
         pats = [ ]
         pats.append(self.templates["Pat:RscOpOK"] % ("migrate_to", self.remote_node))
         pats.append(self.templates["Pat:RscOpOK"] % ("migrate_from", self.remote_node))
         pats.append(self.templates["Pat:DC_IDLE"])
         watch = self.create_watch(pats, 120)
         watch.set_watch()
 
         (rc, _) = self.rsh(node, "crm_resource -M -r %s" % (self.remote_node), verbose=1)
         if rc != 0:
             self.fail("failed to move remote node connection resource")
             return
 
         self.set_timer("remoteMetalMigrate")
         watch.look_for_all()
         self.log_timer("remoteMetalMigrate")
 
         if watch.unmatched:
             self.fail("Unmatched patterns: %s" % watch.unmatched)
             return
 
     def fail_rsc(self, node):
         if self.failed:
             return
 
         watchpats = [ ]
         watchpats.append(self.templates["Pat:RscRemoteOpOK"] % ("stop", self.remote_rsc, self.remote_node))
         watchpats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))
         watchpats.append(self.templates["Pat:DC_IDLE"])
 
         watch = self.create_watch(watchpats, 120)
         watch.set_watch()
 
         self.debug("causing dummy rsc to fail.")
 
         self.rsh(node, "rm -f /var/run/resource-agents/Dummy*")
 
         self.set_timer("remoteRscFail")
         watch.look_for_all()
         self.log_timer("remoteRscFail")
         if watch.unmatched:
             self.fail("Unmatched patterns during rsc fail: %s" % watch.unmatched)
 
     def fail_connection(self, node):
         if self.failed:
             return
 
         watchpats = [ ]
         watchpats.append(self.templates["Pat:Fencing_ok"] % self.remote_node)
         watchpats.append(self.templates["Pat:NodeFenced"] % self.remote_node)
 
         watch = self.create_watch(watchpats, 120)
         watch.set_watch()
 
         # freeze the pcmk remote daemon. this will result in fencing
         self.debug("Force stopped active remote node")
         self.freeze_pcmk_remote(node)
 
         self.debug("Waiting for remote node to be fenced.")
         self.set_timer("remoteMetalFence")
         watch.look_for_all()
         self.log_timer("remoteMetalFence")
         if watch.unmatched:
             self.fail("Unmatched patterns: %s" % watch.unmatched)
             return
 
         self.debug("Waiting for the remote node to come back up")
         self.CM.ns.wait_for_node(node, 120);
 
         pats = [ ]
         watch = self.create_watch(pats, 240)
         watch.set_watch()
         pats.append(self.templates["Pat:RscOpOK"] % ("start", self.remote_node))
         if self.remote_rsc_added == 1:
             pats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))
 
         # start the remote node again watch it integrate back into cluster.
         self.start_pcmk_remote(node)
         if self.pcmk_started == 0:
             self.fail("Failed to start pacemaker_remote on node %s" % node)
             return
 
         self.debug("Waiting for remote node to rejoin cluster after being fenced.")
         self.set_timer("remoteMetalRestart")
         watch.look_for_all()
         self.log_timer("remoteMetalRestart")
         if watch.unmatched:
             self.fail("Unmatched patterns: %s" % watch.unmatched)
             return
 
     def add_dummy_rsc(self, node):
         if self.failed:
             return
 
         # verify we can put a resource on the remote node
         pats = [ ]
         watch = self.create_watch(pats, 120)
         watch.set_watch()
         pats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))
         pats.append(self.templates["Pat:DC_IDLE"])
 
         # Add a resource that must live on remote-node
         self.add_primitive_rsc(node)
 
         # force that rsc to prefer the remote node. 
         (rc, _) = self.CM.rsh(node, "crm_resource -M -r %s -N %s -f" % (self.remote_rsc, self.remote_node), verbose=1)
         if rc != 0:
             self.fail("Failed to place remote resource on remote node.")
             return
 
         self.set_timer("remoteMetalRsc")
         watch.look_for_all()
         self.log_timer("remoteMetalRsc")
         if watch.unmatched:
             self.fail("Unmatched patterns: %s" % watch.unmatched)
 
     def test_attributes(self, node):
         if self.failed:
             return
 
         # This verifies permanent attributes can be set on a remote-node. It also
         # verifies the remote-node can edit its own cib node section remotely.
         (rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -v testval -N %s" % (self.remote_node), verbose=1)
         if rc != 0:
             self.fail("Failed to set remote-node attribute. rc:%s output:%s" % (rc, line))
             return
 
         (rc, _) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -q -N %s" % (self.remote_node), verbose=1)
         if rc != 0:
             self.fail("Failed to get remote-node attribute")
             return
 
         (rc, _) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -D -N %s" % (self.remote_node), verbose=1)
         if rc != 0:
             self.fail("Failed to delete remote-node attribute")
             return
 
     def cleanup_metal(self, node):
         self.restore_services(node)
 
         if self.pcmk_started == 0:
             return
 
         pats = [ ]
 
         watch = self.create_watch(pats, 120)
         watch.set_watch()
 
         if self.remote_rsc_added == 1:
             pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.remote_rsc))
         if self.remote_node_added == 1:
             pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.remote_node))
 
         self.set_timer("remoteMetalCleanup")
 
         self.resume_pcmk_remote(node)
 
         if self.remote_rsc_added == 1:
 
             # Remove dummy resource added for remote node tests
             self.debug("Cleaning up dummy rsc put on remote node")
             self.rsh(self.get_othernode(node), "crm_resource -U -r %s" % self.remote_rsc)
             self.del_rsc(node, self.remote_rsc)
 
         if self.remote_node_added == 1:
 
             # Remove remote node's connection resource
             self.debug("Cleaning up remote node connection resource")
             self.rsh(self.get_othernode(node), "crm_resource -U -r %s" % (self.remote_node))
             self.del_rsc(node, self.remote_node)
 
         watch.look_for_all()
         self.log_timer("remoteMetalCleanup")
 
         if watch.unmatched:
             self.fail("Unmatched patterns: %s" % watch.unmatched)
 
         self.stop_pcmk_remote(node)
 
         self.debug("Waiting for the cluster to recover")
         self.CM.cluster_stable()
 
         if self.remote_node_added == 1:
             # Remove remote node itself
             self.debug("Cleaning up node entry for remote node")
             self.rsh(self.get_othernode(node), "crm_node --force --remove %s" % self.remote_node)
 
     def setup_env(self, node):
 
         self.remote_node = "remote-%s" % (node)
 
         # we are assuming if all nodes have a key, that it is
         # the right key... If any node doesn't have a remote
         # key, we regenerate it everywhere.
         if self.rsh.exists_on_all("/etc/pacemaker/authkey", self.Env["nodes"]):
             return
 
         # create key locally
         (handle, keyfile) = tempfile.mkstemp(".cts")
         os.close(handle)
         subprocess.check_call(["dd", "if=/dev/urandom", "of=%s" % keyfile, "bs=4096", "count=1"],
             stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 
         # sync key throughout the cluster
         for node in self.Env["nodes"]:
             self.rsh(node, "mkdir -p --mode=0750 /etc/pacemaker")
             self.rsh.copy(keyfile, "root@%s:/etc/pacemaker/authkey" % node)
             self.rsh(node, "chgrp haclient /etc/pacemaker /etc/pacemaker/authkey")
             self.rsh(node, "chmod 0640 /etc/pacemaker/authkey")
         os.unlink(keyfile)
 
     def is_applicable(self):
         if not self.is_applicable_common():
             return False
 
         for node in self.Env["nodes"]:
             (rc, _) = self.rsh(node, "which pacemaker-remoted >/dev/null 2>&1")
             if rc != 0:
                 return False
         return True
 
     def start_new_test(self, node):
         self.incr("calls")
         self.reset()
 
         ret = self.startall(None)
         if not ret:
             return self.failure("setup failed: could not start all nodes")
 
         self.setup_env(node)
         self.start_metal(node)
         self.add_dummy_rsc(node)
         return True
 
     def __call__(self, node):
         return self.failure("This base class is not meant to be called directly.")
 
     def errorstoignore(self):
         '''Return list of errors which should be ignored'''
         return [ r"""is running on remote.*which isn't allowed""",
                  r"""Connection terminated""",
                  r"""Could not send remote""",
                 ]
 
 # RemoteDriver is just a base class for other tests, so it is not added to AllTestClasses
 
 
 class RemoteBasic(RemoteDriver):
 
     def __call__(self, node):
         '''Perform the 'RemoteBaremetal' test. '''
 
         if not self.start_new_test(node):
             return self.failure(self.fail_string)
 
         self.test_attributes(node)
         self.cleanup_metal(node)
 
         self.debug("Waiting for the cluster to recover")
         self.CM.cluster_stable()
         if self.failed:
             return self.failure(self.fail_string)
 
         return self.success()
 
 AllTestClasses.append(RemoteBasic)
 
 class RemoteStonithd(RemoteDriver):
 
     def __call__(self, node):
         '''Perform the 'RemoteStonithd' test. '''
 
         if not self.start_new_test(node):
             return self.failure(self.fail_string)
 
         self.fail_connection(node)
         self.cleanup_metal(node)
 
         self.debug("Waiting for the cluster to recover")
         self.CM.cluster_stable()
         if self.failed:
             return self.failure(self.fail_string)
 
         return self.success()
 
     def is_applicable(self):
         if not RemoteDriver.is_applicable(self):
             return False
 
         if "DoFencing" in list(self.Env.keys()):
             return self.Env["DoFencing"]
 
         return True
 
     def errorstoignore(self):
         ignore_pats = [
             r"Lost connection to Pacemaker Remote node",
             r"Software caused connection abort",
             r"pacemaker-controld.*:\s+error.*: Operation remote-.*_monitor",
             r"pacemaker-controld.*:\s+error.*: Result of monitor operation for remote-.*",
             r"schedulerd.*:\s+Recover\s+remote-.*\s+\(.*\)",
             r"error: Result of monitor operation for .* on remote-.*: Internal communication failure",
         ]
 
         ignore_pats.extend(RemoteDriver.errorstoignore(self))
         return ignore_pats
 
 AllTestClasses.append(RemoteStonithd)
 
 
 class RemoteMigrate(RemoteDriver):
 
     def __call__(self, node):
         '''Perform the 'RemoteMigrate' test. '''
 
         if not self.start_new_test(node):
             return self.failure(self.fail_string)
 
         self.migrate_connection(node)
         self.cleanup_metal(node)
 
         self.debug("Waiting for the cluster to recover")
         self.CM.cluster_stable()
         if self.failed:
             return self.failure(self.fail_string)
 
         return self.success()
 
     def is_applicable(self):
         if not RemoteDriver.is_applicable(self):
             return 0
         # This test requires at least three nodes: one to convert to a
         # remote node, one to host the connection originally, and one
         # to migrate the connection to.
         if len(self.Env["nodes"]) < 3:
             return 0
         return 1
 
 AllTestClasses.append(RemoteMigrate)
 
 
 class RemoteRscFailure(RemoteDriver):
 
     def __call__(self, node):
         '''Perform the 'RemoteRscFailure' test. '''
 
         if not self.start_new_test(node):
             return self.failure(self.fail_string)
 
         # This is an important step. We are migrating the connection
         # before failing the resource. This verifies that the migration
         # has properly maintained control over the remote-node.
         self.migrate_connection(node)
 
         self.fail_rsc(node)
         self.cleanup_metal(node)
 
         self.debug("Waiting for the cluster to recover")
         self.CM.cluster_stable()
         if self.failed:
             return self.failure(self.fail_string)
 
         return self.success()
 
     def errorstoignore(self):
         ignore_pats = [
             r"schedulerd.*: Recover\s+remote-rsc\s+\(.*\)",
             r"Dummy.*: No process state file found",
         ]
 
         ignore_pats.extend(RemoteDriver.errorstoignore(self))
         return ignore_pats
 
     def is_applicable(self):
         if not RemoteDriver.is_applicable(self):
             return 0
         # This test requires at least three nodes: one to convert to a
         # remote node, one to host the connection originally, and one
         # to migrate the connection to.
         if len(self.Env["nodes"]) < 3:
             return 0
         return 1
 
 AllTestClasses.append(RemoteRscFailure)
 
 # vim:ts=4:sw=4:et:
diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c
index cdfa08192b..89cb61fa3d 100644
--- a/daemons/controld/controld_fencing.c
+++ b/daemons/controld/controld_fencing.c
@@ -1,1106 +1,1108 @@
 /*
  * Copyright 2004-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/stonith-ng.h>
 #include <crm/fencing/internal.h>
 
 #include <pacemaker-controld.h>
 
 static void
 tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event);
 
 /*
  * stonith failure counting
  *
  * We don't want to get stuck in a permanent fencing loop. Keep track of the
  * number of fencing failures for each target node, and the most we'll restart a
  * transition for.
  */
 
 struct st_fail_rec {
     int count;
 };
 
 static bool fence_reaction_panic = false;
 static unsigned long int stonith_max_attempts = 10;
 static GHashTable *stonith_failures = NULL;
 
 /*!
  * \internal
  * \brief Update max fencing attempts before giving up
  *
  * \param[in] value  New max fencing attempts
  */
 static void
 update_stonith_max_attempts(const char *value)
 {
     stonith_max_attempts = char2score(value);
     if (stonith_max_attempts < 1UL) {
         stonith_max_attempts = 10UL;
     }
 }
 
 /*!
  * \internal
  * \brief Configure reaction to notification of local node being fenced
  *
  * \param[in] reaction_s  Reaction type
  */
 static void
 set_fence_reaction(const char *reaction_s)
 {
     if (pcmk__str_eq(reaction_s, "panic", pcmk__str_casei)) {
         fence_reaction_panic = true;
 
     } else {
         if (!pcmk__str_eq(reaction_s, "stop", pcmk__str_casei)) {
             crm_warn("Invalid value '%s' for %s, using 'stop'",
                      reaction_s, XML_CONFIG_ATTR_FENCE_REACTION);
         }
         fence_reaction_panic = false;
     }
 }
 
 /*!
  * \internal
  * \brief Configure fencing options based on the CIB
  *
  * \param[in,out] options  Name/value pairs for configured options
  */
 void
 controld_configure_fencing(GHashTable *options)
 {
     const char *value = NULL;
 
     value = g_hash_table_lookup(options, XML_CONFIG_ATTR_FENCE_REACTION);
     set_fence_reaction(value);
 
     value = g_hash_table_lookup(options, "stonith-max-attempts");
     update_stonith_max_attempts(value);
 }
 
 static gboolean
 too_many_st_failures(const char *target)
 {
     GHashTableIter iter;
     const char *key = NULL;
     struct st_fail_rec *value = NULL;
 
     if (stonith_failures == NULL) {
         return FALSE;
     }
 
     if (target == NULL) {
         g_hash_table_iter_init(&iter, stonith_failures);
         while (g_hash_table_iter_next(&iter, (gpointer *) &key,
                (gpointer *) &value)) {
 
             if (value->count >= stonith_max_attempts) {
                 target = (const char*)key;
                 goto too_many;
             }
         }
     } else {
         value = g_hash_table_lookup(stonith_failures, target);
         if ((value != NULL) && (value->count >= stonith_max_attempts)) {
             goto too_many;
         }
     }
     return FALSE;
 
 too_many:
     crm_warn("Too many failures (%d) to fence %s, giving up",
              value->count, target);
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Reset a stonith fail count
  *
  * \param[in] target  Name of node to reset, or NULL for all
  */
 void
 st_fail_count_reset(const char *target)
 {
     if (stonith_failures == NULL) {
         return;
     }
 
     if (target) {
         struct st_fail_rec *rec = NULL;
 
         rec = g_hash_table_lookup(stonith_failures, target);
         if (rec) {
             rec->count = 0;
         }
     } else {
         GHashTableIter iter;
         const char *key = NULL;
         struct st_fail_rec *rec = NULL;
 
         g_hash_table_iter_init(&iter, stonith_failures);
         while (g_hash_table_iter_next(&iter, (gpointer *) &key,
                                       (gpointer *) &rec)) {
             rec->count = 0;
         }
     }
 }
 
 static void
 st_fail_count_increment(const char *target)
 {
     struct st_fail_rec *rec = NULL;
 
     if (stonith_failures == NULL) {
         stonith_failures = pcmk__strkey_table(free, free);
     }
 
     rec = g_hash_table_lookup(stonith_failures, target);
     if (rec) {
         rec->count++;
     } else {
         rec = malloc(sizeof(struct st_fail_rec));
         if(rec == NULL) {
             return;
         }
 
         rec->count = 1;
         g_hash_table_insert(stonith_failures, strdup(target), rec);
     }
 }
 
 /* end stonith fail count functions */
 
 
 static void
 cib_fencing_updated(xmlNode *msg, int call_id, int rc, xmlNode *output,
                     void *user_data)
 {
     if (rc < pcmk_ok) {
         crm_err("Fencing update %d for %s: failed - %s (%d)",
                 call_id, (char *)user_data, pcmk_strerror(rc), rc);
         crm_log_xml_warn(msg, "Failed update");
         abort_transition(INFINITY, pcmk__graph_shutdown, "CIB update failed",
                          NULL);
 
     } else {
         crm_info("Fencing update %d for %s: complete", call_id, (char *)user_data);
     }
 }
 
 static void
 send_stonith_update(pcmk__graph_action_t *action, const char *target,
                     const char *uuid)
 {
     int rc = pcmk_ok;
     crm_node_t *peer = NULL;
 
     /* We (usually) rely on the membership layer to do node_update_cluster,
      * and the peer status callback to do node_update_peer, because the node
      * might have already rejoined before we get the stonith result here.
      */
     int flags = node_update_join | node_update_expected;
 
     /* zero out the node-status & remove all LRM status info */
     xmlNode *node_state = NULL;
 
     CRM_CHECK(target != NULL, return);
     CRM_CHECK(uuid != NULL, return);
 
     /* Make sure the membership and join caches are accurate */
     peer = crm_get_peer_full(0, target, CRM_GET_PEER_ANY);
 
     CRM_CHECK(peer != NULL, return);
 
     if (peer->state == NULL) {
         /* Usually, we rely on the membership layer to update the cluster state
          * in the CIB. However, if the node has never been seen, do it here, so
          * the node is not considered unclean.
          */
         flags |= node_update_cluster;
     }
 
     if (peer->uuid == NULL) {
         crm_info("Recording uuid '%s' for node '%s'", uuid, target);
         peer->uuid = strdup(uuid);
     }
 
     crmd_peer_down(peer, TRUE);
 
     /* Generate a node state update for the CIB */
     node_state = create_node_state_update(peer, flags, NULL, __func__);
 
     /* we have to mark whether or not remote nodes have already been fenced */
     if (peer->flags & crm_remote_node) {
         char *now_s = pcmk__ttoa(time(NULL));
 
         crm_xml_add(node_state, XML_NODE_IS_FENCED, now_s);
         free(now_s);
     }
 
     /* Force our known ID */
     crm_xml_add(node_state, XML_ATTR_ID, uuid);
 
     rc = controld_globals.cib_conn->cmds->modify(controld_globals.cib_conn,
                                                  XML_CIB_TAG_STATUS, node_state,
                                                  cib_scope_local
                                                  |cib_can_create);
 
     /* Delay processing the trigger until the update completes */
     crm_debug("Sending fencing update %d for %s", rc, target);
     fsa_register_cib_callback(rc, strdup(target), cib_fencing_updated);
 
     // Make sure it sticks
     /* controld_globals.cib_conn->cmds->bump_epoch(controld_globals.cib_conn,
      *                                             cib_scope_local);
      */
 
     controld_delete_node_state(peer->uname, controld_section_all,
                                cib_scope_local);
     free_xml(node_state);
     return;
 }
 
 /*!
  * \internal
  * \brief Abort transition due to stonith failure
  *
  * \param[in] abort_action  Whether to restart or stop transition
  * \param[in] target  Don't restart if this (NULL for any) has too many failures
  * \param[in] reason  Log this stonith action XML as abort reason (or NULL)
  */
 static void
 abort_for_stonith_failure(enum pcmk__graph_next abort_action,
                           const char *target, const xmlNode *reason)
 {
     /* If stonith repeatedly fails, we eventually give up on starting a new
      * transition for that reason.
      */
     if ((abort_action != pcmk__graph_wait) && too_many_st_failures(target)) {
         abort_action = pcmk__graph_wait;
     }
     abort_transition(INFINITY, abort_action, "Stonith failed", reason);
 }
 
 
 /*
  * stonith cleanup list
  *
  * If the DC is shot, proper notifications might not go out.
  * The stonith cleanup list allows the cluster to (re-)send
  * notifications once a new DC is elected.
  */
 
 static GList *stonith_cleanup_list = NULL;
 
 /*!
  * \internal
  * \brief Add a node to the stonith cleanup list
  *
  * \param[in] target  Name of node to add
  */
 void
 add_stonith_cleanup(const char *target) {
     stonith_cleanup_list = g_list_append(stonith_cleanup_list, strdup(target));
 }
 
 /*!
  * \internal
  * \brief Remove a node from the stonith cleanup list
  *
  * \param[in] Name of node to remove
  */
 void
 remove_stonith_cleanup(const char *target)
 {
     GList *iter = stonith_cleanup_list;
 
     while (iter != NULL) {
         GList *tmp = iter;
         char *iter_name = tmp->data;
 
         iter = iter->next;
         if (pcmk__str_eq(target, iter_name, pcmk__str_casei)) {
             crm_trace("Removing %s from the cleanup list", iter_name);
             stonith_cleanup_list = g_list_delete_link(stonith_cleanup_list, tmp);
             free(iter_name);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Purge all entries from the stonith cleanup list
  */
 void
 purge_stonith_cleanup(void)
 {
     if (stonith_cleanup_list) {
         GList *iter = NULL;
 
         for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) {
             char *target = iter->data;
 
             crm_info("Purging %s from stonith cleanup list", target);
             free(target);
         }
         g_list_free(stonith_cleanup_list);
         stonith_cleanup_list = NULL;
     }
 }
 
 /*!
  * \internal
  * \brief Send stonith updates for all entries in cleanup list, then purge it
  */
 void
 execute_stonith_cleanup(void)
 {
     GList *iter;
 
     for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) {
         char *target = iter->data;
         crm_node_t *target_node = crm_get_peer(0, target);
         const char *uuid = crm_peer_uuid(target_node);
 
         crm_notice("Marking %s, target of a previous stonith action, as clean", target);
         send_stonith_update(NULL, target, uuid);
         free(target);
     }
     g_list_free(stonith_cleanup_list);
     stonith_cleanup_list = NULL;
 }
 
 /* end stonith cleanup list functions */
 
 
 /* stonith API client
  *
  * Functions that need to interact directly with the fencer via its API
  */
 
 static stonith_t *stonith_api = NULL;
 static crm_trigger_t *stonith_reconnect = NULL;
 static char *te_client_id = NULL;
 
 static gboolean
 fail_incompletable_stonith(pcmk__graph_t *graph)
 {
     GList *lpc = NULL;
     const char *task = NULL;
     xmlNode *last_action = NULL;
 
     if (graph == NULL) {
         return FALSE;
     }
 
     for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) {
         GList *lpc2 = NULL;
         pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) lpc->data;
 
         if (pcmk_is_set(synapse->flags, pcmk__synapse_confirmed)) {
             continue;
         }
 
         for (lpc2 = synapse->actions; lpc2 != NULL; lpc2 = lpc2->next) {
             pcmk__graph_action_t *action = (pcmk__graph_action_t *) lpc2->data;
 
             if ((action->type != pcmk__cluster_graph_action)
                 || pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
                 continue;
             }
 
             task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
             if (task && pcmk__str_eq(task, CRM_OP_FENCE, pcmk__str_casei)) {
                 pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
                 last_action = action->xml;
                 pcmk__update_graph(graph, action);
                 crm_notice("Failing action %d (%s): fencer terminated",
                            action->id, ID(action->xml));
             }
         }
     }
 
     if (last_action != NULL) {
         crm_warn("Fencer failure resulted in unrunnable actions");
         abort_for_stonith_failure(pcmk__graph_restart, NULL, last_action);
         return TRUE;
     }
 
     return FALSE;
 }
 
 static void
 tengine_stonith_connection_destroy(stonith_t *st, stonith_event_t *e)
 {
     te_cleanup_stonith_history_sync(st, FALSE);
 
     if (pcmk_is_set(controld_globals.fsa_input_register, R_ST_REQUIRED)) {
         crm_crit("Fencing daemon connection failed");
         mainloop_set_trigger(stonith_reconnect);
 
     } else {
         crm_info("Fencing daemon disconnected");
     }
 
     if (stonith_api) {
         /* the client API won't properly reconnect notifications
          * if they are still in the table - so remove them
          */
         if (stonith_api->state != stonith_disconnected) {
             stonith_api->cmds->disconnect(st);
         }
         stonith_api->cmds->remove_notification(stonith_api, NULL);
     }
 
     if (AM_I_DC) {
         fail_incompletable_stonith(controld_globals.transition_graph);
         trigger_graph();
     }
 }
 
 /*!
  * \internal
  * \brief Handle an event notification from the fencing API
  *
  * \param[in] st     Fencing API connection (ignored)
  * \param[in] event  Fencing API event notification
  */
 static void
 handle_fence_notification(stonith_t *st, stonith_event_t *event)
 {
     bool succeeded = true;
     const char *executioner = "the cluster";
     const char *client = "a client";
     const char *reason = NULL;
     int exec_status;
 
     if (te_client_id == NULL) {
         te_client_id = crm_strdup_printf("%s.%lu", crm_system_name,
                                          (unsigned long) getpid());
     }
 
     if (event == NULL) {
         crm_err("Notify data not found");
         return;
     }
 
     if (event->executioner != NULL) {
         executioner = event->executioner;
     }
     if (event->client_origin != NULL) {
         client = event->client_origin;
     }
 
     exec_status = stonith__event_execution_status(event);
     if ((stonith__event_exit_status(event) != CRM_EX_OK)
         || (exec_status != PCMK_EXEC_DONE)) {
         succeeded = false;
         if (exec_status == PCMK_EXEC_DONE) {
             exec_status = PCMK_EXEC_ERROR;
         }
     }
     reason = stonith__event_exit_reason(event);
 
     crmd_alert_fencing_op(event);
 
     if (pcmk__str_eq("on", event->action, pcmk__str_none)) {
         // Unfencing doesn't need special handling, just a log message
         if (succeeded) {
             crm_notice("%s was unfenced by %s at the request of %s@%s",
                        event->target, executioner, client, event->origin);
         } else {
             crm_err("Unfencing of %s by %s failed (%s%s%s) with exit status %d",
                     event->target, executioner,
                     pcmk_exec_status_str(exec_status),
                     ((reason == NULL)? "" : ": "),
                     ((reason == NULL)? "" : reason),
                     stonith__event_exit_status(event));
         }
         return;
     }
 
     if (succeeded
         && pcmk__str_eq(event->target, controld_globals.our_nodename,
                         pcmk__str_casei)) {
         /* We were notified of our own fencing. Most likely, either fencing was
          * misconfigured, or fabric fencing that doesn't cut cluster
          * communication is in use.
          *
          * Either way, shutting down the local host is a good idea, to require
          * administrator intervention. Also, other nodes would otherwise likely
          * set our status to lost because of the fencing callback and discard
          * our subsequent election votes as "not part of our cluster".
          */
         crm_crit("We were allegedly just fenced by %s for %s!",
                  executioner, event->origin); // Dumps blackbox if enabled
         if (fence_reaction_panic) {
             pcmk__panic(__func__);
         } else {
             crm_exit(CRM_EX_FATAL);
         }
         return; // Should never get here
     }
 
     /* Update the count of fencing failures for this target, in case we become
      * DC later. The current DC has already updated its fail count in
      * tengine_stonith_callback().
      */
     if (!AM_I_DC) {
         if (succeeded) {
             st_fail_count_reset(event->target);
         } else {
             st_fail_count_increment(event->target);
         }
     }
 
     crm_notice("Peer %s was%s terminated (%s) by %s on behalf of %s@%s: "
                "%s%s%s%s " CRM_XS " event=%s",
                event->target, (succeeded? "" : " not"),
                event->action, executioner, client, event->origin,
                (succeeded? "OK" : pcmk_exec_status_str(exec_status)),
                ((reason == NULL)? "" : " ("),
                ((reason == NULL)? "" : reason),
                ((reason == NULL)? "" : ")"),
                event->id);
 
     if (succeeded) {
         crm_node_t *peer = pcmk__search_known_node_cache(0, event->target,
                                                          CRM_GET_PEER_ANY);
         const char *uuid = NULL;
 
         if (peer == NULL) {
             return;
         }
 
         uuid = crm_peer_uuid(peer);
 
         if (AM_I_DC) {
             /* The DC always sends updates */
             send_stonith_update(NULL, event->target, uuid);
 
             /* @TODO Ideally, at this point, we'd check whether the fenced node
              * hosted any guest nodes, and call remote_node_down() for them.
              * Unfortunately, the controller doesn't have a simple, reliable way
              * to map hosts to guests. It might be possible to track this in the
              * peer cache via crm_remote_peer_cache_refresh(). For now, we rely
              * on the scheduler creating fence pseudo-events for the guests.
              */
 
             if (!pcmk__str_eq(client, te_client_id, pcmk__str_casei)) {
                 /* Abort the current transition if it wasn't the cluster that
                  * initiated fencing.
                  */
                 crm_info("External fencing operation from %s fenced %s",
                          client, event->target);
                 abort_transition(INFINITY, pcmk__graph_restart,
                                  "External Fencing Operation", NULL);
             }
 
         } else if (pcmk__str_eq(controld_globals.dc_name, event->target,
                                 pcmk__str_null_matches|pcmk__str_casei)
                    && !pcmk_is_set(peer->flags, crm_remote_node)) {
             // Assume the target was our DC if we don't currently have one
 
             if (controld_globals.dc_name != NULL) {
                 crm_notice("Fencing target %s was our DC", event->target);
             } else {
                 crm_notice("Fencing target %s may have been our DC",
                            event->target);
             }
 
             /* Given the CIB resyncing that occurs around elections,
              * have one node update the CIB now and, if the new DC is different,
              * have them do so too after the election
              */
             if (pcmk__str_eq(event->executioner, controld_globals.our_nodename,
                              pcmk__str_casei)) {
                 send_stonith_update(NULL, event->target, uuid);
             }
             add_stonith_cleanup(event->target);
         }
 
         /* If the target is a remote node, and we host its connection,
          * immediately fail all monitors so it can be recovered quickly.
          * The connection won't necessarily drop when a remote node is fenced,
          * so the failure might not otherwise be detected until the next poke.
          */
         if (pcmk_is_set(peer->flags, crm_remote_node)) {
             remote_ra_fail(event->target);
         }
 
         crmd_peer_down(peer, TRUE);
      }
 }
 
 /*!
  * \brief Connect to fencer
  *
  * \param[in] user_data  If NULL, retry failures now, otherwise retry in main loop
  *
  * \return TRUE
  * \note If user_data is NULL, this will wait 2s between attempts, for up to
  *       30 attempts, meaning the controller could be blocked as long as 58s.
  */
 static gboolean
 te_connect_stonith(gpointer user_data)
 {
     int rc = pcmk_ok;
 
     if (stonith_api == NULL) {
         stonith_api = stonith_api_new();
         if (stonith_api == NULL) {
             crm_err("Could not connect to fencer: API memory allocation failed");
             return TRUE;
         }
     }
 
     if (stonith_api->state != stonith_disconnected) {
         crm_trace("Already connected to fencer, no need to retry");
         return TRUE;
     }
 
     if (user_data == NULL) {
         // Blocking (retry failures now until successful)
         rc = stonith_api_connect_retry(stonith_api, crm_system_name, 30);
         if (rc != pcmk_ok) {
             crm_err("Could not connect to fencer in 30 attempts: %s "
                     CRM_XS " rc=%d", pcmk_strerror(rc), rc);
         }
     } else {
         // Non-blocking (retry failures later in main loop)
         rc = stonith_api->cmds->connect(stonith_api, crm_system_name, NULL);
         if (rc != pcmk_ok) {
             if (pcmk_is_set(controld_globals.fsa_input_register,
                             R_ST_REQUIRED)) {
                 crm_notice("Fencer connection failed (will retry): %s "
                            CRM_XS " rc=%d", pcmk_strerror(rc), rc);
                 mainloop_set_trigger(stonith_reconnect);
             } else {
                 crm_info("Fencer connection failed (ignoring because no longer required): %s "
                          CRM_XS " rc=%d", pcmk_strerror(rc), rc);
             }
             return TRUE;
         }
     }
 
     if (rc == pcmk_ok) {
         stonith_api->cmds->register_notification(stonith_api,
                                                  T_STONITH_NOTIFY_DISCONNECT,
                                                  tengine_stonith_connection_destroy);
         stonith_api->cmds->register_notification(stonith_api,
                                                  T_STONITH_NOTIFY_FENCE,
                                                  handle_fence_notification);
         stonith_api->cmds->register_notification(stonith_api,
                                                  T_STONITH_NOTIFY_HISTORY_SYNCED,
                                                  tengine_stonith_history_synced);
         te_trigger_stonith_history_sync(TRUE);
         crm_notice("Fencer successfully connected");
     }
 
     return TRUE;
 }
 
 /*!
     \internal
     \brief Schedule fencer connection attempt in main loop
 */
 void
 controld_trigger_fencer_connect(void)
 {
     if (stonith_reconnect == NULL) {
         stonith_reconnect = mainloop_add_trigger(G_PRIORITY_LOW,
                                                  te_connect_stonith,
                                                  GINT_TO_POINTER(TRUE));
     }
     controld_set_fsa_input_flags(R_ST_REQUIRED);
     mainloop_set_trigger(stonith_reconnect);
 }
 
 void
 controld_disconnect_fencer(bool destroy)
 {
     if (stonith_api) {
         // Prevent fencer connection from coming up again
         controld_clear_fsa_input_flags(R_ST_REQUIRED);
 
         if (stonith_api->state != stonith_disconnected) {
             stonith_api->cmds->disconnect(stonith_api);
         }
         stonith_api->cmds->remove_notification(stonith_api, NULL);
     }
     if (destroy) {
         if (stonith_api) {
             stonith_api->cmds->free(stonith_api);
             stonith_api = NULL;
         }
         if (stonith_reconnect) {
             mainloop_destroy_trigger(stonith_reconnect);
             stonith_reconnect = NULL;
         }
         if (te_client_id) {
             free(te_client_id);
             te_client_id = NULL;
         }
     }
 }
 
 static gboolean
 do_stonith_history_sync(gpointer user_data)
 {
     if (stonith_api && (stonith_api->state != stonith_disconnected)) {
         stonith_history_t *history = NULL;
 
         te_cleanup_stonith_history_sync(stonith_api, FALSE);
         stonith_api->cmds->history(stonith_api,
                                    st_opt_sync_call | st_opt_broadcast,
                                    NULL, &history, 5);
         stonith_history_free(history);
         return TRUE;
     } else {
         crm_info("Skip triggering stonith history-sync as stonith is disconnected");
         return FALSE;
     }
 }
 
 static void
 tengine_stonith_callback(stonith_t *stonith, stonith_callback_data_t *data)
 {
     char *uuid = NULL;
     int stonith_id = -1;
     int transition_id = -1;
     pcmk__graph_action_t *action = NULL;
     const char *target = NULL;
 
     if ((data == NULL) || (data->userdata == NULL)) {
         crm_err("Ignoring fence operation %d result: "
                 "No transition key given (bug?)",
                 ((data == NULL)? -1 : data->call_id));
         return;
     }
 
     if (!AM_I_DC) {
         const char *reason = stonith__exit_reason(data);
 
         if (reason == NULL) {
            reason = pcmk_exec_status_str(stonith__execution_status(data));
         }
         crm_notice("Result of fence operation %d: %d (%s) " CRM_XS " key=%s",
                    data->call_id, stonith__exit_status(data), reason,
                    (const char *) data->userdata);
         return;
     }
 
     CRM_CHECK(decode_transition_key(data->userdata, &uuid, &transition_id,
                                     &stonith_id, NULL),
               goto bail);
 
     if (controld_globals.transition_graph->complete || (stonith_id < 0)
         || !pcmk__str_eq(uuid, controld_globals.te_uuid, pcmk__str_none)
         || (controld_globals.transition_graph->id != transition_id)) {
         crm_info("Ignoring fence operation %d result: "
                  "Not from current transition " CRM_XS
                  " complete=%s action=%d uuid=%s (vs %s) transition=%d (vs %d)",
                  data->call_id,
                  pcmk__btoa(controld_globals.transition_graph->complete),
                  stonith_id, uuid, controld_globals.te_uuid, transition_id,
                  controld_globals.transition_graph->id);
         goto bail;
     }
 
     action = controld_get_action(stonith_id);
     if (action == NULL) {
         crm_err("Ignoring fence operation %d result: "
                 "Action %d not found in transition graph (bug?) "
                 CRM_XS " uuid=%s transition=%d",
                 data->call_id, stonith_id, uuid, transition_id);
         goto bail;
     }
 
     target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     if (target == NULL) {
         crm_err("Ignoring fence operation %d result: No target given (bug?)",
                 data->call_id);
         goto bail;
     }
 
     stop_te_timer(action);
     if (stonith__exit_status(data) == CRM_EX_OK) {
         const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
         const char *op = crm_meta_value(action->params, "stonith_action");
 
         crm_info("Fence operation %d for %s succeeded", data->call_id, target);
         if (!(pcmk_is_set(action->flags, pcmk__graph_action_confirmed))) {
             te_action_confirmed(action, NULL);
             if (pcmk__str_eq("on", op, pcmk__str_casei)) {
                 const char *value = NULL;
                 char *now = pcmk__ttoa(time(NULL));
                 gboolean is_remote_node = FALSE;
 
                 /* This check is not 100% reliable, since this node is not
                  * guaranteed to have the remote node cached. However, it
                  * doesn't have to be reliable, since the attribute manager can
                  * learn a node's "remoteness" by other means sooner or later.
                  * This allows it to learn more quickly if this node does have
                  * the information.
                  */
                 if (g_hash_table_lookup(crm_remote_peer_cache, uuid) != NULL) {
                     is_remote_node = TRUE;
                 }
 
                 update_attrd(target, CRM_ATTR_UNFENCED, now, NULL,
                              is_remote_node);
                 free(now);
 
                 value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_ALL);
                 update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL,
                              is_remote_node);
 
                 value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_SECURE);
                 update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL,
                              is_remote_node);
 
             } else if (!(pcmk_is_set(action->flags, pcmk__graph_action_sent_update))) {
                 send_stonith_update(action, target, uuid);
                 pcmk__set_graph_action_flags(action,
                                              pcmk__graph_action_sent_update);
             }
         }
         st_fail_count_reset(target);
 
     } else {
         enum pcmk__graph_next abort_action = pcmk__graph_restart;
         int status = stonith__execution_status(data);
         const char *reason = stonith__exit_reason(data);
 
         if (reason == NULL) {
             if (status == PCMK_EXEC_DONE) {
                 reason = "Agent returned error";
             } else {
                 reason = pcmk_exec_status_str(status);
             }
         }
         pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
 
         /* If no fence devices were available, there's no use in immediately
          * checking again, so don't start a new transition in that case.
          */
         if (status == PCMK_EXEC_NO_FENCE_DEVICE) {
             crm_warn("Fence operation %d for %s failed: %s "
                      "(aborting transition and giving up for now)",
                      data->call_id, target, reason);
             abort_action = pcmk__graph_wait;
         } else {
             crm_notice("Fence operation %d for %s failed: %s "
                        "(aborting transition)", data->call_id, target, reason);
         }
 
         /* Increment the fail count now, so abort_for_stonith_failure() can
          * check it. Non-DC nodes will increment it in
          * handle_fence_notification().
          */
         st_fail_count_increment(target);
         abort_for_stonith_failure(abort_action, target, NULL);
     }
 
     pcmk__update_graph(controld_globals.transition_graph, action);
     trigger_graph();
 
   bail:
     free(data->userdata);
     free(uuid);
     return;
 }
 
 static int
-fence_with_delay(const char *target, const char *type, const char *delay)
+fence_with_delay(const char *target, const char *type, int delay)
 {
     uint32_t options = st_opt_none; // Group of enum stonith_call_options
     int timeout_sec = (int) (controld_globals.transition_graph->stonith_timeout
                              / 1000);
-    int delay_i;
 
     if (crmd_join_phase_count(crm_join_confirmed) == 1) {
         stonith__set_call_options(options, target, st_opt_allow_suicide);
     }
-    pcmk__scan_min_int(delay, &delay_i, 0);
     return stonith_api->cmds->fence_with_delay(stonith_api, options, target,
-                                               type, timeout_sec, 0, delay_i);
+                                               type, timeout_sec, 0, delay);
 }
 
 /*!
  * \internal
  * \brief Execute a fencing action from a transition graph
  *
  * \param[in] graph   Transition graph being executed (ignored)
  * \param[in] action  Fencing action to execute
  *
  * \return Standard Pacemaker return code
  */
 int
 controld_execute_fence_action(pcmk__graph_t *graph,
                               pcmk__graph_action_t *action)
 {
     int rc = 0;
     const char *id = ID(action->xml);
     const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
     const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *type = crm_meta_value(action->params, "stonith_action");
     char *transition_key = NULL;
     const char *priority_delay = NULL;
+    int delay_i = 0;
     gboolean invalid_action = FALSE;
-    guint stonith_timeout = controld_globals.transition_graph->stonith_timeout;
+    int stonith_timeout = (int) (controld_globals.transition_graph->stonith_timeout
+                                 / 1000);
 
     CRM_CHECK(id != NULL, invalid_action = TRUE);
     CRM_CHECK(uuid != NULL, invalid_action = TRUE);
     CRM_CHECK(type != NULL, invalid_action = TRUE);
     CRM_CHECK(target != NULL, invalid_action = TRUE);
 
     if (invalid_action) {
         crm_log_xml_warn(action->xml, "BadAction");
         return EPROTO;
     }
 
     priority_delay = crm_meta_value(action->params, XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY);
 
-    crm_notice("Requesting fencing (%s) of node %s "
-               CRM_XS " action=%s timeout=%u%s%s",
+    crm_notice("Requesting fencing (%s) targeting node %s "
+               CRM_XS " action=%s timeout=%i%s%s",
                type, target, id, stonith_timeout,
                priority_delay ? " priority_delay=" : "",
                priority_delay ? priority_delay : "");
 
     /* Passing NULL means block until we can connect... */
     te_connect_stonith(NULL);
 
-    rc = fence_with_delay(target, type, priority_delay);
+    pcmk__scan_min_int(priority_delay, &delay_i, 0);
+    rc = fence_with_delay(target, type, delay_i);
     transition_key = pcmk__transition_key(controld_globals.transition_graph->id,
                                           action->id, 0,
                                           controld_globals.te_uuid),
     stonith_api->cmds->register_callback(stonith_api, rc,
-                                         (int) (stonith_timeout / 1000),
+                                         (stonith_timeout
+                                          + (delay_i > 0 ? delay_i : 0)),
                                          st_opt_timeout_updates, transition_key,
                                          "tengine_stonith_callback",
                                          tengine_stonith_callback);
     return pcmk_rc_ok;
 }
 
 bool
 controld_verify_stonith_watchdog_timeout(const char *value)
 {
     const char *our_nodename = controld_globals.our_nodename;
     gboolean rv = TRUE;
 
     if (stonith_api && (stonith_api->state != stonith_disconnected) &&
         stonith__watchdog_fencing_enabled_for_node_api(stonith_api,
                                                        our_nodename)) {
         rv = pcmk__valid_sbd_timeout(value);
     }
     return rv;
 }
 
 /* end stonith API client functions */
 
 
 /*
  * stonith history synchronization
  *
  * Each node's fencer keeps track of a cluster-wide fencing history. When a node
  * joins or leaves, we need to synchronize the history across all nodes.
  */
 
 static crm_trigger_t *stonith_history_sync_trigger = NULL;
 static mainloop_timer_t *stonith_history_sync_timer_short = NULL;
 static mainloop_timer_t *stonith_history_sync_timer_long = NULL;
 
 void
 te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers)
 {
     if (free_timers) {
         mainloop_timer_del(stonith_history_sync_timer_short);
         stonith_history_sync_timer_short = NULL;
         mainloop_timer_del(stonith_history_sync_timer_long);
         stonith_history_sync_timer_long = NULL;
     } else {
         mainloop_timer_stop(stonith_history_sync_timer_short);
         mainloop_timer_stop(stonith_history_sync_timer_long);
     }
 
     if (st) {
         st->cmds->remove_notification(st, T_STONITH_NOTIFY_HISTORY_SYNCED);
     }
 }
 
 static void
 tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event)
 {
     te_cleanup_stonith_history_sync(st, FALSE);
     crm_debug("Fence-history synced - cancel all timers");
 }
 
 static gboolean
 stonith_history_sync_set_trigger(gpointer user_data)
 {
     mainloop_set_trigger(stonith_history_sync_trigger);
     return FALSE;
 }
 
 void
 te_trigger_stonith_history_sync(bool long_timeout)
 {
     /* trigger a sync in 5s to give more nodes the
      * chance to show up so that we don't create
      * unnecessary stonith-history-sync traffic
      *
      * the long timeout of 30s is there as a fallback
      * so that after a successful connection to fenced
      * we will wait for 30s for the DC to trigger a
      * history-sync
      * if this doesn't happen we trigger a sync locally
      * (e.g. fenced segfaults and is restarted by pacemakerd)
      */
 
     /* as we are finally checking the stonith-connection
      * in do_stonith_history_sync we should be fine
      * leaving stonith_history_sync_time & stonith_history_sync_trigger
      * around
      */
     if (stonith_history_sync_trigger == NULL) {
         stonith_history_sync_trigger =
             mainloop_add_trigger(G_PRIORITY_LOW,
                                  do_stonith_history_sync, NULL);
     }
 
     if (long_timeout) {
         if(stonith_history_sync_timer_long == NULL) {
             stonith_history_sync_timer_long =
                 mainloop_timer_add("history_sync_long", 30000,
                                    FALSE, stonith_history_sync_set_trigger,
                                    NULL);
         }
         crm_info("Fence history will be synchronized cluster-wide within 30 seconds");
         mainloop_timer_start(stonith_history_sync_timer_long);
     } else {
         if(stonith_history_sync_timer_short == NULL) {
             stonith_history_sync_timer_short =
                 mainloop_timer_add("history_sync_short", 5000,
                                    FALSE, stonith_history_sync_set_trigger,
                                    NULL);
         }
         crm_info("Fence history will be synchronized cluster-wide within 5 seconds");
         mainloop_timer_start(stonith_history_sync_timer_short);
     }
 
 }
 
 /* end stonith history synchronization functions */
diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c
index 82644bcddf..ba63cf890e 100644
--- a/daemons/fenced/fenced_commands.c
+++ b/daemons/fenced/fenced_commands.c
@@ -1,3674 +1,3674 @@
 /*
  * Copyright 2009-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/stat.h>
 #include <unistd.h>
 #include <sys/utsname.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <ctype.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/ipc_internal.h>
 #include <crm/cluster/internal.h>
 #include <crm/common/mainloop.h>
 
 #include <crm/stonith-ng.h>
 #include <crm/fencing/internal.h>
 #include <crm/common/xml.h>
 
 #include <pacemaker-fenced.h>
 
 GHashTable *device_list = NULL;
 GHashTable *topology = NULL;
 static GList *cmd_list = NULL;
 
 static GHashTable *fenced_handlers = NULL;
 
 struct device_search_s {
     /* target of fence action */
     char *host;
     /* requested fence action */
     char *action;
     /* timeout to use if a device is queried dynamically for possible targets */
     int per_device_timeout;
     /* number of registered fencing devices at time of request */
     int replies_needed;
     /* number of device replies received so far */
     int replies_received;
     /* whether the target is eligible to perform requested action (or off) */
     bool allow_suicide;
 
     /* private data to pass to search callback function */
     void *user_data;
     /* function to call when all replies have been received */
     void (*callback) (GList * devices, void *user_data);
     /* devices capable of performing requested action (or off if remapping) */
     GList *capable;
     /* Whether to perform searches that support the action */
     uint32_t support_action_only;
 };
 
 static gboolean stonith_device_dispatch(gpointer user_data);
 static void st_child_done(int pid, const pcmk__action_result_t *result,
                           void *user_data);
 static void stonith_send_reply(xmlNode * reply, int call_options, const char *remote_peer,
                                pcmk__client_t *client);
 
 static void search_devices_record_result(struct device_search_s *search, const char *device,
                                          gboolean can_fence);
 
 static int get_agent_metadata(const char *agent, xmlNode **metadata);
 static void read_action_metadata(stonith_device_t *device);
 static enum fenced_target_by unpack_level_kind(const xmlNode *level);
 
 typedef struct async_command_s {
 
     int id;
     int pid;
     int fd_stdout;
     int options;
     int default_timeout; /* seconds */
     int timeout; /* seconds */
 
     int start_delay; // seconds (-1 means disable static/random fencing delays)
     int delay_id;
 
     char *op;
     char *origin;
     char *client;
     char *client_name;
     char *remote_op_id;
 
     char *target;
     uint32_t target_nodeid;
     char *action;
     char *device;
 
     GList *device_list;
     GList *next_device_iter; // device_list entry for next device to execute
 
     void *internal_user_data;
     void (*done_cb) (int pid, const pcmk__action_result_t *result,
                      void *user_data);
     guint timer_sigterm;
     guint timer_sigkill;
     /*! If the operation timed out, this is the last signal
      *  we sent to the process to get it to terminate */
     int last_timeout_signo;
 
     stonith_device_t *active_on;
     stonith_device_t *activating_on;
 } async_command_t;
 
 static xmlNode *construct_async_reply(const async_command_t *cmd,
                                       const pcmk__action_result_t *result);
 
 static gboolean
 is_action_required(const char *action, const stonith_device_t *device)
 {
     return (device != NULL) && device->automatic_unfencing
            && pcmk__str_eq(action, "on", pcmk__str_none);
 }
 
 static int
 get_action_delay_max(const stonith_device_t *device, const char *action)
 {
     const char *value = NULL;
     int delay_max = 0;
 
     if (!pcmk__is_fencing_action(action)) {
         return 0;
     }
 
     value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_MAX);
     if (value) {
        delay_max = crm_parse_interval_spec(value) / 1000;
     }
 
     return delay_max;
 }
 
 static int
 get_action_delay_base(const stonith_device_t *device, const char *action,
                       const char *target)
 {
     char *hash_value = NULL;
     int delay_base = 0;
 
     if (!pcmk__is_fencing_action(action)) {
         return 0;
     }
 
     hash_value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_BASE);
 
     if (hash_value) {
         char *value = strdup(hash_value);
         char *valptr = value;
 
         CRM_ASSERT(value != NULL);
 
         if (target != NULL) {
             for (char *val = strtok(value, "; \t"); val != NULL; val = strtok(NULL, "; \t")) {
                 char *mapval = strchr(val, ':');
 
                 if (mapval == NULL || mapval[1] == 0) {
                     crm_err("pcmk_delay_base: empty value in mapping", val);
                     continue;
                 }
 
                 if (mapval != val && strncasecmp(target, val, (size_t)(mapval - val)) == 0) {
                     value = mapval + 1;
                     crm_debug("pcmk_delay_base mapped to %s for %s",
                               value, target);
                     break;
                 }
             }
         }
 
         if (strchr(value, ':') == 0) {
            delay_base = crm_parse_interval_spec(value) / 1000;
         }
 
         free(valptr);
     }
 
     return delay_base;
 }
 
 /*!
  * \internal
  * \brief Override STONITH timeout with pcmk_*_timeout if available
  *
  * \param[in] device           STONITH device to use
  * \param[in] action           STONITH action name
  * \param[in] default_timeout  Timeout to use if device does not have
  *                             a pcmk_*_timeout parameter for action
  *
  * \return Value of pcmk_(action)_timeout if available, otherwise default_timeout
  * \note For consistency, it would be nice if reboot/off/on timeouts could be
  *       set the same way as start/stop/monitor timeouts, i.e. with an
  *       <operation> entry in the fencing resource configuration. However that
  *       is insufficient because fencing devices may be registered directly via
  *       the fencer's register_device() API instead of going through the CIB
  *       (e.g. stonith_admin uses it for its -R option, and the executor uses it
  *       to ensure a device is registered when a command is issued). As device
  *       properties, pcmk_*_timeout parameters can be grabbed by the fencer when
  *       the device is registered, whether by CIB change or API call.
  */
 static int
 get_action_timeout(const stonith_device_t *device, const char *action,
                    int default_timeout)
 {
     if (action && device && device->params) {
         char buffer[64] = { 0, };
         const char *value = NULL;
 
         /* If "reboot" was requested but the device does not support it,
          * we will remap to "off", so check timeout for "off" instead
          */
         if (pcmk__str_eq(action, "reboot", pcmk__str_none)
             && !pcmk_is_set(device->flags, st_device_supports_reboot)) {
             crm_trace("%s doesn't support reboot, using timeout for off instead",
                       device->id);
             action = "off";
         }
 
         /* If the device config specified an action-specific timeout, use it */
         snprintf(buffer, sizeof(buffer), "pcmk_%s_timeout", action);
         value = g_hash_table_lookup(device->params, buffer);
         if (value) {
             return atoi(value);
         }
     }
     return default_timeout;
 }
 
 /*!
  * \internal
  * \brief Get the currently executing device for a fencing operation
  *
  * \param[in] cmd  Fencing operation to check
  *
  * \return Currently executing device for \p cmd if any, otherwise NULL
  */
 static stonith_device_t *
 cmd_device(const async_command_t *cmd)
 {
     if ((cmd == NULL) || (cmd->device == NULL) || (device_list == NULL)) {
         return NULL;
     }
     return g_hash_table_lookup(device_list, cmd->device);
 }
 
 /*!
  * \internal
  * \brief Return the configured reboot action for a given device
  *
  * \param[in] device_id  Device ID
  *
  * \return Configured reboot action for \p device_id
  */
 const char *
 fenced_device_reboot_action(const char *device_id)
 {
     const char *action = NULL;
 
     if ((device_list != NULL) && (device_id != NULL)) {
         stonith_device_t *device = g_hash_table_lookup(device_list, device_id);
 
         if ((device != NULL) && (device->params != NULL)) {
             action = g_hash_table_lookup(device->params, "pcmk_reboot_action");
         }
     }
     return pcmk__s(action, "reboot");
 }
 
 /*!
  * \internal
  * \brief Check whether a given device supports the "on" action
  *
  * \param[in] device_id  Device ID
  *
  * \return true if \p device_id supports "on", otherwise false
  */
 bool
 fenced_device_supports_on(const char *device_id)
 {
     if ((device_list != NULL) && (device_id != NULL)) {
         stonith_device_t *device = g_hash_table_lookup(device_list, device_id);
 
         if (device != NULL) {
             return pcmk_is_set(device->flags, st_device_supports_on);
         }
     }
     return false;
 }
 
 static void
 free_async_command(async_command_t * cmd)
 {
     if (!cmd) {
         return;
     }
 
     if (cmd->delay_id) {
         g_source_remove(cmd->delay_id);
     }
 
     cmd_list = g_list_remove(cmd_list, cmd);
 
     g_list_free_full(cmd->device_list, free);
     free(cmd->device);
     free(cmd->action);
     free(cmd->target);
     free(cmd->remote_op_id);
     free(cmd->client);
     free(cmd->client_name);
     free(cmd->origin);
     free(cmd->op);
     free(cmd);
 }
 
 /*!
  * \internal
  * \brief Create a new asynchronous fencing operation from request XML
  *
  * \param[in] msg  Fencing request XML (from IPC or CPG)
  *
  * \return Newly allocated fencing operation on success, otherwise NULL
  *
  * \note This asserts on memory errors, so a NULL return indicates an
  *       unparseable message.
  */
 static async_command_t *
 create_async_command(xmlNode *msg)
 {
     xmlNode *op = NULL;
     async_command_t *cmd = NULL;
 
     if (msg == NULL) {
         return NULL;
     }
 
     op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR);
     if (op == NULL) {
         return NULL;
     }
 
     cmd = calloc(1, sizeof(async_command_t));
     CRM_ASSERT(cmd != NULL);
 
     // All messages must include these
     cmd->action = crm_element_value_copy(op, F_STONITH_ACTION);
     cmd->op = crm_element_value_copy(msg, F_STONITH_OPERATION);
     cmd->client = crm_element_value_copy(msg, F_STONITH_CLIENTID);
     if ((cmd->action == NULL) || (cmd->op == NULL) || (cmd->client == NULL)) {
         free_async_command(cmd);
         return NULL;
     }
 
     crm_element_value_int(msg, F_STONITH_CALLID, &(cmd->id));
     crm_element_value_int(msg, F_STONITH_CALLOPTS, &(cmd->options));
     crm_element_value_int(msg, F_STONITH_DELAY, &(cmd->start_delay));
     crm_element_value_int(msg, F_STONITH_TIMEOUT, &(cmd->default_timeout));
     cmd->timeout = cmd->default_timeout;
 
     cmd->origin = crm_element_value_copy(msg, F_ORIG);
     cmd->remote_op_id = crm_element_value_copy(msg, F_STONITH_REMOTE_OP_ID);
     cmd->client_name = crm_element_value_copy(msg, F_STONITH_CLIENTNAME);
     cmd->target = crm_element_value_copy(op, F_STONITH_TARGET);
     cmd->device = crm_element_value_copy(op, F_STONITH_DEVICE);
 
     cmd->done_cb = st_child_done;
 
     // Track in global command list
     cmd_list = g_list_append(cmd_list, cmd);
 
     return cmd;
 }
 
 static int
 get_action_limit(stonith_device_t * device)
 {
     const char *value = NULL;
     int action_limit = 1;
 
     value = g_hash_table_lookup(device->params, PCMK_STONITH_ACTION_LIMIT);
     if ((value == NULL)
         || (pcmk__scan_min_int(value, &action_limit, INT_MIN) != pcmk_rc_ok)
         || (action_limit == 0)) {
         action_limit = 1;
     }
     return action_limit;
 }
 
 static int
 get_active_cmds(stonith_device_t * device)
 {
     int counter = 0;
     GList *gIter = NULL;
     GList *gIterNext = NULL;
 
     CRM_CHECK(device != NULL, return 0);
 
     for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) {
         async_command_t *cmd = gIter->data;
 
         gIterNext = gIter->next;
 
         if (cmd->active_on == device) {
             counter++;
         }
     }
 
     return counter;
 }
 
 static void
 fork_cb(int pid, void *user_data)
 {
     async_command_t *cmd = (async_command_t *) user_data;
     stonith_device_t * device =
         /* in case of a retry we've done the move from
            activating_on to active_on already
          */
         cmd->activating_on?cmd->activating_on:cmd->active_on;
 
     CRM_ASSERT(device);
     crm_debug("Operation '%s' [%d]%s%s using %s now running with %ds timeout",
               cmd->action, pid,
               ((cmd->target == NULL)? "" : " targeting "),
               pcmk__s(cmd->target, ""), device->id, cmd->timeout);
     cmd->active_on = device;
     cmd->activating_on = NULL;
 }
 
 static int
 get_agent_metadata_cb(gpointer data) {
     stonith_device_t *device = data;
     guint period_ms;
 
     switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
         case pcmk_rc_ok:
             if (device->agent_metadata) {
                 read_action_metadata(device);
                 stonith__device_parameter_flags(&(device->flags), device->id,
                                         device->agent_metadata);
             }
             return G_SOURCE_REMOVE;
 
         case EAGAIN:
             period_ms = pcmk__mainloop_timer_get_period(device->timer);
             if (period_ms < 160 * 1000) {
                 mainloop_timer_set_period(device->timer, 2 * period_ms);
             }
             return G_SOURCE_CONTINUE;
 
         default:
             return G_SOURCE_REMOVE;
     }
 }
 
 /*!
  * \internal
  * \brief Call a command's action callback for an internal (not library) result
  *
  * \param[in,out] cmd               Command to report result for
  * \param[in]     execution_status  Execution status to use for result
  * \param[in]     exit_status       Exit status to use for result
  * \param[in]     exit_reason       Exit reason to use for result
  */
 static void
 report_internal_result(async_command_t *cmd, int exit_status,
                        int execution_status, const char *exit_reason)
 {
     pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
 
     pcmk__set_result(&result, exit_status, execution_status, exit_reason);
     cmd->done_cb(0, &result, cmd);
     pcmk__reset_result(&result);
 }
 
 static gboolean
 stonith_device_execute(stonith_device_t * device)
 {
     int exec_rc = 0;
     const char *action_str = NULL;
     const char *host_arg = NULL;
     async_command_t *cmd = NULL;
     stonith_action_t *action = NULL;
     int active_cmds = 0;
     int action_limit = 0;
     GList *gIter = NULL;
     GList *gIterNext = NULL;
 
     CRM_CHECK(device != NULL, return FALSE);
 
     active_cmds = get_active_cmds(device);
     action_limit = get_action_limit(device);
     if (action_limit > -1 && active_cmds >= action_limit) {
         crm_trace("%s is over its action limit of %d (%u active action%s)",
                   device->id, action_limit, active_cmds,
                   pcmk__plural_s(active_cmds));
         return TRUE;
     }
 
     for (gIter = device->pending_ops; gIter != NULL; gIter = gIterNext) {
         async_command_t *pending_op = gIter->data;
 
         gIterNext = gIter->next;
 
         if (pending_op && pending_op->delay_id) {
             crm_trace("Operation '%s'%s%s using %s was asked to run too early, "
                       "waiting for start delay of %ds",
                       pending_op->action,
                       ((pending_op->target == NULL)? "" : " targeting "),
                       pcmk__s(pending_op->target, ""),
                       device->id, pending_op->start_delay);
             continue;
         }
 
         device->pending_ops = g_list_remove_link(device->pending_ops, gIter);
         g_list_free_1(gIter);
 
         cmd = pending_op;
         break;
     }
 
     if (cmd == NULL) {
         crm_trace("No actions using %s are needed", device->id);
         return TRUE;
     }
 
     if (pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
                          STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
         if (pcmk__is_fencing_action(cmd->action)) {
             if (node_does_watchdog_fencing(stonith_our_uname)) {
                 pcmk__panic(__func__);
                 goto done;
             }
         } else {
             crm_info("Faking success for %s watchdog operation", cmd->action);
             report_internal_result(cmd, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
             goto done;
         }
     }
 
 #if SUPPORT_CIBSECRETS
     exec_rc = pcmk__substitute_secrets(device->id, device->params);
     if (exec_rc != pcmk_rc_ok) {
         if (pcmk__str_eq(cmd->action, "stop", pcmk__str_none)) {
             crm_info("Proceeding with stop operation for %s "
                      "despite being unable to load CIB secrets (%s)",
                      device->id, pcmk_rc_str(exec_rc));
         } else {
             crm_err("Considering %s unconfigured "
                     "because unable to load CIB secrets: %s",
                      device->id, pcmk_rc_str(exec_rc));
             report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_SECRETS,
                                    "Failed to get CIB secrets");
             goto done;
         }
     }
 #endif
 
     action_str = cmd->action;
     if (pcmk__str_eq(cmd->action, "reboot", pcmk__str_none)
         && !pcmk_is_set(device->flags, st_device_supports_reboot)) {
 
         crm_notice("Remapping 'reboot' action%s%s using %s to 'off' "
                    "because agent '%s' does not support reboot",
                    ((cmd->target == NULL)? "" : " targeting "),
                    pcmk__s(cmd->target, ""), device->id, device->agent);
         action_str = "off";
     }
 
     if (pcmk_is_set(device->flags, st_device_supports_parameter_port)) {
         host_arg = "port";
 
     } else if (pcmk_is_set(device->flags, st_device_supports_parameter_plug)) {
         host_arg = "plug";
     }
 
     action = stonith__action_create(device->agent, action_str, cmd->target,
                                     cmd->target_nodeid, cmd->timeout,
                                     device->params, device->aliases, host_arg);
 
     /* for async exec, exec_rc is negative for early error exit
        otherwise handling of success/errors is done via callbacks */
     cmd->activating_on = device;
     exec_rc = stonith__execute_async(action, (void *)cmd, cmd->done_cb,
                                      fork_cb);
     if (exec_rc < 0) {
         cmd->activating_on = NULL;
         cmd->done_cb(0, stonith__action_result(action), cmd);
         stonith__destroy_action(action);
     }
 
 done:
     /* Device might get triggered to work by multiple fencing commands
      * simultaneously. Trigger the device again to make sure any
      * remaining concurrent commands get executed. */
     if (device->pending_ops) {
         mainloop_set_trigger(device->work);
     }
     return TRUE;
 }
 
 static gboolean
 stonith_device_dispatch(gpointer user_data)
 {
     return stonith_device_execute(user_data);
 }
 
 static gboolean
 start_delay_helper(gpointer data)
 {
     async_command_t *cmd = data;
     stonith_device_t *device = cmd_device(cmd);
 
     cmd->delay_id = 0;
     if (device) {
         mainloop_set_trigger(device->work);
     }
 
     return FALSE;
 }
 
 static void
 schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
 {
     int delay_max = 0;
     int delay_base = 0;
     int requested_delay = cmd->start_delay;
 
     CRM_CHECK(cmd != NULL, return);
     CRM_CHECK(device != NULL, return);
 
     if (cmd->device) {
         free(cmd->device);
     }
 
     if (device->include_nodeid && (cmd->target != NULL)) {
         crm_node_t *node = crm_get_peer(0, cmd->target);
 
         cmd->target_nodeid = node->id;
     }
 
     cmd->device = strdup(device->id);
     cmd->timeout = get_action_timeout(device, cmd->action, cmd->default_timeout);
 
     if (cmd->remote_op_id) {
         crm_debug("Scheduling '%s' action%s%s using %s for remote peer %s "
                   "with op id %.8s and timeout %ds",
                   cmd->action,
                   (cmd->target == NULL)? "" : " targeting ",
                   pcmk__s(cmd->target, ""),
                   device->id, cmd->origin, cmd->remote_op_id, cmd->timeout);
     } else {
         crm_debug("Scheduling '%s' action%s%s using %s for %s with timeout %ds",
                   cmd->action,
                   (cmd->target == NULL)? "" : " targeting ",
                   pcmk__s(cmd->target, ""),
                   device->id, cmd->client, cmd->timeout);
     }
 
     device->pending_ops = g_list_append(device->pending_ops, cmd);
     mainloop_set_trigger(device->work);
 
     // Value -1 means disable any static/random fencing delays
     if (requested_delay < 0) {
         return;
     }
 
     delay_max = get_action_delay_max(device, cmd->action);
     delay_base = get_action_delay_base(device, cmd->action, cmd->target);
     if (delay_max == 0) {
         delay_max = delay_base;
     }
     if (delay_max < delay_base) {
         crm_warn(PCMK_STONITH_DELAY_BASE " (%ds) is larger than "
                  PCMK_STONITH_DELAY_MAX " (%ds) for %s using %s "
                  "(limiting to maximum delay)",
                  delay_base, delay_max, cmd->action, device->id);
         delay_base = delay_max;
     }
     if (delay_max > 0) {
         // coverity[dont_call] We're not using rand() for security
         cmd->start_delay +=
             ((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0)
             + delay_base;
     }
 
     if (cmd->start_delay > 0) {
         crm_notice("Delaying '%s' action%s%s using %s for %ds " CRM_XS
                    " timeout=%ds requested_delay=%ds base=%ds max=%ds",
                    cmd->action,
                    (cmd->target == NULL)? "" : " targeting ",
                    pcmk__s(cmd->target, ""),
                    device->id, cmd->start_delay, cmd->timeout,
                    requested_delay, delay_base, delay_max);
         cmd->delay_id =
             g_timeout_add_seconds(cmd->start_delay, start_delay_helper, cmd);
     }
 }
 
 static void
 free_device(gpointer data)
 {
     GList *gIter = NULL;
     stonith_device_t *device = data;
 
     g_hash_table_destroy(device->params);
     g_hash_table_destroy(device->aliases);
 
     for (gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) {
         async_command_t *cmd = gIter->data;
 
         crm_warn("Removal of device '%s' purged operation '%s'", device->id, cmd->action);
         report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
                                "Device was removed before action could be executed");
     }
     g_list_free(device->pending_ops);
 
     g_list_free_full(device->targets, free);
 
     if (device->timer) {
         mainloop_timer_stop(device->timer);
         mainloop_timer_del(device->timer);
     }
 
     mainloop_destroy_trigger(device->work);
 
     free_xml(device->agent_metadata);
     free(device->namespace);
     if (device->on_target_actions != NULL) {
         g_string_free(device->on_target_actions, TRUE);
     }
     free(device->agent);
     free(device->id);
     free(device);
 }
 
 void free_device_list(void)
 {
     if (device_list != NULL) {
         g_hash_table_destroy(device_list);
         device_list = NULL;
     }
 }
 
 void
 init_device_list(void)
 {
     if (device_list == NULL) {
         device_list = pcmk__strkey_table(NULL, free_device);
     }
 }
 
 static GHashTable *
 build_port_aliases(const char *hostmap, GList ** targets)
 {
     char *name = NULL;
     int last = 0, lpc = 0, max = 0, added = 0;
     GHashTable *aliases = pcmk__strikey_table(free, free);
 
     if (hostmap == NULL) {
         return aliases;
     }
 
     max = strlen(hostmap);
     for (; lpc <= max; lpc++) {
         switch (hostmap[lpc]) {
                 /* Skip escaped chars */
             case '\\':
                 lpc++;
                 break;
 
                 /* Assignment chars */
             case '=':
             case ':':
                 if (lpc > last) {
                     free(name);
                     name = calloc(1, 1 + lpc - last);
                     memcpy(name, hostmap + last, lpc - last);
                 }
                 last = lpc + 1;
                 break;
 
                 /* Delimeter chars */
                 /* case ',': Potentially used to specify multiple ports */
             case 0:
             case ';':
             case ' ':
             case '\t':
                 if (name) {
                     char *value = NULL;
                     int k = 0;
 
                     value = calloc(1, 1 + lpc - last);
                     memcpy(value, hostmap + last, lpc - last);
 
                     for (int i = 0; value[i] != '\0'; i++) {
                         if (value[i] != '\\') {
                             value[k++] = value[i];
                         }
                     }
                     value[k] = '\0';
 
                     crm_debug("Adding alias '%s'='%s'", name, value);
                     g_hash_table_replace(aliases, name, value);
                     if (targets) {
                         *targets = g_list_append(*targets, strdup(value));
                     }
                     value = NULL;
                     name = NULL;
                     added++;
 
                 } else if (lpc > last) {
                     crm_debug("Parse error at offset %d near '%s'", lpc - last, hostmap + last);
                 }
 
                 last = lpc + 1;
                 break;
         }
 
         if (hostmap[lpc] == 0) {
             break;
         }
     }
 
     if (added == 0) {
         crm_info("No host mappings detected in '%s'", hostmap);
     }
 
     free(name);
     return aliases;
 }
 
 GHashTable *metadata_cache = NULL;
 
 void
 free_metadata_cache(void) {
     if (metadata_cache != NULL) {
         g_hash_table_destroy(metadata_cache);
         metadata_cache = NULL;
     }
 }
 
 static void
 init_metadata_cache(void) {
     if (metadata_cache == NULL) {
         metadata_cache = pcmk__strkey_table(free, free);
     }
 }
 
 int
 get_agent_metadata(const char *agent, xmlNode ** metadata)
 {
     char *buffer = NULL;
 
     if (metadata == NULL) {
         return EINVAL;
     }
     *metadata = NULL;
     if (pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT_INTERNAL, pcmk__str_none)) {
         return pcmk_rc_ok;
     }
     init_metadata_cache();
     buffer = g_hash_table_lookup(metadata_cache, agent);
     if (buffer == NULL) {
         stonith_t *st = stonith_api_new();
         int rc;
 
         if (st == NULL) {
             crm_warn("Could not get agent meta-data: "
                      "API memory allocation failed");
             return EAGAIN;
         }
         rc = st->cmds->metadata(st, st_opt_sync_call, agent,
                                 NULL, &buffer, 10);
         stonith_api_delete(st);
         if (rc || !buffer) {
             crm_err("Could not retrieve metadata for fencing agent %s", agent);
             return EAGAIN;
         }
         g_hash_table_replace(metadata_cache, strdup(agent), buffer);
     }
 
     *metadata = string2xml(buffer);
     return pcmk_rc_ok;
 }
 
 static gboolean
 is_nodeid_required(xmlNode * xml)
 {
     xmlXPathObjectPtr xpath = NULL;
 
     if (stand_alone) {
         return FALSE;
     }
 
     if (!xml) {
         return FALSE;
     }
 
     xpath = xpath_search(xml, "//parameter[@name='nodeid']");
     if (numXpathResults(xpath)  <= 0) {
         freeXpathObject(xpath);
         return FALSE;
     }
 
     freeXpathObject(xpath);
     return TRUE;
 }
 
 static void
 read_action_metadata(stonith_device_t *device)
 {
     xmlXPathObjectPtr xpath = NULL;
     int max = 0;
     int lpc = 0;
 
     if (device->agent_metadata == NULL) {
         return;
     }
 
     xpath = xpath_search(device->agent_metadata, "//action");
     max = numXpathResults(xpath);
 
     if (max <= 0) {
         freeXpathObject(xpath);
         return;
     }
 
     for (lpc = 0; lpc < max; lpc++) {
         const char *action = NULL;
         xmlNode *match = getXpathResult(xpath, lpc);
 
         CRM_LOG_ASSERT(match != NULL);
         if(match == NULL) { continue; };
 
         action = crm_element_value(match, "name");
 
         if (pcmk__str_eq(action, "list", pcmk__str_none)) {
             stonith__set_device_flags(device->flags, device->id,
                                       st_device_supports_list);
         } else if (pcmk__str_eq(action, "status", pcmk__str_none)) {
             stonith__set_device_flags(device->flags, device->id,
                                       st_device_supports_status);
         } else if (pcmk__str_eq(action, "reboot", pcmk__str_none)) {
             stonith__set_device_flags(device->flags, device->id,
                                       st_device_supports_reboot);
         } else if (pcmk__str_eq(action, "on", pcmk__str_none)) {
             /* "automatic" means the cluster will unfence node when it joins */
             /* "required" is a deprecated synonym for "automatic" */
             if (pcmk__xe_attr_is_true(match, "automatic") || pcmk__xe_attr_is_true(match, "required")) {
                 device->automatic_unfencing = TRUE;
             }
             stonith__set_device_flags(device->flags, device->id,
                                       st_device_supports_on);
         }
 
         if ((action != NULL) && pcmk__xe_attr_is_true(match, "on_target")) {
             pcmk__add_word(&(device->on_target_actions), 64, action);
         }
     }
 
     freeXpathObject(xpath);
 }
 
 /*!
  * \internal
  * \brief Set a pcmk_*_action parameter if not already set
  *
  * \param[in,out] params  Device parameters
  * \param[in]     action  Name of action
  * \param[in]     value   Value to use if action is not already set
  */
 static void
 map_action(GHashTable *params, const char *action, const char *value)
 {
     char *key = crm_strdup_printf("pcmk_%s_action", action);
 
     if (g_hash_table_lookup(params, key)) {
         crm_warn("Ignoring %s='%s', see %s instead",
                  STONITH_ATTR_ACTION_OP, value, key);
         free(key);
     } else {
         crm_warn("Mapping %s='%s' to %s='%s'",
                  STONITH_ATTR_ACTION_OP, value, key, value);
         g_hash_table_insert(params, key, strdup(value));
     }
 }
 
 /*!
  * \internal
  * \brief Create device parameter table from XML
  *
  * \param[in]  name    Device name (used for logging only)
  * \param[in]  dev     XML containing device parameters
  */
 static GHashTable *
 xml2device_params(const char *name, const xmlNode *dev)
 {
     GHashTable *params = xml2list(dev);
     const char *value;
 
     /* Action should never be specified in the device configuration,
      * but we support it for users who are familiar with other software
      * that worked that way.
      */
     value = g_hash_table_lookup(params, STONITH_ATTR_ACTION_OP);
     if (value != NULL) {
         crm_warn("%s has '%s' parameter, which should never be specified in configuration",
                  name, STONITH_ATTR_ACTION_OP);
 
         if (*value == '\0') {
             crm_warn("Ignoring empty '%s' parameter", STONITH_ATTR_ACTION_OP);
 
         } else if (strcmp(value, "reboot") == 0) {
             crm_warn("Ignoring %s='reboot' (see stonith-action cluster property instead)",
                      STONITH_ATTR_ACTION_OP);
 
         } else if (strcmp(value, "off") == 0) {
             map_action(params, "reboot", value);
 
         } else {
             map_action(params, "off", value);
             map_action(params, "reboot", value);
         }
 
         g_hash_table_remove(params, STONITH_ATTR_ACTION_OP);
     }
 
     return params;
 }
 
 static const char *
 target_list_type(stonith_device_t * dev)
 {
     const char *check_type = NULL;
 
     check_type = g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK);
 
     if (check_type == NULL) {
 
         if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_LIST)) {
             check_type = "static-list";
         } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)) {
             check_type = "static-list";
         } else if (pcmk_is_set(dev->flags, st_device_supports_list)) {
             check_type = "dynamic-list";
         } else if (pcmk_is_set(dev->flags, st_device_supports_status)) {
             check_type = "status";
         } else {
             check_type = PCMK__VALUE_NONE;
         }
     }
 
     return check_type;
 }
 
 static stonith_device_t *
 build_device_from_xml(xmlNode *dev)
 {
     const char *value;
     stonith_device_t *device = NULL;
     char *agent = crm_element_value_copy(dev, "agent");
 
     CRM_CHECK(agent != NULL, return device);
 
     device = calloc(1, sizeof(stonith_device_t));
 
     CRM_CHECK(device != NULL, {free(agent); return device;});
 
     device->id = crm_element_value_copy(dev, XML_ATTR_ID);
     device->agent = agent;
     device->namespace = crm_element_value_copy(dev, "namespace");
     device->params = xml2device_params(device->id, dev);
 
     value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_LIST);
     if (value) {
         device->targets = stonith__parse_targets(value);
     }
 
     value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_MAP);
     device->aliases = build_port_aliases(value, &(device->targets));
 
     value = target_list_type(device);
     if (!pcmk__str_eq(value, "static-list", pcmk__str_casei) && device->targets) {
         /* Other than "static-list", dev-> targets is unnecessary. */
         g_list_free_full(device->targets, free);
         device->targets = NULL;
     }
     switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
         case pcmk_rc_ok:
             if (device->agent_metadata) {
                 read_action_metadata(device);
                 stonith__device_parameter_flags(&(device->flags), device->id,
                                                 device->agent_metadata);
             }
             break;
 
         case EAGAIN:
             if (device->timer == NULL) {
                 device->timer = mainloop_timer_add("get_agent_metadata", 10 * 1000,
                                            TRUE, get_agent_metadata_cb, device);
             }
             if (!mainloop_timer_running(device->timer)) {
                 mainloop_timer_start(device->timer);
             }
             break;
 
         default:
             break;
     }
 
     value = g_hash_table_lookup(device->params, "nodeid");
     if (!value) {
         device->include_nodeid = is_nodeid_required(device->agent_metadata);
     }
 
     value = crm_element_value(dev, "rsc_provides");
     if (pcmk__str_eq(value, PCMK__VALUE_UNFENCING, pcmk__str_casei)) {
         device->automatic_unfencing = TRUE;
     }
 
     if (is_action_required("on", device)) {
         crm_info("Fencing device '%s' requires unfencing", device->id);
     }
 
     if (device->on_target_actions != NULL) {
         crm_info("Fencing device '%s' requires actions (%s) to be executed "
                  "on target", device->id,
                  (const char *) device->on_target_actions->str);
     }
 
     device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device);
     /* TODO: Hook up priority */
 
     return device;
 }
 
 static void
 schedule_internal_command(const char *origin,
                           stonith_device_t * device,
                           const char *action,
                           const char *target,
                           int timeout,
                           void *internal_user_data,
                           void (*done_cb) (int pid,
                                            const pcmk__action_result_t *result,
                                            void *user_data))
 {
     async_command_t *cmd = NULL;
 
     cmd = calloc(1, sizeof(async_command_t));
 
     cmd->id = -1;
     cmd->default_timeout = timeout ? timeout : 60;
     cmd->timeout = cmd->default_timeout;
     cmd->action = strdup(action);
     pcmk__str_update(&cmd->target, target);
     cmd->device = strdup(device->id);
     cmd->origin = strdup(origin);
     cmd->client = strdup(crm_system_name);
     cmd->client_name = strdup(crm_system_name);
 
     cmd->internal_user_data = internal_user_data;
     cmd->done_cb = done_cb; /* cmd, not internal_user_data, is passed to 'done_cb' as the userdata */
 
     schedule_stonith_command(cmd, device);
 }
 
 // Fence agent status commands use custom exit status codes
 enum fence_status_code {
     fence_status_invalid    = -1,
     fence_status_active     = 0,
     fence_status_unknown    = 1,
     fence_status_inactive   = 2,
 };
 
 static void
 status_search_cb(int pid, const pcmk__action_result_t *result, void *user_data)
 {
     async_command_t *cmd = user_data;
     struct device_search_s *search = cmd->internal_user_data;
     stonith_device_t *dev = cmd_device(cmd);
     gboolean can = FALSE;
 
     free_async_command(cmd);
 
     if (!dev) {
         search_devices_record_result(search, NULL, FALSE);
         return;
     }
 
     mainloop_set_trigger(dev->work);
 
     if (result->execution_status != PCMK_EXEC_DONE) {
         crm_warn("Assuming %s cannot fence %s "
                  "because status could not be executed: %s%s%s%s",
                  dev->id, search->host,
                  pcmk_exec_status_str(result->execution_status),
                  ((result->exit_reason == NULL)? "" : " ("),
                  ((result->exit_reason == NULL)? "" : result->exit_reason),
                  ((result->exit_reason == NULL)? "" : ")"));
         search_devices_record_result(search, dev->id, FALSE);
         return;
     }
 
     switch (result->exit_status) {
         case fence_status_unknown:
             crm_trace("%s reported it cannot fence %s", dev->id, search->host);
             break;
 
         case fence_status_active:
         case fence_status_inactive:
             crm_trace("%s reported it can fence %s", dev->id, search->host);
             can = TRUE;
             break;
 
         default:
             crm_warn("Assuming %s cannot fence %s "
                      "(status returned unknown code %d)",
                      dev->id, search->host, result->exit_status);
             break;
     }
     search_devices_record_result(search, dev->id, can);
 }
 
 static void
 dynamic_list_search_cb(int pid, const pcmk__action_result_t *result,
                        void *user_data)
 {
     async_command_t *cmd = user_data;
     struct device_search_s *search = cmd->internal_user_data;
     stonith_device_t *dev = cmd_device(cmd);
     gboolean can_fence = FALSE;
 
     free_async_command(cmd);
 
     /* Host/alias must be in the list output to be eligible to be fenced
      *
      * Will cause problems if down'd nodes aren't listed or (for virtual nodes)
      *  if the guest is still listed despite being moved to another machine
      */
     if (!dev) {
         search_devices_record_result(search, NULL, FALSE);
         return;
     }
 
     mainloop_set_trigger(dev->work);
 
     if (pcmk__result_ok(result)) {
         crm_info("Refreshing target list for %s", dev->id);
         g_list_free_full(dev->targets, free);
         dev->targets = stonith__parse_targets(result->action_stdout);
         dev->targets_age = time(NULL);
 
     } else if (dev->targets != NULL) {
         if (result->execution_status == PCMK_EXEC_DONE) {
             crm_info("Reusing most recent target list for %s "
                      "because list returned error code %d",
                      dev->id, result->exit_status);
         } else {
             crm_info("Reusing most recent target list for %s "
                      "because list could not be executed: %s%s%s%s",
                      dev->id, pcmk_exec_status_str(result->execution_status),
                      ((result->exit_reason == NULL)? "" : " ("),
                      ((result->exit_reason == NULL)? "" : result->exit_reason),
                      ((result->exit_reason == NULL)? "" : ")"));
         }
 
     } else { // We have never successfully executed list
         if (result->execution_status == PCMK_EXEC_DONE) {
             crm_warn("Assuming %s cannot fence %s "
                      "because list returned error code %d",
                      dev->id, search->host, result->exit_status);
         } else {
             crm_warn("Assuming %s cannot fence %s "
                      "because list could not be executed: %s%s%s%s",
                      dev->id, search->host,
                      pcmk_exec_status_str(result->execution_status),
                      ((result->exit_reason == NULL)? "" : " ("),
                      ((result->exit_reason == NULL)? "" : result->exit_reason),
                      ((result->exit_reason == NULL)? "" : ")"));
         }
 
         /* Fall back to pcmk_host_check="status" if the user didn't explicitly
          * specify "dynamic-list".
          */
         if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK) == NULL) {
             crm_notice("Switching to pcmk_host_check='status' for %s", dev->id);
             g_hash_table_replace(dev->params, strdup(PCMK_STONITH_HOST_CHECK),
                                  strdup("status"));
         }
     }
 
     if (dev->targets) {
         const char *alias = g_hash_table_lookup(dev->aliases, search->host);
 
         if (!alias) {
             alias = search->host;
         }
         if (pcmk__str_in_list(alias, dev->targets, pcmk__str_casei)) {
             can_fence = TRUE;
         }
     }
     search_devices_record_result(search, dev->id, can_fence);
 }
 
 /*!
  * \internal
  * \brief Returns true if any key in first is not in second or second has a different value for key
  */
 static int
 device_params_diff(GHashTable *first, GHashTable *second) {
     char *key = NULL;
     char *value = NULL;
     GHashTableIter gIter;
 
     g_hash_table_iter_init(&gIter, first);
     while (g_hash_table_iter_next(&gIter, (void **)&key, (void **)&value)) {
 
         if(strstr(key, "CRM_meta") == key) {
             continue;
         } else if(strcmp(key, "crm_feature_set") == 0) {
             continue;
         } else {
             char *other_value = g_hash_table_lookup(second, key);
 
             if (!other_value || !pcmk__str_eq(other_value, value, pcmk__str_casei)) {
                 crm_trace("Different value for %s: %s != %s", key, other_value, value);
                 return 1;
             }
         }
     }
 
     return 0;
 }
 
 /*!
  * \internal
  * \brief Checks to see if an identical device already exists in the device_list
  */
 static stonith_device_t *
 device_has_duplicate(const stonith_device_t *device)
 {
     stonith_device_t *dup = g_hash_table_lookup(device_list, device->id);
 
     if (!dup) {
         crm_trace("No match for %s", device->id);
         return NULL;
 
     } else if (!pcmk__str_eq(dup->agent, device->agent, pcmk__str_casei)) {
         crm_trace("Different agent: %s != %s", dup->agent, device->agent);
         return NULL;
     }
 
     /* Use calculate_operation_digest() here? */
     if (device_params_diff(device->params, dup->params) ||
         device_params_diff(dup->params, device->params)) {
         return NULL;
     }
 
     crm_trace("Match");
     return dup;
 }
 
 int
 stonith_device_register(xmlNode *dev, gboolean from_cib)
 {
     stonith_device_t *dup = NULL;
     stonith_device_t *device = build_device_from_xml(dev);
     guint ndevices = 0;
     int rv = pcmk_ok;
 
     CRM_CHECK(device != NULL, return -ENOMEM);
 
     /* do we have a watchdog-device? */
     if (pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, pcmk__str_none) ||
         pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
                      STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) do {
         if (stonith_watchdog_timeout_ms <= 0) {
             crm_err("Ignoring watchdog fence device without "
                     "stonith-watchdog-timeout set.");
             rv = -ENODEV;
             /* fall through to cleanup & return */
         } else if (!pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
                                  STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
             crm_err("Ignoring watchdog fence device with unknown "
                     "agent '%s' unequal '" STONITH_WATCHDOG_AGENT "'.",
                     device->agent?device->agent:"");
             rv = -ENODEV;
             /* fall through to cleanup & return */
         } else if (!pcmk__str_eq(device->id, STONITH_WATCHDOG_ID,
                                  pcmk__str_none)) {
             crm_err("Ignoring watchdog fence device "
                     "named %s !='"STONITH_WATCHDOG_ID"'.",
                     device->id?device->id:"");
             rv = -ENODEV;
             /* fall through to cleanup & return */
         } else {
             if (pcmk__str_eq(device->agent, STONITH_WATCHDOG_AGENT,
                              pcmk__str_none)) {
                 /* this either has an empty list or the targets
                    configured for watchdog-fencing
                  */
                 g_list_free_full(stonith_watchdog_targets, free);
                 stonith_watchdog_targets = device->targets;
                 device->targets = NULL;
             }
             if (node_does_watchdog_fencing(stonith_our_uname)) {
                 g_list_free_full(device->targets, free);
                 device->targets = stonith__parse_targets(stonith_our_uname);
                 g_hash_table_replace(device->params,
                                      strdup(PCMK_STONITH_HOST_LIST),
                                      strdup(stonith_our_uname));
                 /* proceed as with any other stonith-device */
                 break;
             }
 
             crm_debug("Skip registration of watchdog fence device on node not in host-list.");
             /* cleanup and fall through to more cleanup and return */
             device->targets = NULL;
             stonith_device_remove(device->id, from_cib);
         }
         free_device(device);
         return rv;
     } while (0);
 
     dup = device_has_duplicate(device);
     if (dup) {
         ndevices = g_hash_table_size(device_list);
         crm_debug("Device '%s' already in device list (%d active device%s)",
                   device->id, ndevices, pcmk__plural_s(ndevices));
         free_device(device);
         device = dup;
         dup = g_hash_table_lookup(device_list, device->id);
         dup->dirty = FALSE;
 
     } else {
         stonith_device_t *old = g_hash_table_lookup(device_list, device->id);
 
         if (from_cib && old && old->api_registered) {
             /* If the cib is writing over an entry that is shared with a stonith client,
              * copy any pending ops that currently exist on the old entry to the new one.
              * Otherwise the pending ops will be reported as failures
              */
             crm_info("Overwriting existing entry for %s from CIB", device->id);
             device->pending_ops = old->pending_ops;
             device->api_registered = TRUE;
             old->pending_ops = NULL;
             if (device->pending_ops) {
                 mainloop_set_trigger(device->work);
             }
         }
         g_hash_table_replace(device_list, device->id, device);
 
         ndevices = g_hash_table_size(device_list);
         crm_notice("Added '%s' to device list (%d active device%s)",
                    device->id, ndevices, pcmk__plural_s(ndevices));
     }
 
     if (from_cib) {
         device->cib_registered = TRUE;
     } else {
         device->api_registered = TRUE;
     }
 
     return pcmk_ok;
 }
 
 void
 stonith_device_remove(const char *id, bool from_cib)
 {
     stonith_device_t *device = g_hash_table_lookup(device_list, id);
     guint ndevices = 0;
 
     if (!device) {
         ndevices = g_hash_table_size(device_list);
         crm_info("Device '%s' not found (%d active device%s)",
                  id, ndevices, pcmk__plural_s(ndevices));
         return;
     }
 
     if (from_cib) {
         device->cib_registered = FALSE;
     } else {
         device->verified = FALSE;
         device->api_registered = FALSE;
     }
 
     if (!device->cib_registered && !device->api_registered) {
         g_hash_table_remove(device_list, id);
         ndevices = g_hash_table_size(device_list);
         crm_info("Removed '%s' from device list (%d active device%s)",
                  id, ndevices, pcmk__plural_s(ndevices));
     } else {
         crm_trace("Not removing '%s' from device list (%d active) because "
                   "still registered via:%s%s",
                   id, g_hash_table_size(device_list),
                   (device->cib_registered? " cib" : ""),
                   (device->api_registered? " api" : ""));
     }
 }
 
 /*!
  * \internal
  * \brief Return the number of stonith levels registered for a node
  *
  * \param[in] tp  Node's topology table entry
  *
  * \return Number of non-NULL levels in topology entry
  * \note This function is used only for log messages.
  */
 static int
 count_active_levels(const stonith_topology_t *tp)
 {
     int lpc = 0;
     int count = 0;
 
     for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
         if (tp->levels[lpc] != NULL) {
             count++;
         }
     }
     return count;
 }
 
 static void
 free_topology_entry(gpointer data)
 {
     stonith_topology_t *tp = data;
 
     int lpc = 0;
 
     for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
         if (tp->levels[lpc] != NULL) {
             g_list_free_full(tp->levels[lpc], free);
         }
     }
     free(tp->target);
     free(tp->target_value);
     free(tp->target_pattern);
     free(tp->target_attribute);
     free(tp);
 }
 
 void
 free_topology_list(void)
 {
     if (topology != NULL) {
         g_hash_table_destroy(topology);
         topology = NULL;
     }
 }
 
 void
 init_topology_list(void)
 {
     if (topology == NULL) {
         topology = pcmk__strkey_table(NULL, free_topology_entry);
     }
 }
 
 char *
 stonith_level_key(const xmlNode *level, enum fenced_target_by mode)
 {
     if (mode == fenced_target_by_unknown) {
         mode = unpack_level_kind(level);
     }
     switch (mode) {
         case fenced_target_by_name:
             return crm_element_value_copy(level, XML_ATTR_STONITH_TARGET);
 
         case fenced_target_by_pattern:
             return crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_PATTERN);
 
         case fenced_target_by_attribute:
             return crm_strdup_printf("%s=%s",
                 crm_element_value(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE),
                 crm_element_value(level, XML_ATTR_STONITH_TARGET_VALUE));
 
         default:
             return crm_strdup_printf("unknown-%s", ID(level));
     }
 }
 
 /*!
  * \internal
  * \brief Parse target identification from topology level XML
  *
  * \param[in] level  Topology level XML to parse
  *
  * \return How to identify target of \p level
  */
 static enum fenced_target_by
 unpack_level_kind(const xmlNode *level)
 {
     if (crm_element_value(level, XML_ATTR_STONITH_TARGET) != NULL) {
         return fenced_target_by_name;
     }
     if (crm_element_value(level, XML_ATTR_STONITH_TARGET_PATTERN) != NULL) {
         return fenced_target_by_pattern;
     }
     if (!stand_alone /* if standalone, there's no attribute manager */
         && (crm_element_value(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE) != NULL)
         && (crm_element_value(level, XML_ATTR_STONITH_TARGET_VALUE) != NULL)) {
         return fenced_target_by_attribute;
     }
     return fenced_target_by_unknown;
 }
 
 static stonith_key_value_t *
 parse_device_list(const char *devices)
 {
     int lpc = 0;
     int max = 0;
     int last = 0;
     stonith_key_value_t *output = NULL;
 
     if (devices == NULL) {
         return output;
     }
 
     max = strlen(devices);
     for (lpc = 0; lpc <= max; lpc++) {
         if (devices[lpc] == ',' || devices[lpc] == 0) {
             char *line = strndup(devices + last, lpc - last);
 
             output = stonith_key_value_add(output, NULL, line);
             free(line);
 
             last = lpc + 1;
         }
     }
 
     return output;
 }
 
 /*!
  * \internal
  * \brief Unpack essential information from topology request XML
  *
  * \param[in]  xml     Request XML to search
  * \param[out] mode    If not NULL, where to store level kind
  * \param[out] target  If not NULL, where to store representation of target
  * \param[out] id      If not NULL, where to store level number
  * \param[out] desc    If not NULL, where to store log-friendly level description
  *
  * \return Topology level XML from within \p xml, or NULL if not found
  * \note The caller is responsible for freeing \p *target and \p *desc if set.
  */
 static xmlNode *
 unpack_level_request(xmlNode *xml, enum fenced_target_by *mode, char **target,
                      int *id, char **desc)
 {
     enum fenced_target_by local_mode = fenced_target_by_unknown;
     char *local_target = NULL;
     int local_id = 0;
 
     /* The level element can be the top element or lower. If top level, don't
      * search by xpath, because it might give multiple hits if the XML is the
      * CIB.
      */
     if ((xml != NULL)
         && !pcmk__str_eq(TYPE(xml), XML_TAG_FENCING_LEVEL, pcmk__str_none)) {
         xml = get_xpath_object("//" XML_TAG_FENCING_LEVEL, xml, LOG_WARNING);
     }
 
     if (xml == NULL) {
         if (desc != NULL) {
             *desc = crm_strdup_printf("missing");
         }
     } else {
         local_mode = unpack_level_kind(xml);
         local_target = stonith_level_key(xml, local_mode);
         crm_element_value_int(xml, XML_ATTR_STONITH_INDEX, &local_id);
         if (desc != NULL) {
             *desc = crm_strdup_printf("%s[%d]", local_target, local_id);
         }
     }
 
     if (mode != NULL) {
         *mode = local_mode;
     }
     if (id != NULL) {
         *id = local_id;
     }
 
     if (target != NULL) {
         *target = local_target;
     } else {
         free(local_target);
     }
 
     return xml;
 }
 
 /*!
  * \internal
  * \brief Register a fencing topology level for a target
  *
  * Given an XML request specifying the target name, level index, and device IDs
  * for the level, this will create an entry for the target in the global topology
  * table if one does not already exist, then append the specified device IDs to
  * the entry's device list for the specified level.
  *
  * \param[in]  msg     XML request for STONITH level registration
  * \param[out] desc    If not NULL, set to string representation "TARGET[LEVEL]"
  * \param[out] result  Where to set result of registration
  */
 void
 fenced_register_level(xmlNode *msg, char **desc, pcmk__action_result_t *result)
 {
     int id = 0;
     xmlNode *level;
     enum fenced_target_by mode;
     char *target;
 
     stonith_topology_t *tp;
     stonith_key_value_t *dIter = NULL;
     stonith_key_value_t *devices = NULL;
 
     CRM_CHECK((msg != NULL) && (result != NULL), return);
 
     level = unpack_level_request(msg, &mode, &target, &id, desc);
     if (level == NULL) {
         fenced_set_protocol_error(result);
         return;
     }
 
     // Ensure an ID was given (even the client API adds an ID)
     if (pcmk__str_empty(ID(level))) {
         crm_warn("Ignoring registration for topology level without ID");
         free(target);
         crm_log_xml_trace(level, "Bad level");
         pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
                             "Topology level is invalid without ID");
         return;
     }
 
     // Ensure a valid target was specified
     if (mode == fenced_target_by_unknown) {
         crm_warn("Ignoring registration for topology level '%s' "
                  "without valid target", ID(level));
         free(target);
         crm_log_xml_trace(level, "Bad level");
         pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
                             "Invalid target for topology level '%s'",
                             ID(level));
         return;
     }
 
     // Ensure level ID is in allowed range
     if ((id <= 0) || (id >= ST_LEVEL_MAX)) {
         crm_warn("Ignoring topology registration for %s with invalid level %d",
                   target, id);
         free(target);
         crm_log_xml_trace(level, "Bad level");
         pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
                             "Invalid level number '%s' for topology level '%s'",
                             pcmk__s(crm_element_value(level,
                                                       XML_ATTR_STONITH_INDEX),
                                     ""),
                             ID(level));
         return;
     }
 
     /* Find or create topology table entry */
     tp = g_hash_table_lookup(topology, target);
     if (tp == NULL) {
         tp = calloc(1, sizeof(stonith_topology_t));
         if (tp == NULL) {
             pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
                              strerror(ENOMEM));
             free(target);
             return;
         }
         tp->kind = mode;
         tp->target = target;
         tp->target_value = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_VALUE);
         tp->target_pattern = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_PATTERN);
         tp->target_attribute = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE);
 
         g_hash_table_replace(topology, tp->target, tp);
         crm_trace("Added %s (%d) to the topology (%d active entries)",
                   target, (int) mode, g_hash_table_size(topology));
     } else {
         free(target);
     }
 
     if (tp->levels[id] != NULL) {
         crm_info("Adding to the existing %s[%d] topology entry",
                  tp->target, id);
     }
 
     devices = parse_device_list(crm_element_value(level, XML_ATTR_STONITH_DEVICES));
     for (dIter = devices; dIter; dIter = dIter->next) {
         const char *device = dIter->value;
 
         crm_trace("Adding device '%s' for %s[%d]", device, tp->target, id);
         tp->levels[id] = g_list_append(tp->levels[id], strdup(device));
     }
     stonith_key_value_freeall(devices, 1, 1);
 
     {
         int nlevels = count_active_levels(tp);
 
         crm_info("Target %s has %d active fencing level%s",
                  tp->target, nlevels, pcmk__plural_s(nlevels));
     }
 
     pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
 }
 
 /*!
  * \internal
  * \brief Unregister a fencing topology level for a target
  *
  * Given an XML request specifying the target name and level index (or 0 for all
  * levels), this will remove any corresponding entry for the target from the
  * global topology table.
  *
  * \param[in]  msg     XML request for STONITH level registration
  * \param[out] desc    If not NULL, set to string representation "TARGET[LEVEL]"
  * \param[out] result  Where to set result of unregistration
  */
 void
 fenced_unregister_level(xmlNode *msg, char **desc,
                         pcmk__action_result_t *result)
 {
     int id = -1;
     stonith_topology_t *tp;
     char *target;
     xmlNode *level = NULL;
 
     CRM_CHECK(result != NULL, return);
 
     level = unpack_level_request(msg, NULL, &target, &id, desc);
     if (level == NULL) {
         fenced_set_protocol_error(result);
         return;
     }
 
     // Ensure level ID is in allowed range
     if ((id < 0) || (id >= ST_LEVEL_MAX)) {
         crm_warn("Ignoring topology unregistration for %s with invalid level %d",
                   target, id);
         free(target);
         crm_log_xml_trace(level, "Bad level");
         pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
                             "Invalid level number '%s' for topology level %s",
                             pcmk__s(crm_element_value(level,
                                                       XML_ATTR_STONITH_INDEX),
                                     "<null>"),
 
                             // Client API doesn't add ID to unregistration XML
                             pcmk__s(ID(level), ""));
         return;
     }
 
     tp = g_hash_table_lookup(topology, target);
     if (tp == NULL) {
         guint nentries = g_hash_table_size(topology);
 
         crm_info("No fencing topology found for %s (%d active %s)",
                  target, nentries,
                  pcmk__plural_alt(nentries, "entry", "entries"));
 
     } else if (id == 0 && g_hash_table_remove(topology, target)) {
         guint nentries = g_hash_table_size(topology);
 
         crm_info("Removed all fencing topology entries related to %s "
                  "(%d active %s remaining)", target, nentries,
                  pcmk__plural_alt(nentries, "entry", "entries"));
 
     } else if (tp->levels[id] != NULL) {
         guint nlevels;
 
         g_list_free_full(tp->levels[id], free);
         tp->levels[id] = NULL;
 
         nlevels = count_active_levels(tp);
         crm_info("Removed level %d from fencing topology for %s "
                  "(%d active level%s remaining)",
                  id, target, nlevels, pcmk__plural_s(nlevels));
     }
 
     free(target);
     pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
 }
 
 static char *
 list_to_string(GList *list, const char *delim, gboolean terminate_with_delim)
 {
     int max = g_list_length(list);
     size_t delim_len = delim?strlen(delim):0;
     size_t alloc_size = 1 + (max?((max-1+(terminate_with_delim?1:0))*delim_len):0);
     char *rv;
     GList *gIter;
 
     for (gIter = list; gIter != NULL; gIter = gIter->next) {
         const char *value = (const char *) gIter->data;
 
         alloc_size += strlen(value);
     }
     rv = calloc(alloc_size, sizeof(char));
     if (rv) {
         char *pos = rv;
         const char *lead_delim = "";
 
         for (gIter = list; gIter != NULL; gIter = gIter->next) {
             const char *value = (const char *) gIter->data;
 
             pos = &pos[sprintf(pos, "%s%s", lead_delim, value)];
             lead_delim = delim;
         }
         if (max && terminate_with_delim) {
             sprintf(pos, "%s", delim);
         }
     }
     return rv;
 }
 
 /*!
  * \internal
  * \brief Execute a fence agent action directly (and asynchronously)
  *
  * Handle a STONITH_OP_EXEC API message by scheduling a requested agent action
  * directly on a specified device. Only list, monitor, and status actions are
  * expected to use this call, though it should work with any agent command.
  *
  * \param[in]  msg     Request XML specifying action
  * \param[out] result  Where to store result of action
  *
  * \note If the action is monitor, the device must be registered via the API
  *       (CIB registration is not sufficient), because monitor should not be
  *       possible unless the device is "started" (API registered).
  */
 static void
 execute_agent_action(xmlNode *msg, pcmk__action_result_t *result)
 {
     xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, msg, LOG_ERR);
     xmlNode *op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR);
     const char *id = crm_element_value(dev, F_STONITH_DEVICE);
     const char *action = crm_element_value(op, F_STONITH_ACTION);
     async_command_t *cmd = NULL;
     stonith_device_t *device = NULL;
 
     if ((id == NULL) || (action == NULL)) {
         crm_info("Malformed API action request: device %s, action %s",
                  (id? id : "not specified"),
                  (action? action : "not specified"));
         fenced_set_protocol_error(result);
         return;
     }
 
     if (pcmk__str_eq(id, STONITH_WATCHDOG_ID, pcmk__str_none)) {
         // Watchdog agent actions are implemented internally
         if (stonith_watchdog_timeout_ms <= 0) {
             pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
                              "Watchdog fence device not configured");
             return;
 
         } else if (pcmk__str_eq(action, "list", pcmk__str_none)) {
             pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
             pcmk__set_result_output(result,
                                     list_to_string(stonith_watchdog_targets,
                                                    "\n", TRUE),
                                     NULL);
             return;
 
         } else if (pcmk__str_eq(action, "monitor", pcmk__str_none)) {
             pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
             return;
         }
     }
 
     device = g_hash_table_lookup(device_list, id);
     if (device == NULL) {
         crm_info("Ignoring API '%s' action request because device %s not found",
                  action, id);
         pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
                             "'%s' not found", id);
         return;
 
     } else if (!device->api_registered && !strcmp(action, "monitor")) {
         // Monitors may run only on "started" (API-registered) devices
         crm_info("Ignoring API '%s' action request because device %s not active",
                  action, id);
         pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
                             "'%s' not active", id);
         return;
     }
 
     cmd = create_async_command(msg);
     if (cmd == NULL) {
         crm_log_xml_warn(msg, "invalid");
         fenced_set_protocol_error(result);
         return;
     }
 
     schedule_stonith_command(cmd, device);
     pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
 }
 
 static void
 search_devices_record_result(struct device_search_s *search, const char *device, gboolean can_fence)
 {
     search->replies_received++;
     if (can_fence && device) {
         if (search->support_action_only != st_device_supports_none) {
             stonith_device_t *dev = g_hash_table_lookup(device_list, device);
             if (dev && !pcmk_is_set(dev->flags, search->support_action_only)) {
                 return;
             }
         }
         search->capable = g_list_append(search->capable, strdup(device));
     }
 
     if (search->replies_needed == search->replies_received) {
 
         guint ndevices = g_list_length(search->capable);
 
         crm_debug("Search found %d device%s that can perform '%s' targeting %s",
                   ndevices, pcmk__plural_s(ndevices),
                   (search->action? search->action : "unknown action"),
                   (search->host? search->host : "any node"));
 
         search->callback(search->capable, search->user_data);
         free(search->host);
         free(search->action);
         free(search);
     }
 }
 
 /*!
  * \internal
  * \brief Check whether the local host is allowed to execute a fencing action
  *
  * \param[in] device         Fence device to check
  * \param[in] action         Fence action to check
  * \param[in] target         Hostname of fence target
  * \param[in] allow_suicide  Whether self-fencing is allowed for this operation
  *
  * \return TRUE if local host is allowed to execute action, FALSE otherwise
  */
 static gboolean
 localhost_is_eligible(const stonith_device_t *device, const char *action,
                       const char *target, gboolean allow_suicide)
 {
     gboolean localhost_is_target = pcmk__str_eq(target, stonith_our_uname,
                                                 pcmk__str_casei);
 
     if ((device != NULL) && (action != NULL)
         && (device->on_target_actions != NULL)
         && (strstr((const char*) device->on_target_actions->str,
                    action) != NULL)) {
 
         if (!localhost_is_target) {
             crm_trace("Operation '%s' using %s can only be executed for local "
                       "host, not %s", action, device->id, target);
             return FALSE;
         }
 
     } else if (localhost_is_target && !allow_suicide) {
         crm_trace("'%s' operation does not support self-fencing", action);
         return FALSE;
     }
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Check if local node is allowed to execute (possibly remapped) action
  *
  * \param[in] device      Fence device to check
  * \param[in] action      Fence action to check
  * \param[in] target      Node name of fence target
  * \param[in] allow_self  Whether self-fencing is allowed for this operation
  *
  * \return true if local node is allowed to execute \p action or any actions it
  *         might be remapped to, otherwise false
  */
 static bool
 localhost_is_eligible_with_remap(const stonith_device_t *device,
                                  const char *action, const char *target,
                                  gboolean allow_self)
 {
     // Check exact action
     if (localhost_is_eligible(device, action, target, allow_self)) {
         return true;
     }
 
     // Check potential remaps
 
     if (pcmk__str_eq(action, "reboot", pcmk__str_none)) {
         /* "reboot" might get remapped to "off" then "on", so even if reboot is
          * disallowed, return true if either of those is allowed. We'll report
          * the disallowed actions with the results. We never allow self-fencing
          * for remapped "on" actions because the target is off at that point.
          */
         if (localhost_is_eligible(device, "off", target, allow_self)
             || localhost_is_eligible(device, "on", target, FALSE)) {
             return true;
         }
     }
 
     return false;
 }
 
 static void
 can_fence_host_with_device(stonith_device_t *dev,
                            struct device_search_s *search)
 {
     gboolean can = FALSE;
     const char *check_type = "Internal bug";
     const char *target = NULL;
     const char *alias = NULL;
     const char *dev_id = "Unspecified device";
     const char *action = (search == NULL)? NULL : search->action;
 
     CRM_CHECK((dev != NULL) && (action != NULL), goto search_report_results);
 
     if (dev->id != NULL) {
         dev_id = dev->id;
     }
 
     target = search->host;
     if (target == NULL) {
         can = TRUE;
         check_type = "No target";
         goto search_report_results;
     }
 
     /* Answer immediately if the device does not support the action
      * or the local node is not allowed to perform it
      */
     if (pcmk__str_eq(action, "on", pcmk__str_none)
         && !pcmk_is_set(dev->flags, st_device_supports_on)) {
         check_type = "Agent does not support 'on'";
         goto search_report_results;
 
     } else if (!localhost_is_eligible_with_remap(dev, action, target,
                                                  search->allow_suicide)) {
         check_type = "This node is not allowed to execute action";
         goto search_report_results;
     }
 
     // Check eligibility as specified by pcmk_host_check
     check_type = target_list_type(dev);
     alias = g_hash_table_lookup(dev->aliases, target);
     if (pcmk__str_eq(check_type, PCMK__VALUE_NONE, pcmk__str_casei)) {
         can = TRUE;
 
     } else if (pcmk__str_eq(check_type, "static-list", pcmk__str_casei)) {
         if (pcmk__str_in_list(target, dev->targets, pcmk__str_casei)) {
             can = TRUE;
         } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)
                    && g_hash_table_lookup(dev->aliases, target)) {
             can = TRUE;
         }
 
     } else if (pcmk__str_eq(check_type, "dynamic-list", pcmk__str_casei)) {
         time_t now = time(NULL);
 
         if (dev->targets == NULL || dev->targets_age + 60 < now) {
             int device_timeout = get_action_timeout(dev, "list", search->per_device_timeout);
 
             if (device_timeout > search->per_device_timeout) {
                 crm_notice("Since the pcmk_list_timeout(%ds) parameter of %s is larger than stonith-timeout(%ds), timeout may occur",
                     device_timeout, dev_id, search->per_device_timeout);
             }
 
             crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
                       check_type, dev_id, target, action);
 
             schedule_internal_command(__func__, dev, "list", NULL,
                                       search->per_device_timeout, search, dynamic_list_search_cb);
 
             /* we'll respond to this search request async in the cb */
             return;
         }
 
         if (pcmk__str_in_list(((alias == NULL)? target : alias), dev->targets,
                               pcmk__str_casei)) {
             can = TRUE;
         }
 
     } else if (pcmk__str_eq(check_type, "status", pcmk__str_casei)) {
         int device_timeout = get_action_timeout(dev, check_type, search->per_device_timeout);
 
         if (device_timeout > search->per_device_timeout) {
             crm_notice("Since the pcmk_status_timeout(%ds) parameter of %s is larger than stonith-timeout(%ds), timeout may occur",
                 device_timeout, dev_id, search->per_device_timeout);
         }
 
         crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
                   check_type, dev_id, target, action);
         schedule_internal_command(__func__, dev, "status", target,
                                   search->per_device_timeout, search, status_search_cb);
         /* we'll respond to this search request async in the cb */
         return;
     } else {
         crm_err("Invalid value for " PCMK_STONITH_HOST_CHECK ": %s", check_type);
         check_type = "Invalid " PCMK_STONITH_HOST_CHECK;
     }
 
   search_report_results:
     crm_info("%s is%s eligible to fence (%s) %s%s%s%s: %s",
              dev_id, (can? "" : " not"), pcmk__s(action, "unspecified action"),
              pcmk__s(target, "unspecified target"),
              (alias == NULL)? "" : " (as '", pcmk__s(alias, ""),
              (alias == NULL)? "" : "')", check_type);
     search_devices_record_result(search, ((dev == NULL)? NULL : dev_id), can);
 }
 
 static void
 search_devices(gpointer key, gpointer value, gpointer user_data)
 {
     stonith_device_t *dev = value;
     struct device_search_s *search = user_data;
 
     can_fence_host_with_device(dev, search);
 }
 
 #define DEFAULT_QUERY_TIMEOUT 20
 static void
 get_capable_devices(const char *host, const char *action, int timeout, bool suicide, void *user_data,
                     void (*callback) (GList * devices, void *user_data), uint32_t support_action_only)
 {
     struct device_search_s *search;
     guint ndevices = g_hash_table_size(device_list);
 
     if (ndevices == 0) {
         callback(NULL, user_data);
         return;
     }
 
     search = calloc(1, sizeof(struct device_search_s));
     if (!search) {
         crm_crit("Cannot search for capable fence devices: %s",
                  strerror(ENOMEM));
         callback(NULL, user_data);
         return;
     }
 
     pcmk__str_update(&search->host, host);
     pcmk__str_update(&search->action, action);
     search->per_device_timeout = timeout;
     search->allow_suicide = suicide;
     search->callback = callback;
     search->user_data = user_data;
     search->support_action_only = support_action_only;
 
     /* We are guaranteed this many replies, even if a device is
      * unregistered while the search is in progress.
      */
     search->replies_needed = ndevices;
 
     crm_debug("Searching %d device%s to see which can execute '%s' targeting %s",
               ndevices, pcmk__plural_s(ndevices),
               (search->action? search->action : "unknown action"),
               (search->host? search->host : "any node"));
     g_hash_table_foreach(device_list, search_devices, search);
 }
 
 struct st_query_data {
     xmlNode *reply;
     char *remote_peer;
     char *client_id;
     char *target;
     char *action;
     int call_options;
 };
 
 /*!
  * \internal
  * \brief Add action-specific attributes to query reply XML
  *
  * \param[in,out] xml     XML to add attributes to
  * \param[in]     action  Fence action
  * \param[in]     device  Fence device
  * \param[in]     target  Fence target
  */
 static void
 add_action_specific_attributes(xmlNode *xml, const char *action,
                                const stonith_device_t *device,
                                const char *target)
 {
     int action_specific_timeout;
     int delay_max;
     int delay_base;
 
     CRM_CHECK(xml && action && device, return);
 
     if (is_action_required(action, device)) {
         crm_trace("Action '%s' is required using %s", action, device->id);
         crm_xml_add_int(xml, F_STONITH_DEVICE_REQUIRED, 1);
     }
 
     action_specific_timeout = get_action_timeout(device, action, 0);
     if (action_specific_timeout) {
         crm_trace("Action '%s' has timeout %dms using %s",
                   action, action_specific_timeout, device->id);
         crm_xml_add_int(xml, F_STONITH_ACTION_TIMEOUT, action_specific_timeout);
     }
 
     delay_max = get_action_delay_max(device, action);
     if (delay_max > 0) {
-        crm_trace("Action '%s' has maximum random delay %dms using %s",
+        crm_trace("Action '%s' has maximum random delay %ds using %s",
                   action, delay_max, device->id);
-        crm_xml_add_int(xml, F_STONITH_DELAY_MAX, delay_max / 1000);
+        crm_xml_add_int(xml, F_STONITH_DELAY_MAX, delay_max);
     }
 
     delay_base = get_action_delay_base(device, action, target);
     if (delay_base > 0) {
-        crm_xml_add_int(xml, F_STONITH_DELAY_BASE, delay_base / 1000);
+        crm_xml_add_int(xml, F_STONITH_DELAY_BASE, delay_base);
     }
 
     if ((delay_max > 0) && (delay_base == 0)) {
-        crm_trace("Action '%s' has maximum random delay %dms using %s",
+        crm_trace("Action '%s' has maximum random delay %ds using %s",
                   action, delay_max, device->id);
     } else if ((delay_max == 0) && (delay_base > 0)) {
-        crm_trace("Action '%s' has a static delay of %dms using %s",
+        crm_trace("Action '%s' has a static delay of %ds using %s",
                   action, delay_base, device->id);
     } else if ((delay_max > 0) && (delay_base > 0)) {
-        crm_trace("Action '%s' has a minimum delay of %dms and a randomly chosen "
-                  "maximum delay of %dms using %s",
+        crm_trace("Action '%s' has a minimum delay of %ds and a randomly chosen "
+                  "maximum delay of %ds using %s",
                   action, delay_base, delay_max, device->id);
     }
 }
 
 /*!
  * \internal
  * \brief Add "disallowed" attribute to query reply XML if appropriate
  *
  * \param[in,out] xml            XML to add attribute to
  * \param[in]     action         Fence action
  * \param[in]     device         Fence device
  * \param[in]     target         Fence target
  * \param[in]     allow_suicide  Whether self-fencing is allowed
  */
 static void
 add_disallowed(xmlNode *xml, const char *action, const stonith_device_t *device,
                const char *target, gboolean allow_suicide)
 {
     if (!localhost_is_eligible(device, action, target, allow_suicide)) {
         crm_trace("Action '%s' using %s is disallowed for local host",
                   action, device->id);
         pcmk__xe_set_bool_attr(xml, F_STONITH_ACTION_DISALLOWED, true);
     }
 }
 
 /*!
  * \internal
  * \brief Add child element with action-specific values to query reply XML
  *
  * \param[in,out] xml            XML to add attribute to
  * \param[in]     action         Fence action
  * \param[in]     device         Fence device
  * \param[in]     target         Fence target
  * \param[in]     allow_suicide  Whether self-fencing is allowed
  */
 static void
 add_action_reply(xmlNode *xml, const char *action,
                  const stonith_device_t *device, const char *target,
                  gboolean allow_suicide)
 {
     xmlNode *child = create_xml_node(xml, F_STONITH_ACTION);
 
     crm_xml_add(child, XML_ATTR_ID, action);
     add_action_specific_attributes(child, action, device, target);
     add_disallowed(child, action, device, target, allow_suicide);
 }
 
 static void
 stonith_query_capable_device_cb(GList * devices, void *user_data)
 {
     struct st_query_data *query = user_data;
     int available_devices = 0;
     xmlNode *dev = NULL;
     xmlNode *list = NULL;
     GList *lpc = NULL;
     pcmk__client_t *client = NULL;
 
     if (query->client_id != NULL) {
         client = pcmk__find_client_by_id(query->client_id);
         if ((client == NULL) && (query->remote_peer == NULL)) {
             crm_trace("Skipping reply to %s: no longer a client",
                       query->client_id);
             goto done;
         }
     }
 
     /* Pack the results into XML */
     list = create_xml_node(NULL, __func__);
     crm_xml_add(list, F_STONITH_TARGET, query->target);
     for (lpc = devices; lpc != NULL; lpc = lpc->next) {
         stonith_device_t *device = g_hash_table_lookup(device_list, lpc->data);
         const char *action = query->action;
 
         if (!device) {
             /* It is possible the device got unregistered while
              * determining who can fence the target */
             continue;
         }
 
         available_devices++;
 
         dev = create_xml_node(list, F_STONITH_DEVICE);
         crm_xml_add(dev, XML_ATTR_ID, device->id);
         crm_xml_add(dev, "namespace", device->namespace);
         crm_xml_add(dev, "agent", device->agent);
         crm_xml_add_int(dev, F_STONITH_DEVICE_VERIFIED, device->verified);
         crm_xml_add_int(dev, F_STONITH_DEVICE_SUPPORT_FLAGS, device->flags);
 
         /* If the originating fencer wants to reboot the node, and we have a
          * capable device that doesn't support "reboot", remap to "off" instead.
          */
         if (!pcmk_is_set(device->flags, st_device_supports_reboot)
             && pcmk__str_eq(query->action, "reboot", pcmk__str_none)) {
             crm_trace("%s doesn't support reboot, using values for off instead",
                       device->id);
             action = "off";
         }
 
         /* Add action-specific values if available */
         add_action_specific_attributes(dev, action, device, query->target);
         if (pcmk__str_eq(query->action, "reboot", pcmk__str_none)) {
             /* A "reboot" *might* get remapped to "off" then "on", so after
              * sending the "reboot"-specific values in the main element, we add
              * sub-elements for "off" and "on" values.
              *
              * We short-circuited earlier if "reboot", "off" and "on" are all
              * disallowed for the local host. However if only one or two are
              * disallowed, we send back the results and mark which ones are
              * disallowed. If "reboot" is disallowed, this might cause problems
              * with older fencer versions, which won't check for it. Older
              * versions will ignore "off" and "on", so they are not a problem.
              */
             add_disallowed(dev, action, device, query->target,
                            pcmk_is_set(query->call_options, st_opt_allow_suicide));
             add_action_reply(dev, "off", device, query->target,
                              pcmk_is_set(query->call_options, st_opt_allow_suicide));
             add_action_reply(dev, "on", device, query->target, FALSE);
         }
 
         /* A query without a target wants device parameters */
         if (query->target == NULL) {
             xmlNode *attrs = create_xml_node(dev, XML_TAG_ATTRS);
 
             g_hash_table_foreach(device->params, hash2field, attrs);
         }
     }
 
     crm_xml_add_int(list, F_STONITH_AVAILABLE_DEVICES, available_devices);
     if (query->target) {
         crm_debug("Found %d matching device%s for target '%s'",
                   available_devices, pcmk__plural_s(available_devices),
                   query->target);
     } else {
         crm_debug("%d device%s installed",
                   available_devices, pcmk__plural_s(available_devices));
     }
 
     if (list != NULL) {
         crm_log_xml_trace(list, "Add query results");
         add_message_xml(query->reply, F_STONITH_CALLDATA, list);
     }
 
     stonith_send_reply(query->reply, query->call_options, query->remote_peer,
                        client);
 
 done:
     free_xml(query->reply);
     free(query->remote_peer);
     free(query->client_id);
     free(query->target);
     free(query->action);
     free(query);
     free_xml(list);
     g_list_free_full(devices, free);
 }
 
 /*!
  * \internal
  * \brief Log the result of an asynchronous command
  *
  * \param[in] cmd        Command the result is for
  * \param[in] result     Result of command
  * \param[in] pid        Process ID of command, if available
  * \param[in] next       Alternate device that will be tried if command failed
  * \param[in] op_merged  Whether this command was merged with an earlier one
  */
 static void
 log_async_result(const async_command_t *cmd,
                  const pcmk__action_result_t *result,
                  int pid, const char *next, bool op_merged)
 {
     int log_level = LOG_ERR;
     int output_log_level = LOG_NEVER;
     guint devices_remaining = g_list_length(cmd->next_device_iter);
 
     GString *msg = g_string_sized_new(80); // Reasonable starting size
 
     // Choose log levels appropriately if we have a result
     if (pcmk__result_ok(result)) {
         log_level = (cmd->target == NULL)? LOG_DEBUG : LOG_NOTICE;
         if ((result->action_stdout != NULL)
             && !pcmk__str_eq(cmd->action, "metadata", pcmk__str_none)) {
             output_log_level = LOG_DEBUG;
         }
         next = NULL;
     } else {
         log_level = (cmd->target == NULL)? LOG_NOTICE : LOG_ERR;
         if ((result->action_stdout != NULL)
             && !pcmk__str_eq(cmd->action, "metadata", pcmk__str_none)) {
             output_log_level = LOG_WARNING;
         }
     }
 
     // Build the log message piece by piece
     pcmk__g_strcat(msg, "Operation '", cmd->action, "' ", NULL);
     if (pid != 0) {
         g_string_append_printf(msg, "[%d] ", pid);
     }
     if (cmd->target != NULL) {
         pcmk__g_strcat(msg, "targeting ", cmd->target, " ", NULL);
     }
     if (cmd->device != NULL) {
         pcmk__g_strcat(msg, "using ", cmd->device, " ", NULL);
     }
 
     // Add exit status or execution status as appropriate
     if (result->execution_status == PCMK_EXEC_DONE) {
         g_string_append_printf(msg, "returned %d", result->exit_status);
     } else {
         pcmk__g_strcat(msg, "could not be executed: ",
                        pcmk_exec_status_str(result->execution_status), NULL);
     }
 
     // Add exit reason and next device if appropriate
     if (result->exit_reason != NULL) {
         pcmk__g_strcat(msg, " (", result->exit_reason, ")", NULL);
     }
     if (next != NULL) {
         pcmk__g_strcat(msg, ", retrying with ", next, NULL);
     }
     if (devices_remaining > 0) {
         g_string_append_printf(msg, " (%u device%s remaining)",
                                (unsigned int) devices_remaining,
                                pcmk__plural_s(devices_remaining));
     }
     g_string_append_printf(msg, " " CRM_XS " %scall %d from %s",
                            (op_merged? "merged " : ""), cmd->id,
                            cmd->client_name);
 
     // Log the result
     do_crm_log(log_level, "%s", msg->str);
     g_string_free(msg, TRUE);
 
     // Log the output (which may have multiple lines), if appropriate
     if (output_log_level != LOG_NEVER) {
         char *prefix = crm_strdup_printf("%s[%d]", cmd->device, pid);
 
         crm_log_output(output_log_level, prefix, result->action_stdout);
         free(prefix);
     }
 }
 
 /*!
  * \internal
  * \brief Reply to requester after asynchronous command completion
  *
  * \param[in] cmd      Command that completed
  * \param[in] result   Result of command
  * \param[in] pid      Process ID of command, if available
  * \param[in] merged   If true, command was merged with another, not executed
  */
 static void
 send_async_reply(const async_command_t *cmd, const pcmk__action_result_t *result,
                  int pid, bool merged)
 {
     xmlNode *reply = NULL;
     pcmk__client_t *client = NULL;
 
     CRM_CHECK((cmd != NULL) && (result != NULL), return);
 
     log_async_result(cmd, result, pid, NULL, merged);
 
     if (cmd->client != NULL) {
         client = pcmk__find_client_by_id(cmd->client);
         if ((client == NULL) && (cmd->origin == NULL)) {
             crm_trace("Skipping reply to %s: no longer a client", cmd->client);
             return;
         }
     }
 
     reply = construct_async_reply(cmd, result);
     if (merged) {
         pcmk__xe_set_bool_attr(reply, F_STONITH_MERGED, true);
     }
 
     if (!stand_alone && pcmk__is_fencing_action(cmd->action)
         && pcmk__str_eq(cmd->origin, cmd->target, pcmk__str_casei)) {
         /* The target was also the originator, so broadcast the result on its
          * behalf (since it will be unable to).
          */
         crm_trace("Broadcast '%s' result for %s (target was also originator)",
                   cmd->action, cmd->target);
         crm_xml_add(reply, F_SUBTYPE, "broadcast");
         crm_xml_add(reply, F_STONITH_OPERATION, T_STONITH_NOTIFY);
         send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE);
     } else {
         // Reply only to the originator
         stonith_send_reply(reply, cmd->options, cmd->origin, client);
     }
 
     crm_log_xml_trace(reply, "Reply");
     free_xml(reply);
 
     if (stand_alone) {
         /* Do notification with a clean data object */
         xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE);
 
         stonith__xe_set_result(notify_data, result);
         crm_xml_add(notify_data, F_STONITH_TARGET, cmd->target);
         crm_xml_add(notify_data, F_STONITH_OPERATION, cmd->op);
         crm_xml_add(notify_data, F_STONITH_DELEGATE, "localhost");
         crm_xml_add(notify_data, F_STONITH_DEVICE, cmd->device);
         crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id);
         crm_xml_add(notify_data, F_STONITH_ORIGIN, cmd->client);
 
         fenced_send_notification(T_STONITH_NOTIFY_FENCE, result, notify_data);
         fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
     }
 }
 
 static void
 cancel_stonith_command(async_command_t * cmd)
 {
     stonith_device_t *device = cmd_device(cmd);
 
     if (device) {
         crm_trace("Cancel scheduled '%s' action using %s",
                   cmd->action, device->id);
         device->pending_ops = g_list_remove(device->pending_ops, cmd);
     }
 }
 
 /*!
  * \internal
  * \brief Cancel and reply to any duplicates of a just-completed operation
  *
  * Check whether any fencing operations are scheduled to do the same thing as
  * one that just succeeded. If so, rather than performing the same operation
  * twice, return the result of this operation for all matching pending commands.
  *
  * \param[in,out] cmd     Fencing operation that just succeeded
  * \param[in]     result  Result of \p cmd
  * \param[in]     pid     If nonzero, process ID of agent invocation (for logs)
  *
  * \note Duplicate merging will do the right thing for either type of remapped
  *       reboot. If the executing fencer remapped an unsupported reboot to off,
  *       then cmd->action will be "reboot" and will be merged with any other
  *       reboot requests. If the originating fencer remapped a topology reboot
  *       to off then on, we will get here once with cmd->action "off" and once
  *       with "on", and they will be merged separately with similar requests.
  */
 static void
 reply_to_duplicates(async_command_t *cmd, const pcmk__action_result_t *result,
                     int pid)
 {
     GList *next = NULL;
 
     for (GList *iter = cmd_list; iter != NULL; iter = next) {
         async_command_t *cmd_other = iter->data;
 
         next = iter->next; // We might delete this entry, so grab next now
 
         if (cmd == cmd_other) {
             continue;
         }
 
         /* A pending operation matches if:
          * 1. The client connections are different.
          * 2. The target is the same.
          * 3. The fencing action is the same.
          * 4. The device scheduled to execute the action is the same.
          */
         if (pcmk__str_eq(cmd->client, cmd_other->client, pcmk__str_casei) ||
             !pcmk__str_eq(cmd->target, cmd_other->target, pcmk__str_casei) ||
             !pcmk__str_eq(cmd->action, cmd_other->action, pcmk__str_none) ||
             !pcmk__str_eq(cmd->device, cmd_other->device, pcmk__str_casei)) {
 
             continue;
         }
 
         crm_notice("Merging fencing action '%s'%s%s originating from "
                    "client %s with identical fencing request from client %s",
                    cmd_other->action,
                    (cmd_other->target == NULL)? "" : " targeting ",
                    pcmk__s(cmd_other->target, ""), cmd_other->client_name,
                    cmd->client_name);
 
         // Stop tracking the duplicate, send its result, and cancel it
         cmd_list = g_list_remove_link(cmd_list, iter);
         send_async_reply(cmd_other, result, pid, true);
         cancel_stonith_command(cmd_other);
 
         free_async_command(cmd_other);
         g_list_free_1(iter);
     }
 }
 
 /*!
  * \internal
  * \brief Return the next required device (if any) for an operation
  *
  * \param[in,out] cmd  Fencing operation that just succeeded
  *
  * \return Next device required for action if any, otherwise NULL
  */
 static stonith_device_t *
 next_required_device(async_command_t *cmd)
 {
     for (GList *iter = cmd->next_device_iter; iter != NULL; iter = iter->next) {
         stonith_device_t *next_device = g_hash_table_lookup(device_list,
                                                             iter->data);
 
         if (is_action_required(cmd->action, next_device)) {
             /* This is only called for successful actions, so it's OK to skip
              * non-required devices.
              */
             cmd->next_device_iter = iter->next;
             return next_device;
         }
     }
     return NULL;
 }
 
 static void
 st_child_done(int pid, const pcmk__action_result_t *result, void *user_data)
 {
     async_command_t *cmd = user_data;
 
     stonith_device_t *device = NULL;
     stonith_device_t *next_device = NULL;
 
     CRM_CHECK(cmd != NULL, return);
 
     device = cmd_device(cmd);
     cmd->active_on = NULL;
 
     /* The device is ready to do something else now */
     if (device) {
         if (!device->verified && pcmk__result_ok(result) &&
             (pcmk__strcase_any_of(cmd->action, "list", "monitor", "status", NULL))) {
 
             device->verified = TRUE;
         }
 
         mainloop_set_trigger(device->work);
     }
 
     if (pcmk__result_ok(result)) {
         next_device = next_required_device(cmd);
 
     } else if ((cmd->next_device_iter != NULL)
                && !is_action_required(cmd->action, device)) {
         /* if this device didn't work out, see if there are any others we can try.
          * if the failed device was 'required', we can't pick another device. */
         next_device = g_hash_table_lookup(device_list,
                                           cmd->next_device_iter->data);
         cmd->next_device_iter = cmd->next_device_iter->next;
     }
 
     if (next_device == NULL) {
         send_async_reply(cmd, result, pid, false);
         if (pcmk__result_ok(result)) {
             reply_to_duplicates(cmd, result, pid);
         }
         free_async_command(cmd);
 
     } else { // This operation requires more fencing
         log_async_result(cmd, result, pid, next_device->id, false);
         schedule_stonith_command(cmd, next_device);
     }
 }
 
 static gint
 sort_device_priority(gconstpointer a, gconstpointer b)
 {
     const stonith_device_t *dev_a = a;
     const stonith_device_t *dev_b = b;
 
     if (dev_a->priority > dev_b->priority) {
         return -1;
     } else if (dev_a->priority < dev_b->priority) {
         return 1;
     }
     return 0;
 }
 
 static void
 stonith_fence_get_devices_cb(GList * devices, void *user_data)
 {
     async_command_t *cmd = user_data;
     stonith_device_t *device = NULL;
     guint ndevices = g_list_length(devices);
 
     crm_info("Found %d matching device%s for target '%s'",
              ndevices, pcmk__plural_s(ndevices), cmd->target);
 
     if (devices != NULL) {
         /* Order based on priority */
         devices = g_list_sort(devices, sort_device_priority);
         device = g_hash_table_lookup(device_list, devices->data);
     }
 
     if (device == NULL) { // No device found
         pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
 
         pcmk__format_result(&result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
                             "No device configured for target '%s'",
                             cmd->target);
         send_async_reply(cmd, &result, 0, false);
         pcmk__reset_result(&result);
         free_async_command(cmd);
         g_list_free_full(devices, free);
 
     } else { // Device found, schedule it for fencing
         cmd->device_list = devices;
         cmd->next_device_iter = devices->next;
         schedule_stonith_command(cmd, device);
     }
 }
 
 /*!
  * \internal
  * \brief Execute a fence action via the local node
  *
  * \param[in]  msg     Fencing request
  * \param[out] result  Where to store result of fence action
  */
 static void
 fence_locally(xmlNode *msg, pcmk__action_result_t *result)
 {
     const char *device_id = NULL;
     stonith_device_t *device = NULL;
     async_command_t *cmd = NULL;
     xmlNode *dev = NULL;
 
     CRM_CHECK((msg != NULL) && (result != NULL), return);
 
     dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_ERR);
 
     cmd = create_async_command(msg);
     if (cmd == NULL) {
         crm_log_xml_warn(msg, "invalid");
         fenced_set_protocol_error(result);
         return;
     }
 
     device_id = crm_element_value(dev, F_STONITH_DEVICE);
     if (device_id != NULL) {
         device = g_hash_table_lookup(device_list, device_id);
         if (device == NULL) {
             crm_err("Requested device '%s' is not available", device_id);
             pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
                                 "Requested device '%s' not found", device_id);
             return;
         }
         schedule_stonith_command(cmd, device);
 
     } else {
         const char *host = crm_element_value(dev, F_STONITH_TARGET);
 
         if (pcmk_is_set(cmd->options, st_opt_cs_nodeid)) {
             int nodeid = 0;
             crm_node_t *node = NULL;
 
             pcmk__scan_min_int(host, &nodeid, 0);
             node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY);
             if (node != NULL) {
                 host = node->uname;
             }
         }
 
         /* If we get to here, then self-fencing is implicitly allowed */
         get_capable_devices(host, cmd->action, cmd->default_timeout,
                             TRUE, cmd, stonith_fence_get_devices_cb,
                             fenced_support_flag(cmd->action));
     }
 
     pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
 }
 
 /*!
  * \internal
  * \brief Build an XML reply for a fencing operation
  *
  * \param[in] request  Request that reply is for
  * \param[in] data     If not NULL, add to reply as call data
  * \param[in] result   Full result of fencing operation
  *
  * \return Newly created XML reply
  * \note The caller is responsible for freeing the result.
  * \note This has some overlap with construct_async_reply(), but that copies
  *       values from an async_command_t, whereas this one copies them from the
  *       request.
  */
 xmlNode *
 fenced_construct_reply(const xmlNode *request, xmlNode *data,
                        const pcmk__action_result_t *result)
 {
     xmlNode *reply = NULL;
 
     reply = create_xml_node(NULL, T_STONITH_REPLY);
 
     crm_xml_add(reply, "st_origin", __func__);
     crm_xml_add(reply, F_TYPE, T_STONITH_NG);
     stonith__xe_set_result(reply, result);
 
     if (request == NULL) {
         /* Most likely, this is the result of a stonith operation that was
          * initiated before we came up. Unfortunately that means we lack enough
          * information to provide clients with a full result.
          *
          * @TODO Maybe synchronize this information at start-up?
          */
         crm_warn("Missing request information for client notifications for "
                  "operation with result '%s' (initiated before we came up?)",
                  pcmk_exec_status_str(result->execution_status));
 
     } else {
         const char *name = NULL;
         const char *value = NULL;
 
         // Attributes to copy from request to reply
         const char *names[] = {
             F_STONITH_OPERATION,
             F_STONITH_CALLID,
             F_STONITH_CLIENTID,
             F_STONITH_CLIENTNAME,
             F_STONITH_REMOTE_OP_ID,
             F_STONITH_CALLOPTS
         };
 
         for (int lpc = 0; lpc < PCMK__NELEM(names); lpc++) {
             name = names[lpc];
             value = crm_element_value(request, name);
             crm_xml_add(reply, name, value);
         }
         if (data != NULL) {
             add_message_xml(reply, F_STONITH_CALLDATA, data);
         }
     }
     return reply;
 }
 
 /*!
  * \internal
  * \brief Build an XML reply to an asynchronous fencing command
  *
  * \param[in] cmd     Fencing command that reply is for
  * \param[in] result  Command result
  */
 static xmlNode *
 construct_async_reply(const async_command_t *cmd,
                       const pcmk__action_result_t *result)
 {
     xmlNode *reply = create_xml_node(NULL, T_STONITH_REPLY);
 
     crm_xml_add(reply, "st_origin", __func__);
     crm_xml_add(reply, F_TYPE, T_STONITH_NG);
     crm_xml_add(reply, F_STONITH_OPERATION, cmd->op);
     crm_xml_add(reply, F_STONITH_DEVICE, cmd->device);
     crm_xml_add(reply, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id);
     crm_xml_add(reply, F_STONITH_CLIENTID, cmd->client);
     crm_xml_add(reply, F_STONITH_CLIENTNAME, cmd->client_name);
     crm_xml_add(reply, F_STONITH_TARGET, cmd->target);
     crm_xml_add(reply, F_STONITH_ACTION, cmd->op);
     crm_xml_add(reply, F_STONITH_ORIGIN, cmd->origin);
     crm_xml_add_int(reply, F_STONITH_CALLID, cmd->id);
     crm_xml_add_int(reply, F_STONITH_CALLOPTS, cmd->options);
 
     stonith__xe_set_result(reply, result);
     return reply;
 }
 
 bool fencing_peer_active(crm_node_t *peer)
 {
     if (peer == NULL) {
         return FALSE;
     } else if (peer->uname == NULL) {
         return FALSE;
     } else if (pcmk_is_set(peer->processes, crm_get_cluster_proc())) {
         return TRUE;
     }
     return FALSE;
 }
 
 void
 set_fencing_completed(remote_fencing_op_t *op)
 {
     struct timespec tv;
 
     qb_util_timespec_from_epoch_get(&tv);
     op->completed = tv.tv_sec;
     op->completed_nsec = tv.tv_nsec;
 }
 
 /*!
  * \internal
  * \brief Look for alternate node needed if local node shouldn't fence target
  *
  * \param[in] target  Node that must be fenced
  *
  * \return Name of an alternate node that should fence \p target if any,
  *         or NULL otherwise
  */
 static const char *
 check_alternate_host(const char *target)
 {
     if (pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) {
         GHashTableIter gIter;
         crm_node_t *entry = NULL;
 
         g_hash_table_iter_init(&gIter, crm_peer_cache);
         while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
             if (fencing_peer_active(entry)
                 && !pcmk__str_eq(entry->uname, target, pcmk__str_casei)) {
                 crm_notice("Forwarding self-fencing request to %s",
                            entry->uname);
                 return entry->uname;
             }
         }
         crm_warn("Will handle own fencing because no peer can");
     }
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Send a reply to a CPG peer or IPC client
  *
  * \param[in]     reply         XML reply to send
  * \param[in]     call_options  Send synchronously if st_opt_sync_call is set
  * \param[in]     remote_peer   If not NULL, name of peer node to send CPG reply
  * \param[in,out] client        If not NULL, client to send IPC reply
  */
 static void
 stonith_send_reply(xmlNode *reply, int call_options, const char *remote_peer,
                    pcmk__client_t *client)
 {
     CRM_CHECK((reply != NULL) && ((remote_peer != NULL) || (client != NULL)),
               return);
 
     if (remote_peer == NULL) {
         do_local_reply(reply, client, call_options);
     } else {
         send_cluster_message(crm_get_peer(0, remote_peer), crm_msg_stonith_ng,
                              reply, FALSE);
     }
 }
 
 static void 
 remove_relay_op(xmlNode * request)
 {
     xmlNode *dev = get_xpath_object("//@" F_STONITH_ACTION, request, LOG_TRACE);
     const char *relay_op_id = NULL; 
     const char *op_id = NULL;
     const char *client_name = NULL;
     const char *target = NULL; 
     remote_fencing_op_t *relay_op = NULL; 
 
     if (dev) { 
         target = crm_element_value(dev, F_STONITH_TARGET); 
     }
 
     relay_op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID_RELAY);
     op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID);
     client_name = crm_element_value(request, F_STONITH_CLIENTNAME);
 
     /* Delete RELAY operation. */
     if (relay_op_id && target && pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) {
         relay_op = g_hash_table_lookup(stonith_remote_op_list, relay_op_id);
 
         if (relay_op) {
             GHashTableIter iter;
             remote_fencing_op_t *list_op = NULL; 
             g_hash_table_iter_init(&iter, stonith_remote_op_list);
 
             /* If the operation to be deleted is registered as a duplicate, delete the registration. */
             while (g_hash_table_iter_next(&iter, NULL, (void **)&list_op)) {
                 GList *dup_iter = NULL;
                 if (list_op != relay_op) {
                     for (dup_iter = list_op->duplicates; dup_iter != NULL; dup_iter = dup_iter->next) {
                         remote_fencing_op_t *other = dup_iter->data;
                         if (other == relay_op) {
                             other->duplicates = g_list_remove(other->duplicates, relay_op);
                             break;
                         }
                     }
                 }
             }
             crm_debug("Deleting relay op %s ('%s'%s%s for %s), "
                       "replaced by op %s ('%s'%s%s for %s)",
                       relay_op->id, relay_op->action,
                       (relay_op->target == NULL)? "" : " targeting ",
                       pcmk__s(relay_op->target, ""),
                       relay_op->client_name, op_id, relay_op->action,
                       (target == NULL)? "" : " targeting ", pcmk__s(target, ""),
                       client_name);
 
             g_hash_table_remove(stonith_remote_op_list, relay_op_id);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Check whether an API request was sent by a privileged user
  *
  * API commands related to fencing configuration may be done only by privileged
  * IPC users (i.e. root or hacluster), because all other users should go through
  * the CIB to have ACLs applied. If no client was given, this is a peer request,
  * which is always allowed.
  *
  * \param[in] c   IPC client that sent request (or NULL if sent by CPG peer)
  * \param[in] op  Requested API operation (for logging only)
  *
  * \return true if sender is peer or privileged client, otherwise false
  */
 static inline bool
 is_privileged(const pcmk__client_t *c, const char *op)
 {
     if ((c == NULL) || pcmk_is_set(c->flags, pcmk__client_privileged)) {
         return true;
     } else {
         crm_warn("Rejecting IPC request '%s' from unprivileged client %s",
                  pcmk__s(op, ""), pcmk__client_name(c));
         return false;
     }
 }
 
 // CRM_OP_REGISTER
 static xmlNode *
 handle_register_request(pcmk__request_t *request)
 {
     xmlNode *reply = create_xml_node(NULL, "reply");
 
     CRM_ASSERT(request->ipc_client != NULL);
     crm_xml_add(reply, F_STONITH_OPERATION, CRM_OP_REGISTER);
     crm_xml_add(reply, F_STONITH_CLIENTID, request->ipc_client->id);
     pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
     pcmk__set_request_flags(request, pcmk__request_reuse_options);
     return reply;
 }
 
 // STONITH_OP_EXEC
 static xmlNode *
 handle_agent_request(pcmk__request_t *request)
 {
     execute_agent_action(request->xml, &request->result);
     if (request->result.execution_status == PCMK_EXEC_PENDING) {
         return NULL;
     }
     return fenced_construct_reply(request->xml, NULL, &request->result);
 }
 
 // STONITH_OP_TIMEOUT_UPDATE
 static xmlNode *
 handle_update_timeout_request(pcmk__request_t *request)
 {
     const char *call_id = crm_element_value(request->xml, F_STONITH_CALLID);
     const char *client_id = crm_element_value(request->xml, F_STONITH_CLIENTID);
     int op_timeout = 0;
 
     crm_element_value_int(request->xml, F_STONITH_TIMEOUT, &op_timeout);
     do_stonith_async_timeout_update(client_id, call_id, op_timeout);
     pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
     return NULL;
 }
 
 // STONITH_OP_QUERY
 static xmlNode *
 handle_query_request(pcmk__request_t *request)
 {
     int timeout = 0;
     xmlNode *dev = NULL;
     const char *action = NULL;
     const char *target = NULL;
     const char *client_id = crm_element_value(request->xml, F_STONITH_CLIENTID);
     struct st_query_data *query = NULL;
 
     if (request->peer != NULL) {
         // Record it for the future notification
         create_remote_stonith_op(client_id, request->xml, TRUE);
     }
 
     /* Delete the DC node RELAY operation. */
     remove_relay_op(request->xml);
 
     pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
 
     dev = get_xpath_object("//@" F_STONITH_ACTION, request->xml, LOG_NEVER);
     if (dev != NULL) {
         const char *device = crm_element_value(dev, F_STONITH_DEVICE);
 
         if (pcmk__str_eq(device, "manual_ack", pcmk__str_casei)) {
             return NULL; // No query or reply necessary
         }
         target = crm_element_value(dev, F_STONITH_TARGET);
         action = crm_element_value(dev, F_STONITH_ACTION);
     }
 
     crm_log_xml_trace(request->xml, "Query");
 
     query = calloc(1, sizeof(struct st_query_data));
     CRM_ASSERT(query != NULL);
 
     query->reply = fenced_construct_reply(request->xml, NULL, &request->result);
     pcmk__str_update(&query->remote_peer, request->peer);
     pcmk__str_update(&query->client_id, client_id);
     pcmk__str_update(&query->target, target);
     pcmk__str_update(&query->action, action);
     query->call_options = request->call_options;
 
     crm_element_value_int(request->xml, F_STONITH_TIMEOUT, &timeout);
     get_capable_devices(target, action, timeout,
                         pcmk_is_set(query->call_options, st_opt_allow_suicide),
                         query, stonith_query_capable_device_cb, st_device_supports_none);
     return NULL;
 }
 
 // T_STONITH_NOTIFY
 static xmlNode *
 handle_notify_request(pcmk__request_t *request)
 {
     const char *flag_name = NULL;
 
     CRM_ASSERT(request->ipc_client != NULL);
     flag_name = crm_element_value(request->xml, F_STONITH_NOTIFY_ACTIVATE);
     if (flag_name != NULL) {
         crm_debug("Enabling %s callbacks for client %s",
                   flag_name, pcmk__request_origin(request));
         pcmk__set_client_flags(request->ipc_client, get_stonith_flag(flag_name));
     }
 
     flag_name = crm_element_value(request->xml, F_STONITH_NOTIFY_DEACTIVATE);
     if (flag_name != NULL) {
         crm_debug("Disabling %s callbacks for client %s",
                   flag_name, pcmk__request_origin(request));
         pcmk__clear_client_flags(request->ipc_client,
                                  get_stonith_flag(flag_name));
     }
 
     pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
     pcmk__set_request_flags(request, pcmk__request_reuse_options);
 
     return pcmk__ipc_create_ack(request->ipc_flags, "ack", NULL, CRM_EX_OK);
 }
 
 // STONITH_OP_RELAY
 static xmlNode *
 handle_relay_request(pcmk__request_t *request)
 {
     xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request->xml,
                                     LOG_TRACE);
 
     crm_notice("Received forwarded fencing request from "
                "%s %s to fence (%s) peer %s",
                pcmk__request_origin_type(request),
                pcmk__request_origin(request),
                crm_element_value(dev, F_STONITH_ACTION),
                crm_element_value(dev, F_STONITH_TARGET));
 
     if (initiate_remote_stonith_op(NULL, request->xml, FALSE) == NULL) {
         fenced_set_protocol_error(&request->result);
         return fenced_construct_reply(request->xml, NULL, &request->result);
     }
 
     pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
     return NULL;
 }
 
 // STONITH_OP_FENCE
 static xmlNode *
 handle_fence_request(pcmk__request_t *request)
 {
     if ((request->peer != NULL) || stand_alone) {
         fence_locally(request->xml, &request->result);
 
     } else if (pcmk_is_set(request->call_options, st_opt_manual_ack)) {
         switch (fenced_handle_manual_confirmation(request->ipc_client,
                                                   request->xml)) {
             case pcmk_rc_ok:
                 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
                                  NULL);
                 break;
             case EINPROGRESS:
                 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
                                  NULL);
                 break;
             default:
                 fenced_set_protocol_error(&request->result);
                 break;
         }
 
     } else {
         const char *alternate_host = NULL;
         xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request->xml,
                                         LOG_TRACE);
         const char *target = crm_element_value(dev, F_STONITH_TARGET);
         const char *action = crm_element_value(dev, F_STONITH_ACTION);
         const char *device = crm_element_value(dev, F_STONITH_DEVICE);
 
         if (request->ipc_client != NULL) {
             int tolerance = 0;
 
             crm_notice("Client %s wants to fence (%s) %s using %s",
                        pcmk__request_origin(request), action,
                        target, (device? device : "any device"));
             crm_element_value_int(dev, F_STONITH_TOLERANCE, &tolerance);
             if (stonith_check_fence_tolerance(tolerance, target, action)) {
                 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
                                  NULL);
                 return fenced_construct_reply(request->xml, NULL,
                                               &request->result);
             }
             alternate_host = check_alternate_host(target);
 
         } else {
             crm_notice("Peer %s wants to fence (%s) '%s' with device '%s'",
                        request->peer, action, target,
                        (device == NULL)? "(any)" : device);
         }
 
         if (alternate_host != NULL) {
             const char *client_id = NULL;
             remote_fencing_op_t *op = NULL;
 
             if (request->ipc_client->id == 0) {
                 client_id = crm_element_value(request->xml, F_STONITH_CLIENTID);
             } else {
                 client_id = request->ipc_client->id;
             }
 
             /* Create a duplicate fencing operation to relay with the client ID.
              * When a query response is received, this operation should be
              * deleted to avoid keeping the duplicate around.
              */
             op = create_remote_stonith_op(client_id, request->xml, FALSE);
 
             crm_xml_add(request->xml, F_STONITH_OPERATION, STONITH_OP_RELAY);
             crm_xml_add(request->xml, F_STONITH_CLIENTID,
                         request->ipc_client->id);
             crm_xml_add(request->xml, F_STONITH_REMOTE_OP_ID, op->id);
             send_cluster_message(crm_get_peer(0, alternate_host),
                                  crm_msg_stonith_ng, request->xml, FALSE);
             pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
                              NULL);
 
         } else if (initiate_remote_stonith_op(request->ipc_client, request->xml,
                                               FALSE) == NULL) {
             fenced_set_protocol_error(&request->result);
 
         } else {
             pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
                              NULL);
         }
     }
 
     if (request->result.execution_status == PCMK_EXEC_PENDING) {
         return NULL;
     }
     return fenced_construct_reply(request->xml, NULL, &request->result);
 }
 
 // STONITH_OP_FENCE_HISTORY
 static xmlNode *
 handle_history_request(pcmk__request_t *request)
 {
     xmlNode *reply = NULL;
     xmlNode *data = NULL;
 
     stonith_fence_history(request->xml, &data, request->peer,
                           request->call_options);
     pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
     if (!pcmk_is_set(request->call_options, st_opt_discard_reply)) {
         /* When the local node broadcasts its history, it sets
          * st_opt_discard_reply and doesn't need a reply.
          */
         reply = fenced_construct_reply(request->xml, data, &request->result);
     }
     free_xml(data);
     return reply;
 }
 
 // STONITH_OP_DEVICE_ADD
 static xmlNode *
 handle_device_add_request(pcmk__request_t *request)
 {
     const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
     xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, request->xml,
                                     LOG_ERR);
 
     if (is_privileged(request->ipc_client, op)) {
         int rc = stonith_device_register(dev, FALSE);
 
         pcmk__set_result(&request->result,
                          ((rc == pcmk_ok)? CRM_EX_OK : CRM_EX_ERROR),
                          stonith__legacy2status(rc),
                          ((rc == pcmk_ok)? NULL : pcmk_strerror(rc)));
     } else {
         pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
                          PCMK_EXEC_INVALID,
                          "Unprivileged users must register device via CIB");
     }
     fenced_send_device_notification(op, &request->result,
                                     (dev == NULL)? NULL : ID(dev));
     return fenced_construct_reply(request->xml, NULL, &request->result);
 }
 
 // STONITH_OP_DEVICE_DEL
 static xmlNode *
 handle_device_delete_request(pcmk__request_t *request)
 {
     xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, request->xml,
                                     LOG_ERR);
     const char *device_id = crm_element_value(dev, XML_ATTR_ID);
     const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
 
     if (is_privileged(request->ipc_client, op)) {
         stonith_device_remove(device_id, false);
         pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
     } else {
         pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
                          PCMK_EXEC_INVALID,
                          "Unprivileged users must delete device via CIB");
     }
     fenced_send_device_notification(op, &request->result, device_id);
     return fenced_construct_reply(request->xml, NULL, &request->result);
 }
 
 // STONITH_OP_LEVEL_ADD
 static xmlNode *
 handle_level_add_request(pcmk__request_t *request)
 {
     char *desc = NULL;
     const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
 
     if (is_privileged(request->ipc_client, op)) {
         fenced_register_level(request->xml, &desc, &request->result);
     } else {
         unpack_level_request(request->xml, NULL, NULL, NULL, &desc);
         pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
                          PCMK_EXEC_INVALID,
                          "Unprivileged users must add level via CIB");
     }
     fenced_send_level_notification(op, &request->result, desc);
     free(desc);
     return fenced_construct_reply(request->xml, NULL, &request->result);
 }
 
 // STONITH_OP_LEVEL_DEL
 static xmlNode *
 handle_level_delete_request(pcmk__request_t *request)
 {
     char *desc = NULL;
     const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
 
     if (is_privileged(request->ipc_client, op)) {
         fenced_unregister_level(request->xml, &desc, &request->result);
     } else {
         unpack_level_request(request->xml, NULL, NULL, NULL, &desc);
         pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
                          PCMK_EXEC_INVALID,
                          "Unprivileged users must delete level via CIB");
     }
     fenced_send_level_notification(op, &request->result, desc);
     free(desc);
     return fenced_construct_reply(request->xml, NULL, &request->result);
 }
 
 // CRM_OP_RM_NODE_CACHE
 static xmlNode *
 handle_cache_request(pcmk__request_t *request)
 {
     int node_id = 0;
     const char *name = NULL;
 
     crm_element_value_int(request->xml, XML_ATTR_ID, &node_id);
     name = crm_element_value(request->xml, XML_ATTR_UNAME);
     reap_crm_member(node_id, name);
     pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
     return NULL;
 }
 
 static xmlNode *
 handle_unknown_request(pcmk__request_t *request)
 {
     crm_err("Unknown IPC request %s from %s %s",
             request->op, pcmk__request_origin_type(request),
             pcmk__request_origin(request));
     pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
                         "Unknown IPC request type '%s' (bug?)", request->op);
     return fenced_construct_reply(request->xml, NULL, &request->result);
 }
 
 static void
 fenced_register_handlers(void)
 {
     pcmk__server_command_t handlers[] = {
         { CRM_OP_REGISTER, handle_register_request },
         { STONITH_OP_EXEC, handle_agent_request },
         { STONITH_OP_TIMEOUT_UPDATE, handle_update_timeout_request },
         { STONITH_OP_QUERY, handle_query_request },
         { T_STONITH_NOTIFY, handle_notify_request },
         { STONITH_OP_RELAY, handle_relay_request },
         { STONITH_OP_FENCE, handle_fence_request },
         { STONITH_OP_FENCE_HISTORY, handle_history_request },
         { STONITH_OP_DEVICE_ADD, handle_device_add_request },
         { STONITH_OP_DEVICE_DEL, handle_device_delete_request },
         { STONITH_OP_LEVEL_ADD, handle_level_add_request },
         { STONITH_OP_LEVEL_DEL, handle_level_delete_request },
         { CRM_OP_RM_NODE_CACHE, handle_cache_request },
         { NULL, handle_unknown_request },
     };
 
     fenced_handlers = pcmk__register_handlers(handlers);
 }
 
 void
 fenced_unregister_handlers(void)
 {
     if (fenced_handlers != NULL) {
         g_hash_table_destroy(fenced_handlers);
         fenced_handlers = NULL;
     }
 }
 
 static void
 handle_request(pcmk__request_t *request)
 {
     xmlNode *reply = NULL;
     const char *reason = NULL;
 
     if (fenced_handlers == NULL) {
         fenced_register_handlers();
     }
     reply = pcmk__process_request(request, fenced_handlers);
     if (reply != NULL) {
         if (pcmk_is_set(request->flags, pcmk__request_reuse_options)
             && (request->ipc_client != NULL)) {
             /* Certain IPC-only commands must reuse the call options from the
              * original request rather than the ones set by stonith_send_reply()
              * -> do_local_reply().
              */
             pcmk__ipc_send_xml(request->ipc_client, request->ipc_id, reply,
                                request->ipc_flags);
             request->ipc_client->request_id = 0;
         } else {
             stonith_send_reply(reply, request->call_options,
                                request->peer, request->ipc_client);
         }
         free_xml(reply);
     }
 
     reason = request->result.exit_reason;
     crm_debug("Processed %s request from %s %s: %s%s%s%s",
               request->op, pcmk__request_origin_type(request),
               pcmk__request_origin(request),
               pcmk_exec_status_str(request->result.execution_status),
               (reason == NULL)? "" : " (",
               (reason == NULL)? "" : reason,
               (reason == NULL)? "" : ")");
 }
 
 static void
 handle_reply(pcmk__client_t *client, xmlNode *request, const char *remote_peer)
 {
     // Copy, because request might be freed before we want to log this
     char *op = crm_element_value_copy(request, F_STONITH_OPERATION);
 
     if (pcmk__str_eq(op, STONITH_OP_QUERY, pcmk__str_none)) {
         process_remote_stonith_query(request);
     } else if (pcmk__str_any_of(op, T_STONITH_NOTIFY, STONITH_OP_FENCE, NULL)) {
         fenced_process_fencing_reply(request);
     } else {
         crm_err("Ignoring unknown %s reply from %s %s",
                 pcmk__s(op, "untyped"), ((client == NULL)? "peer" : "client"),
                 ((client == NULL)? remote_peer : pcmk__client_name(client)));
         crm_log_xml_warn(request, "UnknownOp");
         free(op);
         return;
     }
     crm_debug("Processed %s reply from %s %s",
               op, ((client == NULL)? "peer" : "client"),
               ((client == NULL)? remote_peer : pcmk__client_name(client)));
     free(op);
 }
 
 /*!
  * \internal
  * \brief Handle a message from an IPC client or CPG peer
  *
  * \param[in,out] client      If not NULL, IPC client that sent message
  * \param[in]     id          If from IPC client, IPC message ID
  * \param[in]     flags       Message flags
  * \param[in,out] message     Message XML
  * \param[in]     remote_peer If not NULL, CPG peer that sent message
  */
 void
 stonith_command(pcmk__client_t *client, uint32_t id, uint32_t flags,
                 xmlNode *message, const char *remote_peer)
 {
     int call_options = st_opt_none;
     bool is_reply = false;
 
     CRM_CHECK(message != NULL, return);
 
     if (get_xpath_object("//" T_STONITH_REPLY, message, LOG_NEVER) != NULL) {
         is_reply = true;
     }
     crm_element_value_int(message, F_STONITH_CALLOPTS, &call_options);
     crm_debug("Processing %ssynchronous %s %s %u from %s %s",
               pcmk_is_set(call_options, st_opt_sync_call)? "" : "a",
               crm_element_value(message, F_STONITH_OPERATION),
               (is_reply? "reply" : "request"), id,
               ((client == NULL)? "peer" : "client"),
               ((client == NULL)? remote_peer : pcmk__client_name(client)));
 
     if (pcmk_is_set(call_options, st_opt_sync_call)) {
         CRM_ASSERT(client == NULL || client->request_id == id);
     }
 
     if (is_reply) {
         handle_reply(client, message, remote_peer);
     } else {
         pcmk__request_t request = {
             .ipc_client     = client,
             .ipc_id         = id,
             .ipc_flags      = flags,
             .peer           = remote_peer,
             .xml            = message,
             .call_options   = call_options,
             .result         = PCMK__UNKNOWN_RESULT,
         };
 
         request.op = crm_element_value_copy(request.xml, F_STONITH_OPERATION);
         CRM_CHECK(request.op != NULL, return);
 
         if (pcmk_is_set(request.call_options, st_opt_sync_call)) {
             pcmk__set_request_flags(&request, pcmk__request_sync);
         }
 
         handle_request(&request);
         pcmk__reset_request(&request);
     }
 }
diff --git a/daemons/fenced/fenced_remote.c b/daemons/fenced/fenced_remote.c
index 3b7ab05e90..dc67947954 100644
--- a/daemons/fenced/fenced_remote.c
+++ b/daemons/fenced/fenced_remote.c
@@ -1,2478 +1,2509 @@
 /*
  * Copyright 2009-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/stat.h>
 #include <unistd.h>
 #include <sys/utsname.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <ctype.h>
 #include <regex.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/ipc_internal.h>
 #include <crm/cluster/internal.h>
 
 #include <crm/stonith-ng.h>
 #include <crm/fencing/internal.h>
 #include <crm/common/xml.h>
 #include <crm/common/xml_internal.h>
 
 #include <crm/common/util.h>
 #include <pacemaker-fenced.h>
 
 #define TIMEOUT_MULTIPLY_FACTOR 1.2
 
 /* When one fencer queries its peers for devices able to handle a fencing
  * request, each peer will reply with a list of such devices available to it.
  * Each reply will be parsed into a peer_device_info_t, with each device's
  * information kept in a device_properties_t.
  */
 
 typedef struct device_properties_s {
     /* Whether access to this device has been verified */
     gboolean verified;
 
     /* The remaining members are indexed by the operation's "phase" */
 
     /* Whether this device has been executed in each phase */
     gboolean executed[st_phase_max];
     /* Whether this device is disallowed from executing in each phase */
     gboolean disallowed[st_phase_max];
     /* Action-specific timeout for each phase */
     int custom_action_timeout[st_phase_max];
     /* Action-specific maximum random delay for each phase */
     int delay_max[st_phase_max];
     /* Action-specific base delay for each phase */
     int delay_base[st_phase_max];
     /* Group of enum st_device_flags */
     uint32_t device_support_flags;
 } device_properties_t;
 
 typedef struct {
     /* Name of peer that sent this result */
     char *host;
     /* Only try peers for non-topology based operations once */
     gboolean tried;
     /* Number of entries in the devices table */
     int ndevices;
     /* Devices available to this host that are capable of fencing the target */
     GHashTable *devices;
 } peer_device_info_t;
 
 GHashTable *stonith_remote_op_list = NULL;
 
 extern xmlNode *stonith_create_op(int call_id, const char *token, const char *op, xmlNode * data,
                                   int call_options);
 
 static void request_peer_fencing(remote_fencing_op_t *op,
                                  peer_device_info_t *peer);
 static void finalize_op(remote_fencing_op_t *op, xmlNode *data, bool dup);
 static void report_timeout_period(remote_fencing_op_t * op, int op_timeout);
 static int get_op_total_timeout(const remote_fencing_op_t *op,
                                 const peer_device_info_t *chosen_peer);
 
 static gint
 sort_strings(gconstpointer a, gconstpointer b)
 {
     return strcmp(a, b);
 }
 
 static void
 free_remote_query(gpointer data)
 {
     if (data != NULL) {
         peer_device_info_t *peer = data;
 
         g_hash_table_destroy(peer->devices);
         free(peer->host);
         free(peer);
     }
 }
 
 void
 free_stonith_remote_op_list(void)
 {
     if (stonith_remote_op_list != NULL) {
         g_hash_table_destroy(stonith_remote_op_list);
         stonith_remote_op_list = NULL;
     }
 }
 
 struct peer_count_data {
     const remote_fencing_op_t *op;
     gboolean verified_only;
     uint32_t support_action_only;
     int count;
 };
 
 /*!
  * \internal
  * \brief Increment a counter if a device has not been executed yet
  *
  * \param[in]     key        Device ID (ignored)
  * \param[in]     value      Device properties
  * \param[in,out] user_data  Peer count data
  */
 static void
 count_peer_device(gpointer key, gpointer value, gpointer user_data)
 {
     device_properties_t *props = (device_properties_t*)value;
     struct peer_count_data *data = user_data;
 
     if (!props->executed[data->op->phase]
         && (!data->verified_only || props->verified)
         && ((data->support_action_only == st_device_supports_none) || pcmk_is_set(props->device_support_flags, data->support_action_only))) {
         ++(data->count);
     }
 }
 
 /*!
  * \internal
  * \brief Check the number of available devices in a peer's query results
  *
  * \param[in] op             Operation that results are for
  * \param[in] peer           Peer to count
  * \param[in] verified_only  Whether to count only verified devices
  * \param[in] support_action_only Whether to count only devices that support action
  *
  * \return Number of devices available to peer that were not already executed
  */
 static int
 count_peer_devices(const remote_fencing_op_t *op,
                    const peer_device_info_t *peer, gboolean verified_only, uint32_t support_on_action_only)
 {
     struct peer_count_data data;
 
     data.op = op;
     data.verified_only = verified_only;
     data.support_action_only = support_on_action_only;
     data.count = 0;
     if (peer) {
         g_hash_table_foreach(peer->devices, count_peer_device, &data);
     }
     return data.count;
 }
 
 /*!
  * \internal
  * \brief Search for a device in a query result
  *
  * \param[in] op      Operation that result is for
  * \param[in] peer    Query result for a peer
  * \param[in] device  Device ID to search for
  *
  * \return Device properties if found, NULL otherwise
  */
 static device_properties_t *
 find_peer_device(const remote_fencing_op_t *op, const peer_device_info_t *peer,
                  const char *device, uint32_t support_action_only)
 {
     device_properties_t *props = g_hash_table_lookup(peer->devices, device);
 
     if (props && support_action_only != st_device_supports_none && !pcmk_is_set(props->device_support_flags, support_action_only)) {
         return NULL;
     }
     return (props && !props->executed[op->phase]
            && !props->disallowed[op->phase])? props : NULL;
 }
 
 /*!
  * \internal
  * \brief Find a device in a peer's device list and mark it as executed
  *
  * \param[in]     op                     Operation that peer result is for
  * \param[in,out] peer                   Peer with results to search
  * \param[in]     device                 ID of device to mark as done
  * \param[in]     verified_devices_only  Only consider verified devices
  *
  * \return TRUE if device was found and marked, FALSE otherwise
  */
 static gboolean
 grab_peer_device(const remote_fencing_op_t *op, peer_device_info_t *peer,
                  const char *device, gboolean verified_devices_only)
 {
     device_properties_t *props = find_peer_device(op, peer, device,
                                                   fenced_support_flag(op->action));
 
     if ((props == NULL) || (verified_devices_only && !props->verified)) {
         return FALSE;
     }
 
     crm_trace("Removing %s from %s (%d remaining)",
               device, peer->host, count_peer_devices(op, peer, FALSE, st_device_supports_none));
     props->executed[op->phase] = TRUE;
     return TRUE;
 }
 
 static void
 clear_remote_op_timers(remote_fencing_op_t * op)
 {
     if (op->query_timer) {
         g_source_remove(op->query_timer);
         op->query_timer = 0;
     }
     if (op->op_timer_total) {
         g_source_remove(op->op_timer_total);
         op->op_timer_total = 0;
     }
     if (op->op_timer_one) {
         g_source_remove(op->op_timer_one);
         op->op_timer_one = 0;
     }
 }
 
 static void
 free_remote_op(gpointer data)
 {
     remote_fencing_op_t *op = data;
 
     crm_log_xml_debug(op->request, "Destroying");
 
     clear_remote_op_timers(op);
 
     free(op->id);
     free(op->action);
     free(op->delegate);
     free(op->target);
     free(op->client_id);
     free(op->client_name);
     free(op->originator);
 
     if (op->query_results) {
         g_list_free_full(op->query_results, free_remote_query);
     }
     if (op->request) {
         free_xml(op->request);
         op->request = NULL;
     }
     if (op->devices_list) {
         g_list_free_full(op->devices_list, free);
         op->devices_list = NULL;
     }
     g_list_free_full(op->automatic_list, free);
     g_list_free(op->duplicates);
 
     pcmk__reset_result(&op->result);
     free(op);
 }
 
 void
 init_stonith_remote_op_hash_table(GHashTable **table)
 {
     if (*table == NULL) {
         *table = pcmk__strkey_table(NULL, free_remote_op);
     }
 }
 
 /*!
  * \internal
  * \brief Return an operation's originally requested action (before any remap)
  *
  * \param[in] op  Operation to check
  *
  * \return Operation's original action
  */
 static const char *
 op_requested_action(const remote_fencing_op_t *op)
 {
     return ((op->phase > st_phase_requested)? "reboot" : op->action);
 }
 
 /*!
  * \internal
  * \brief Remap a "reboot" operation to the "off" phase
  *
  * \param[in,out] op      Operation to remap
  */
 static void
 op_phase_off(remote_fencing_op_t *op)
 {
     crm_info("Remapping multiple-device reboot targeting %s to 'off' "
              CRM_XS " id=%.8s", op->target, op->id);
     op->phase = st_phase_off;
 
     /* Happily, "off" and "on" are shorter than "reboot", so we can reuse the
      * memory allocation at each phase.
      */
     strcpy(op->action, "off");
 }
 
 /*!
  * \internal
  * \brief Advance a remapped reboot operation to the "on" phase
  *
  * \param[in,out] op  Operation to remap
  */
 static void
 op_phase_on(remote_fencing_op_t *op)
 {
     GList *iter = NULL;
 
     crm_info("Remapped 'off' targeting %s complete, "
              "remapping to 'on' for %s " CRM_XS " id=%.8s",
              op->target, op->client_name, op->id);
     op->phase = st_phase_on;
     strcpy(op->action, "on");
 
     /* Skip devices with automatic unfencing, because the cluster will handle it
      * when the node rejoins.
      */
     for (iter = op->automatic_list; iter != NULL; iter = iter->next) {
         GList *match = g_list_find_custom(op->devices_list, iter->data,
                                             sort_strings);
 
         if (match) {
             op->devices_list = g_list_remove(op->devices_list, match->data);
         }
     }
     g_list_free_full(op->automatic_list, free);
     op->automatic_list = NULL;
 
     /* Rewind device list pointer */
     op->devices = op->devices_list;
 }
 
 /*!
  * \internal
  * \brief Reset a remapped reboot operation
  *
  * \param[in,out] op  Operation to reset
  */
 static void
 undo_op_remap(remote_fencing_op_t *op)
 {
     if (op->phase > 0) {
         crm_info("Undoing remap of reboot targeting %s for %s "
                  CRM_XS " id=%.8s", op->target, op->client_name, op->id);
         op->phase = st_phase_requested;
         strcpy(op->action, "reboot");
     }
 }
 
 /*!
  * \internal
  * \brief Create notification data XML for a fencing operation result
  *
  * \param[in] op      Fencer operation that completed
  *
  * \return Newly created XML to add as notification data
  * \note The caller is responsible for freeing the result.
  */
 static xmlNode *
 fencing_result2xml(const remote_fencing_op_t *op)
 {
     xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE);
 
     crm_xml_add_int(notify_data, "state", op->state);
     crm_xml_add(notify_data, F_STONITH_TARGET, op->target);
     crm_xml_add(notify_data, F_STONITH_ACTION, op->action);
     crm_xml_add(notify_data, F_STONITH_DELEGATE, op->delegate);
     crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, op->id);
     crm_xml_add(notify_data, F_STONITH_ORIGIN, op->originator);
     crm_xml_add(notify_data, F_STONITH_CLIENTID, op->client_id);
     crm_xml_add(notify_data, F_STONITH_CLIENTNAME, op->client_name);
 
     return notify_data;
 }
 
 /*!
  * \internal
  * \brief Broadcast a fence result notification to all CPG peers
  *
  * \param[in] op         Fencer operation that completed
  * \param[in] op_merged  Whether this operation is a duplicate of another
  */
 void
 fenced_broadcast_op_result(const remote_fencing_op_t *op, bool op_merged)
 {
     static int count = 0;
     xmlNode *bcast = create_xml_node(NULL, T_STONITH_REPLY);
     xmlNode *notify_data = fencing_result2xml(op);
 
     count++;
     crm_trace("Broadcasting result to peers");
     crm_xml_add(bcast, F_TYPE, T_STONITH_NOTIFY);
     crm_xml_add(bcast, F_SUBTYPE, "broadcast");
     crm_xml_add(bcast, F_STONITH_OPERATION, T_STONITH_NOTIFY);
     crm_xml_add_int(bcast, "count", count);
 
     if (op_merged) {
         pcmk__xe_set_bool_attr(bcast, F_STONITH_MERGED, true);
     }
 
     stonith__xe_set_result(notify_data, &op->result);
 
     add_message_xml(bcast, F_STONITH_CALLDATA, notify_data);
     send_cluster_message(NULL, crm_msg_stonith_ng, bcast, FALSE);
     free_xml(notify_data);
     free_xml(bcast);
 
     return;
 }
 
 /*!
  * \internal
  * \brief Reply to a local request originator and notify all subscribed clients
  *
  * \param[in,out] op    Fencer operation that completed
  * \param[in,out] data  Top-level XML to add notification to
  */
 static void
 handle_local_reply_and_notify(remote_fencing_op_t *op, xmlNode *data)
 {
     xmlNode *notify_data = NULL;
     xmlNode *reply = NULL;
     pcmk__client_t *client = NULL;
 
     if (op->notify_sent == TRUE) {
         /* nothing to do */
         return;
     }
 
     /* Do notification with a clean data object */
     crm_xml_add_int(data, "state", op->state);
     crm_xml_add(data, F_STONITH_TARGET, op->target);
     crm_xml_add(data, F_STONITH_OPERATION, op->action);
 
     reply = fenced_construct_reply(op->request, data, &op->result);
     crm_xml_add(reply, F_STONITH_DELEGATE, op->delegate);
 
     /* Send fencing OP reply to local client that initiated fencing */
     client = pcmk__find_client_by_id(op->client_id);
     if (client == NULL) {
         crm_trace("Skipping reply to %s: no longer a client", op->client_id);
     } else {
         do_local_reply(reply, client, op->call_options);
     }
 
     /* bcast to all local clients that the fencing operation happend */
     notify_data = fencing_result2xml(op);
     fenced_send_notification(T_STONITH_NOTIFY_FENCE, &op->result, notify_data);
     free_xml(notify_data);
     fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
 
     /* mark this op as having notify's already sent */
     op->notify_sent = TRUE;
     free_xml(reply);
 }
 
 /*!
  * \internal
  * \brief Finalize all duplicates of a given fencer operation
  *
  * \param[in,out] op    Fencer operation that completed
  * \param[in,out] data  Top-level XML to add notification to
  */
 static void
 finalize_op_duplicates(remote_fencing_op_t *op, xmlNode *data)
 {
     for (GList *iter = op->duplicates; iter != NULL; iter = iter->next) {
         remote_fencing_op_t *other = iter->data;
 
         if (other->state == st_duplicate) {
             other->state = op->state;
             crm_debug("Performing duplicate notification for %s@%s: %s "
                       CRM_XS " id=%.8s",
                       other->client_name, other->originator,
                       pcmk_exec_status_str(op->result.execution_status),
                       other->id);
             pcmk__copy_result(&op->result, &other->result);
             finalize_op(other, data, true);
 
         } else {
             // Possible if (for example) it timed out already
             crm_err("Skipping duplicate notification for %s@%s "
                     CRM_XS " state=%s id=%.8s",
                     other->client_name, other->originator,
                     stonith_op_state_str(other->state), other->id);
         }
     }
 }
 
 static char *
 delegate_from_xml(xmlNode *xml)
 {
     xmlNode *match = get_xpath_object("//@" F_STONITH_DELEGATE, xml, LOG_NEVER);
 
     if (match == NULL) {
         return crm_element_value_copy(xml, F_ORIG);
     } else {
         return crm_element_value_copy(match, F_STONITH_DELEGATE);
     }
 }
 
 /*!
  * \internal
  * \brief Finalize a peer fencing operation
  *
  * Clean up after a fencing operation completes. This function has two code
  * paths: the executioner uses it to broadcast the result to CPG peers, and then
  * each peer (including the executioner) uses it to process that broadcast and
  * notify its IPC clients of the result.
  *
  * \param[in,out] op      Fencer operation that completed
  * \param[in,out] data    If not NULL, XML reply of last delegated operation
  * \param[in]     dup     Whether this operation is a duplicate of another
  *                        (in which case, do not broadcast the result)
  *
  *  \note The operation result should be set before calling this function.
  */
 static void
 finalize_op(remote_fencing_op_t *op, xmlNode *data, bool dup)
 {
     int level = LOG_ERR;
     const char *subt = NULL;
     xmlNode *local_data = NULL;
     gboolean op_merged = FALSE;
 
     CRM_CHECK((op != NULL), return);
 
     // This is a no-op if timers have already been cleared
     clear_remote_op_timers(op);
 
     if (op->notify_sent) {
         // Most likely, this is a timed-out action that eventually completed
         crm_notice("Operation '%s'%s%s by %s for %s@%s%s: "
                    "Result arrived too late " CRM_XS " id=%.8s",
                    op->action, (op->target? " targeting " : ""),
                    (op->target? op->target : ""),
                    (op->delegate? op->delegate : "unknown node"),
                    op->client_name, op->originator,
                    (op_merged? " (merged)" : ""),
                    op->id);
         return;
     }
 
     set_fencing_completed(op);
     undo_op_remap(op);
 
     if (data == NULL) {
         data = create_xml_node(NULL, "remote-op");
         local_data = data;
 
     } else if (op->delegate == NULL) {
         switch (op->result.execution_status) {
             case PCMK_EXEC_NO_FENCE_DEVICE:
                 break;
 
             case PCMK_EXEC_INVALID:
                 if (op->result.exit_status != CRM_EX_EXPIRED) {
                     op->delegate = delegate_from_xml(data);
                 }
                 break;
 
             default:
                 op->delegate = delegate_from_xml(data);
                 break;
         }
     }
 
     if (dup || (crm_element_value(data, F_STONITH_MERGED) != NULL)) {
         op_merged = true;
     }
 
     /* Tell everyone the operation is done, we will continue
      * with doing the local notifications once we receive
      * the broadcast back. */
     subt = crm_element_value(data, F_SUBTYPE);
     if (!dup && !pcmk__str_eq(subt, "broadcast", pcmk__str_casei)) {
         /* Defer notification until the bcast message arrives */
         fenced_broadcast_op_result(op, op_merged);
         free_xml(local_data);
         return;
     }
 
     if (pcmk__result_ok(&op->result) || dup
         || !pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) {
         level = LOG_NOTICE;
     }
     do_crm_log(level, "Operation '%s'%s%s by %s for %s@%s%s: %s (%s%s%s) "
                CRM_XS " id=%.8s", op->action, (op->target? " targeting " : ""),
                (op->target? op->target : ""),
                (op->delegate? op->delegate : "unknown node"),
                op->client_name, op->originator,
                (op_merged? " (merged)" : ""),
                crm_exit_str(op->result.exit_status),
                pcmk_exec_status_str(op->result.execution_status),
                ((op->result.exit_reason == NULL)? "" : ": "),
                ((op->result.exit_reason == NULL)? "" : op->result.exit_reason),
                op->id);
 
     handle_local_reply_and_notify(op, data);
 
     if (!dup) {
         finalize_op_duplicates(op, data);
     }
 
     /* Free non-essential parts of the record
      * Keep the record around so we can query the history
      */
     if (op->query_results) {
         g_list_free_full(op->query_results, free_remote_query);
         op->query_results = NULL;
     }
     if (op->request) {
         free_xml(op->request);
         op->request = NULL;
     }
 
     free_xml(local_data);
 }
 
 /*!
  * \internal
  * \brief Finalize a watchdog fencer op after the waiting time expires
  *
  * \param[in,out] userdata  Fencer operation that completed
  *
  * \return G_SOURCE_REMOVE (which tells glib not to restart timer)
  */
 static gboolean
 remote_op_watchdog_done(gpointer userdata)
 {
     remote_fencing_op_t *op = userdata;
 
     op->op_timer_one = 0;
 
     crm_notice("Self-fencing (%s) by %s for %s assumed complete "
                CRM_XS " id=%.8s",
                op->action, op->target, op->client_name, op->id);
     op->state = st_done;
     pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
     finalize_op(op, NULL, false);
     return G_SOURCE_REMOVE;
 }
 
 static gboolean
 remote_op_timeout_one(gpointer userdata)
 {
     remote_fencing_op_t *op = userdata;
 
     op->op_timer_one = 0;
 
     crm_notice("Peer's '%s' action targeting %s for client %s timed out " CRM_XS
                " id=%.8s", op->action, op->target, op->client_name, op->id);
     pcmk__set_result(&op->result, CRM_EX_ERROR, PCMK_EXEC_TIMEOUT,
                      "Peer did not return fence result within timeout");
 
+    // The requested delay has been applied for the first device
+    if (op->delay > 0) {
+        op->delay = 0;
+        crm_trace("Try another device for '%s' action targeting %s "
+                  "for client %s without delay " CRM_XS " id=%.8s",
+                  op->action, op->target, op->client_name, op->id);
+    }
+
     // Try another device, if appropriate
     request_peer_fencing(op, NULL);
     return G_SOURCE_REMOVE;
 }
 
 /*!
  * \internal
  * \brief Finalize a remote fencer operation that timed out
  *
  * \param[in,out] op      Fencer operation that timed out
  * \param[in]     reason  Readable description of what step timed out
  */
 static void
 finalize_timed_out_op(remote_fencing_op_t *op, const char *reason)
 {
     crm_debug("Action '%s' targeting %s for client %s timed out "
               CRM_XS " id=%.8s",
               op->action, op->target, op->client_name, op->id);
 
     if (op->phase == st_phase_on) {
         /* A remapped reboot operation timed out in the "on" phase, but the
          * "off" phase completed successfully, so quit trying any further
          * devices, and return success.
          */
         op->state = st_done;
         pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
     } else {
         op->state = st_failed;
         pcmk__set_result(&op->result, CRM_EX_ERROR, PCMK_EXEC_TIMEOUT, reason);
     }
     finalize_op(op, NULL, false);
 }
 
 /*!
  * \internal
  * \brief Finalize a remote fencer operation that timed out
  *
  * \param[in,out] userdata  Fencer operation that timed out
  *
  * \return G_SOURCE_REMOVE (which tells glib not to restart timer)
  */
 static gboolean
 remote_op_timeout(gpointer userdata)
 {
     remote_fencing_op_t *op = userdata;
 
     op->op_timer_total = 0;
 
     if (op->state == st_done) {
         crm_debug("Action '%s' targeting %s for client %s already completed "
                   CRM_XS " id=%.8s",
                   op->action, op->target, op->client_name, op->id);
     } else {
         finalize_timed_out_op(userdata, "Fencing did not complete within a "
                                         "total timeout based on the "
                                         "configured timeout and retries for "
                                         "any devices attempted");
     }
     return G_SOURCE_REMOVE;
 }
 
 static gboolean
 remote_op_query_timeout(gpointer data)
 {
     remote_fencing_op_t *op = data;
 
     op->query_timer = 0;
 
     if (op->state == st_done) {
         crm_debug("Operation %.8s targeting %s already completed",
                   op->id, op->target);
     } else if (op->state == st_exec) {
         crm_debug("Operation %.8s targeting %s already in progress",
                   op->id, op->target);
     } else if (op->query_results) {
         // Query succeeded, so attempt the actual fencing
         crm_debug("Query %.8s targeting %s complete (state=%s)",
                   op->id, op->target, stonith_op_state_str(op->state));
         request_peer_fencing(op, NULL);
     } else {
         crm_debug("Query %.8s targeting %s timed out (state=%s)",
                   op->id, op->target, stonith_op_state_str(op->state));
         finalize_timed_out_op(op, "No capable peers replied to device query "
                                   "within timeout");
     }
 
     return G_SOURCE_REMOVE;
 }
 
 static gboolean
 topology_is_empty(stonith_topology_t *tp)
 {
     int i;
 
     if (tp == NULL) {
         return TRUE;
     }
 
     for (i = 0; i < ST_LEVEL_MAX; i++) {
         if (tp->levels[i] != NULL) {
             return FALSE;
         }
     }
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Add a device to an operation's automatic unfencing list
  *
  * \param[in,out] op      Operation to modify
  * \param[in]     device  Device ID to add
  */
 static void
 add_required_device(remote_fencing_op_t *op, const char *device)
 {
     GList *match  = g_list_find_custom(op->automatic_list, device,
                                          sort_strings);
 
     if (!match) {
         op->automatic_list = g_list_prepend(op->automatic_list, strdup(device));
     }
 }
 
 /*!
  * \internal
  * \brief Remove a device from the automatic unfencing list
  *
  * \param[in,out] op      Operation to modify
  * \param[in]     device  Device ID to remove
  */
 static void
 remove_required_device(remote_fencing_op_t *op, const char *device)
 {
     GList *match = g_list_find_custom(op->automatic_list, device,
                                         sort_strings);
 
     if (match) {
         op->automatic_list = g_list_remove(op->automatic_list, match->data);
     }
 }
 
 /* deep copy the device list */
 static void
 set_op_device_list(remote_fencing_op_t * op, GList *devices)
 {
     GList *lpc = NULL;
 
     if (op->devices_list) {
         g_list_free_full(op->devices_list, free);
         op->devices_list = NULL;
     }
     for (lpc = devices; lpc != NULL; lpc = lpc->next) {
         op->devices_list = g_list_append(op->devices_list, strdup(lpc->data));
     }
     op->devices = op->devices_list;
 }
 
 /*!
  * \internal
  * \brief Check whether a node matches a topology target
  *
  * \param[in] tp    Topology table entry to check
  * \param[in] node  Name of node to check
  *
  * \return TRUE if node matches topology target
  */
 static gboolean
 topology_matches(const stonith_topology_t *tp, const char *node)
 {
     regex_t r_patt;
 
     CRM_CHECK(node && tp && tp->target, return FALSE);
     switch (tp->kind) {
         case fenced_target_by_attribute:
             /* This level targets by attribute, so tp->target is a NAME=VALUE pair
              * of a permanent attribute applied to targeted nodes. The test below
              * relies on the locally cached copy of the CIB, so if fencing needs to
              * be done before the initial CIB is received or after a malformed CIB
              * is received, then the topology will be unable to be used.
              */
             if (node_has_attr(node, tp->target_attribute, tp->target_value)) {
                 crm_notice("Matched %s with %s by attribute", node, tp->target);
                 return TRUE;
             }
             break;
 
         case fenced_target_by_pattern:
             /* This level targets node names matching a pattern, so tp->target
              * (and tp->target_pattern) is a regular expression.
              */
             if (regcomp(&r_patt, tp->target_pattern, REG_EXTENDED|REG_NOSUB)) {
                 crm_info("Bad regex '%s' for fencing level", tp->target);
             } else {
                 int status = regexec(&r_patt, node, 0, NULL, 0);
 
                 regfree(&r_patt);
                 if (status == 0) {
                     crm_notice("Matched %s with %s by name", node, tp->target);
                     return TRUE;
                 }
             }
             break;
 
         case fenced_target_by_name:
             crm_trace("Testing %s against %s", node, tp->target);
             return pcmk__str_eq(tp->target, node, pcmk__str_casei);
 
         default:
             break;
     }
     crm_trace("No match for %s with %s", node, tp->target);
     return FALSE;
 }
 
 stonith_topology_t *
 find_topology_for_host(const char *host) 
 {
     GHashTableIter tIter;
     stonith_topology_t *tp = g_hash_table_lookup(topology, host);
 
     if(tp != NULL) {
         crm_trace("Found %s for %s in %d entries", tp->target, host, g_hash_table_size(topology));
         return tp;
     }
 
     g_hash_table_iter_init(&tIter, topology);
     while (g_hash_table_iter_next(&tIter, NULL, (gpointer *) & tp)) {
         if (topology_matches(tp, host)) {
             crm_trace("Found %s for %s in %d entries", tp->target, host, g_hash_table_size(topology));
             return tp;
         }
     }
 
     crm_trace("No matches for %s in %d topology entries", host, g_hash_table_size(topology));
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Set fencing operation's device list to target's next topology level
  *
  * \param[in,out] op        Remote fencing operation to modify
  * \param[in]     empty_ok  If true, an operation without a target (i.e.
  *                          queries) or a target without a topology will get a
  *                          pcmk_rc_ok return value instead of ENODEV
  *
  * \return Standard Pacemaker return value
  */
 static int
 advance_topology_level(remote_fencing_op_t *op, bool empty_ok)
 {
     stonith_topology_t *tp = NULL;
 
     if (op->target) {
         tp = find_topology_for_host(op->target);
     }
     if (topology_is_empty(tp)) {
         return empty_ok? pcmk_rc_ok : ENODEV;
     }
 
     CRM_ASSERT(tp->levels != NULL);
 
     stonith__set_call_options(op->call_options, op->id, st_opt_topology);
 
     /* This is a new level, so undo any remapping left over from previous */
     undo_op_remap(op);
 
     do {
         op->level++;
 
     } while (op->level < ST_LEVEL_MAX && tp->levels[op->level] == NULL);
 
     if (op->level < ST_LEVEL_MAX) {
         crm_trace("Attempting fencing level %d targeting %s (%d devices) "
                   "for client %s@%s (id=%.8s)",
                   op->level, op->target, g_list_length(tp->levels[op->level]),
                   op->client_name, op->originator, op->id);
         set_op_device_list(op, tp->levels[op->level]);
 
         // The requested delay has been applied for the first fencing level
         if (op->level > 1 && op->delay > 0) {
             op->delay = 0;
         }
 
         if ((g_list_next(op->devices_list) != NULL)
             && pcmk__str_eq(op->action, "reboot", pcmk__str_none)) {
             /* A reboot has been requested for a topology level with multiple
              * devices. Instead of rebooting the devices sequentially, we will
              * turn them all off, then turn them all on again. (Think about
              * switched power outlets for redundant power supplies.)
              */
             op_phase_off(op);
         }
         return pcmk_rc_ok;
     }
 
     crm_info("All %sfencing options targeting %s for client %s@%s failed "
              CRM_XS " id=%.8s",
              (stonith_watchdog_timeout_ms > 0)?"non-watchdog ":"",
              op->target, op->client_name, op->originator, op->id);
     return ENODEV;
 }
 
 /*!
  * \internal
  * \brief If fencing operation is a duplicate, merge it into the other one
  *
  * \param[in,out] op  Fencing operation to check
  */
 static void
 merge_duplicates(remote_fencing_op_t *op)
 {
     GHashTableIter iter;
     remote_fencing_op_t *other = NULL;
 
     time_t now = time(NULL);
 
     g_hash_table_iter_init(&iter, stonith_remote_op_list);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&other)) {
         const char *other_action = op_requested_action(other);
 
         if (!strcmp(op->id, other->id)) {
             continue; // Don't compare against self
         }
         if (other->state > st_exec) {
             crm_trace("%.8s not duplicate of %.8s: not in progress",
                       op->id, other->id);
             continue;
         }
         if (!pcmk__str_eq(op->target, other->target, pcmk__str_casei)) {
             crm_trace("%.8s not duplicate of %.8s: node %s vs. %s",
                       op->id, other->id, op->target, other->target);
             continue;
         }
         if (!pcmk__str_eq(op->action, other_action, pcmk__str_none)) {
             crm_trace("%.8s not duplicate of %.8s: action %s vs. %s",
                       op->id, other->id, op->action, other_action);
             continue;
         }
         if (pcmk__str_eq(op->client_name, other->client_name, pcmk__str_casei)) {
             crm_trace("%.8s not duplicate of %.8s: same client %s",
                       op->id, other->id, op->client_name);
             continue;
         }
         if (pcmk__str_eq(other->target, other->originator, pcmk__str_casei)) {
             crm_trace("%.8s not duplicate of %.8s: suicide for %s",
                       op->id, other->id, other->target);
             continue;
         }
         if (!fencing_peer_active(crm_get_peer(0, other->originator))) {
             crm_notice("Failing action '%s' targeting %s originating from "
                        "client %s@%s: Originator is dead " CRM_XS " id=%.8s",
                        other->action, other->target, other->client_name,
                        other->originator, other->id);
             crm_trace("%.8s not duplicate of %.8s: originator dead",
                       op->id, other->id);
             other->state = st_failed;
             continue;
         }
         if ((other->total_timeout > 0)
             && (now > (other->total_timeout + other->created))) {
             crm_trace("%.8s not duplicate of %.8s: old (%ld vs. %ld + %d)",
                       op->id, other->id, now, other->created,
                       other->total_timeout);
             continue;
         }
 
         /* There is another in-flight request to fence the same host
          * Piggyback on that instead.  If it fails, so do we.
          */
         other->duplicates = g_list_append(other->duplicates, op);
         if (other->total_timeout == 0) {
             other->total_timeout = op->total_timeout =
                 TIMEOUT_MULTIPLY_FACTOR * get_op_total_timeout(op, NULL);
             crm_trace("Best guess as to timeout used for %.8s: %d",
                       other->id, other->total_timeout);
         }
         crm_notice("Merging fencing action '%s' targeting %s originating from "
                    "client %s with identical request from %s@%s "
                    CRM_XS " original=%.8s duplicate=%.8s total_timeout=%ds",
                    op->action, op->target, op->client_name,
                    other->client_name, other->originator,
                    op->id, other->id, other->total_timeout);
         report_timeout_period(op, other->total_timeout);
         op->state = st_duplicate;
     }
 }
 
 static uint32_t fencing_active_peers(void)
 {
     uint32_t count = 0;
     crm_node_t *entry;
     GHashTableIter gIter;
 
     g_hash_table_iter_init(&gIter, crm_peer_cache);
     while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
         if(fencing_peer_active(entry)) {
             count++;
         }
     }
     return count;
 }
 
 /*!
  * \internal
  * \brief Process a manual confirmation of a pending fence action
  *
  * \param[in]     client  IPC client that sent confirmation
  * \param[in,out] msg     Request XML with manual confirmation
  *
  * \return Standard Pacemaker return code
  */
 int
 fenced_handle_manual_confirmation(const pcmk__client_t *client, xmlNode *msg)
 {
     remote_fencing_op_t *op = NULL;
     xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_ERR);
 
     CRM_CHECK(dev != NULL, return EPROTO);
 
     crm_notice("Received manual confirmation that %s has been fenced",
                pcmk__s(crm_element_value(dev, F_STONITH_TARGET),
                        "unknown target"));
     op = initiate_remote_stonith_op(client, msg, TRUE);
     if (op == NULL) {
         return EPROTO;
     }
     op->state = st_done;
     set_fencing_completed(op);
     op->delegate = strdup("a human");
 
     // For the fencer's purposes, the fencing operation is done
     pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
     finalize_op(op, msg, false);
 
     /* For the requester's purposes, the operation is still pending. The
      * actual result will be sent asynchronously via the operation's done_cb().
      */
     return EINPROGRESS;
 }
 
 /*!
  * \internal
  * \brief Create a new remote stonith operation
  *
  * \param[in] client   ID of local stonith client that initiated the operation
  * \param[in] request  The request from the client that started the operation
  * \param[in] peer     TRUE if this operation is owned by another stonith peer
  *                     (an operation owned by one peer is stored on all peers,
  *                     but only the owner executes it; all nodes get the results
  *                     once the owner finishes execution)
  */
 void *
 create_remote_stonith_op(const char *client, xmlNode *request, gboolean peer)
 {
     remote_fencing_op_t *op = NULL;
     xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_NEVER);
     int call_options = 0;
     const char *operation = NULL;
 
     init_stonith_remote_op_hash_table(&stonith_remote_op_list);
 
     /* If this operation is owned by another node, check to make
      * sure we haven't already created this operation. */
     if (peer && dev) {
         const char *op_id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);
 
         CRM_CHECK(op_id != NULL, return NULL);
 
         op = g_hash_table_lookup(stonith_remote_op_list, op_id);
         if (op) {
             crm_debug("Reusing existing remote fencing op %.8s for %s",
                       op_id, ((client == NULL)? "unknown client" : client));
             return op;
         }
     }
 
     op = calloc(1, sizeof(remote_fencing_op_t));
     CRM_ASSERT(op != NULL);
 
     crm_element_value_int(request, F_STONITH_TIMEOUT, &(op->base_timeout));
     // Value -1 means disable any static/random fencing delays
     crm_element_value_int(request, F_STONITH_DELAY, &(op->delay));
 
     if (peer && dev) {
         op->id = crm_element_value_copy(dev, F_STONITH_REMOTE_OP_ID);
     } else {
         op->id = crm_generate_uuid();
     }
 
     g_hash_table_replace(stonith_remote_op_list, op->id, op);
 
     op->state = st_query;
     op->replies_expected = fencing_active_peers();
     op->action = crm_element_value_copy(dev, F_STONITH_ACTION);
     op->originator = crm_element_value_copy(dev, F_STONITH_ORIGIN);
     op->delegate = crm_element_value_copy(dev, F_STONITH_DELEGATE); /* May not be set */
     op->created = time(NULL);
 
     if (op->originator == NULL) {
         /* Local or relayed request */
         op->originator = strdup(stonith_our_uname);
     }
 
     CRM_LOG_ASSERT(client != NULL);
     if (client) {
         op->client_id = strdup(client);
     }
 
 
     /* For a RELAY operation, set fenced on the client. */
     operation = crm_element_value(request, F_STONITH_OPERATION);
 
     if (pcmk__str_eq(operation, STONITH_OP_RELAY, pcmk__str_none)) {
         op->client_name = crm_strdup_printf("%s.%lu", crm_system_name,
                                          (unsigned long) getpid());
     } else {
         op->client_name = crm_element_value_copy(request, F_STONITH_CLIENTNAME);
     }
 
     op->target = crm_element_value_copy(dev, F_STONITH_TARGET);
     op->request = copy_xml(request);    /* TODO: Figure out how to avoid this */
     crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
     op->call_options = call_options;
 
     crm_element_value_int(request, F_STONITH_CALLID, &(op->client_callid));
 
     crm_trace("%s new fencing op %s ('%s' targeting %s for client %s, "
               "base timeout %d, %u %s expected)",
               (peer && dev)? "Recorded" : "Generated", op->id, op->action,
               op->target, op->client_name, op->base_timeout,
               op->replies_expected,
               pcmk__plural_alt(op->replies_expected, "reply", "replies"));
 
     if (op->call_options & st_opt_cs_nodeid) {
         int nodeid;
         crm_node_t *node;
 
         pcmk__scan_min_int(op->target, &nodeid, 0);
         node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY);
 
         /* Ensure the conversion only happens once */
         stonith__clear_call_options(op->call_options, op->id, st_opt_cs_nodeid);
 
         if (node && node->uname) {
             free(op->target);
             op->target = strdup(node->uname);
 
         } else {
             crm_warn("Could not expand nodeid '%s' into a host name", op->target);
         }
     }
 
     /* check to see if this is a duplicate operation of another in-flight operation */
     merge_duplicates(op);
 
     if (op->state != st_duplicate) {
         /* kick history readers */
         fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
     }
 
     /* safe to trim as long as that doesn't touch pending ops */
     stonith_fence_history_trim();
 
     return op;
 }
 
 /*!
  * \internal
  * \brief Create a peer fencing operation from a request, and initiate it
  *
  * \param[in] client     IPC client that made request (NULL to get from request)
  * \param[in] request    Request XML
  * \param[in] manual_ack Whether this is a manual action confirmation
  *
  * \return Newly created operation on success, otherwise NULL
  */
 remote_fencing_op_t *
 initiate_remote_stonith_op(const pcmk__client_t *client, xmlNode *request,
                            gboolean manual_ack)
 {
     int query_timeout = 0;
     xmlNode *query = NULL;
     const char *client_id = NULL;
     remote_fencing_op_t *op = NULL;
     const char *relay_op_id = NULL;
     const char *operation = NULL;
 
     if (client) {
         client_id = client->id;
     } else {
         client_id = crm_element_value(request, F_STONITH_CLIENTID);
     }
 
     CRM_LOG_ASSERT(client_id != NULL);
     op = create_remote_stonith_op(client_id, request, FALSE);
     op->owner = TRUE;
     if (manual_ack) {
         return op;
     }
 
     CRM_CHECK(op->action, return NULL);
 
     if (advance_topology_level(op, true) != pcmk_rc_ok) {
         op->state = st_failed;
     }
 
     switch (op->state) {
         case st_failed:
             // advance_topology_level() exhausted levels
             pcmk__set_result(&op->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
                              "All topology levels failed");
             crm_warn("Could not request peer fencing (%s) targeting %s "
                      CRM_XS " id=%.8s", op->action, op->target, op->id);
             finalize_op(op, NULL, false);
             return op;
 
         case st_duplicate:
             crm_info("Requesting peer fencing (%s) targeting %s (duplicate) "
                      CRM_XS " id=%.8s", op->action, op->target, op->id);
             return op;
 
         default:
             crm_notice("Requesting peer fencing (%s) targeting %s "
                        CRM_XS " id=%.8s state=%s base_timeout=%d",
                        op->action, op->target, op->id,
                        stonith_op_state_str(op->state), op->base_timeout);
     }
 
     query = stonith_create_op(op->client_callid, op->id, STONITH_OP_QUERY,
                               NULL, op->call_options);
 
     crm_xml_add(query, F_STONITH_REMOTE_OP_ID, op->id);
     crm_xml_add(query, F_STONITH_TARGET, op->target);
     crm_xml_add(query, F_STONITH_ACTION, op_requested_action(op));
     crm_xml_add(query, F_STONITH_ORIGIN, op->originator);
     crm_xml_add(query, F_STONITH_CLIENTID, op->client_id);
     crm_xml_add(query, F_STONITH_CLIENTNAME, op->client_name);
     crm_xml_add_int(query, F_STONITH_TIMEOUT, op->base_timeout);
 
     /* In case of RELAY operation, RELAY information is added to the query to delete the original operation of RELAY. */
     operation = crm_element_value(request, F_STONITH_OPERATION);
     if (pcmk__str_eq(operation, STONITH_OP_RELAY, pcmk__str_none)) {
         relay_op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID);
         if (relay_op_id) {
             crm_xml_add(query, F_STONITH_REMOTE_OP_ID_RELAY, relay_op_id);
         }
     }
 
     send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE);
     free_xml(query);
 
     query_timeout = op->base_timeout * TIMEOUT_MULTIPLY_FACTOR;
     op->query_timer = g_timeout_add((1000 * query_timeout), remote_op_query_timeout, op);
 
     return op;
 }
 
 enum find_best_peer_options {
     /*! Skip checking the target peer for capable fencing devices */
     FIND_PEER_SKIP_TARGET = 0x0001,
     /*! Only check the target peer for capable fencing devices */
     FIND_PEER_TARGET_ONLY = 0x0002,
     /*! Skip peers and devices that are not verified */
     FIND_PEER_VERIFIED_ONLY = 0x0004,
 };
 
 static peer_device_info_t *
 find_best_peer(const char *device, remote_fencing_op_t * op, enum find_best_peer_options options)
 {
     GList *iter = NULL;
     gboolean verified_devices_only = (options & FIND_PEER_VERIFIED_ONLY) ? TRUE : FALSE;
 
     if (!device && pcmk_is_set(op->call_options, st_opt_topology)) {
         return NULL;
     }
 
     for (iter = op->query_results; iter != NULL; iter = iter->next) {
         peer_device_info_t *peer = iter->data;
 
         crm_trace("Testing result from %s targeting %s with %d device%s: %d %x",
                   peer->host, op->target, peer->ndevices,
                   pcmk__plural_s(peer->ndevices), peer->tried, options);
         if ((options & FIND_PEER_SKIP_TARGET) && pcmk__str_eq(peer->host, op->target, pcmk__str_casei)) {
             continue;
         }
         if ((options & FIND_PEER_TARGET_ONLY) && !pcmk__str_eq(peer->host, op->target, pcmk__str_casei)) {
             continue;
         }
 
         if (pcmk_is_set(op->call_options, st_opt_topology)) {
 
             if (grab_peer_device(op, peer, device, verified_devices_only)) {
                 return peer;
             }
 
         } else if (!peer->tried
                    && count_peer_devices(op, peer, verified_devices_only,
                                          fenced_support_flag(op->action))) {
             /* No topology: Use the current best peer */
             crm_trace("Simple fencing");
             return peer;
         }
     }
 
     return NULL;
 }
 
 static peer_device_info_t *
 stonith_choose_peer(remote_fencing_op_t * op)
 {
     const char *device = NULL;
     peer_device_info_t *peer = NULL;
     uint32_t active = fencing_active_peers();
 
     do {
         if (op->devices) {
             device = op->devices->data;
             crm_trace("Checking for someone to fence (%s) %s using %s",
                       op->action, op->target, device);
         } else {
             crm_trace("Checking for someone to fence (%s) %s",
                       op->action, op->target);
         }
 
         /* Best choice is a peer other than the target with verified access */
         peer = find_best_peer(device, op, FIND_PEER_SKIP_TARGET|FIND_PEER_VERIFIED_ONLY);
         if (peer) {
             crm_trace("Found verified peer %s for %s", peer->host, device?device:"<any>");
             return peer;
         }
 
         if(op->query_timer != 0 && op->replies < QB_MIN(op->replies_expected, active)) {
             crm_trace("Waiting before looking for unverified devices to fence %s", op->target);
             return NULL;
         }
 
         /* If no other peer has verified access, next best is unverified access */
         peer = find_best_peer(device, op, FIND_PEER_SKIP_TARGET);
         if (peer) {
             crm_trace("Found best unverified peer %s", peer->host);
             return peer;
         }
 
         /* If no other peer can do it, last option is self-fencing
          * (which is never allowed for the "on" phase of a remapped reboot)
          */
         if (op->phase != st_phase_on) {
             peer = find_best_peer(device, op, FIND_PEER_TARGET_ONLY);
             if (peer) {
                 crm_trace("%s will fence itself", peer->host);
                 return peer;
             }
         }
 
         /* Try the next fencing level if there is one (unless we're in the "on"
          * phase of a remapped "reboot", because we ignore errors in that case)
          */
     } while ((op->phase != st_phase_on)
              && pcmk_is_set(op->call_options, st_opt_topology)
              && (advance_topology_level(op, false) == pcmk_rc_ok));
 
     if ((stonith_watchdog_timeout_ms > 0)
         && pcmk__is_fencing_action(op->action)
         && pcmk__str_eq(device, STONITH_WATCHDOG_ID, pcmk__str_none)
         && node_does_watchdog_fencing(op->target)) {
         crm_info("Couldn't contact watchdog-fencing target-node (%s)",
                  op->target);
         /* check_watchdog_fencing_and_wait will log additional info */
     } else {
         crm_notice("Couldn't find anyone to fence (%s) %s using %s",
                    op->action, op->target, (device? device : "any device"));
     }
     return NULL;
 }
 
 static int
 get_device_timeout(const remote_fencing_op_t *op,
-                   const peer_device_info_t *peer, const char *device)
+                   const peer_device_info_t *peer, const char *device,
+                   bool with_delay)
 {
     device_properties_t *props;
+    int delay = 0;
 
     if (!peer || !device) {
         return op->base_timeout;
     }
 
     props = g_hash_table_lookup(peer->devices, device);
     if (!props) {
         return op->base_timeout;
     }
 
+    // op->delay < 0 means disable any static/random fencing delays
+    if (with_delay && op->delay >= 0) {
+        // delay_base is eventually limited by delay_max
+        delay = (props->delay_max[op->phase] > 0 ?
+                 props->delay_max[op->phase] : props->delay_base[op->phase]);
+    }
+
     return (props->custom_action_timeout[op->phase]?
-           props->custom_action_timeout[op->phase] : op->base_timeout)
-           + props->delay_max[op->phase];
+            props->custom_action_timeout[op->phase] : op->base_timeout)
+           + delay;
 }
 
 struct timeout_data {
     const remote_fencing_op_t *op;
     const peer_device_info_t *peer;
     int total_timeout;
 };
 
 /*!
  * \internal
  * \brief Add timeout to a total if device has not been executed yet
  *
  * \param[in]     key        GHashTable key (device ID)
  * \param[in]     value      GHashTable value (device properties)
  * \param[in,out] user_data  Timeout data
  */
 static void
 add_device_timeout(gpointer key, gpointer value, gpointer user_data)
 {
     const char *device_id = key;
     device_properties_t *props = value;
     struct timeout_data *timeout = user_data;
 
     if (!props->executed[timeout->op->phase]
         && !props->disallowed[timeout->op->phase]) {
-        timeout->total_timeout += get_device_timeout(timeout->op,
-                                                     timeout->peer, device_id);
+        timeout->total_timeout += get_device_timeout(timeout->op, timeout->peer,
+                                                     device_id, true);
     }
 }
 
 static int
 get_peer_timeout(const remote_fencing_op_t *op, const peer_device_info_t *peer)
 {
     struct timeout_data timeout;
 
     timeout.op = op;
     timeout.peer = peer;
     timeout.total_timeout = 0;
 
     g_hash_table_foreach(peer->devices, add_device_timeout, &timeout);
 
     return (timeout.total_timeout? timeout.total_timeout : op->base_timeout);
 }
 
 static int
 get_op_total_timeout(const remote_fencing_op_t *op,
                      const peer_device_info_t *chosen_peer)
 {
     int total_timeout = 0;
     stonith_topology_t *tp = find_topology_for_host(op->target);
 
     if (pcmk_is_set(op->call_options, st_opt_topology) && tp) {
         int i;
         GList *device_list = NULL;
         GList *iter = NULL;
         GList *auto_list = NULL;
 
         if (pcmk__str_eq(op->action, "on", pcmk__str_none)
             && (op->automatic_list != NULL)) {
             auto_list = g_list_copy(op->automatic_list);
         }
 
         /* Yep, this looks scary, nested loops all over the place.
          * Here is what is going on.
          * Loop1: Iterate through fencing levels.
          * Loop2: If a fencing level has devices, loop through each device
          * Loop3: For each device in a fencing level, see what peer owns it
          *        and what that peer has reported the timeout is for the device.
          */
         for (i = 0; i < ST_LEVEL_MAX; i++) {
             if (!tp->levels[i]) {
                 continue;
             }
             for (device_list = tp->levels[i]; device_list; device_list = device_list->next) {
                 /* in case of watchdog-device we add the timeout to the budget
                    regardless of if we got a reply or not
                  */
                 if ((stonith_watchdog_timeout_ms > 0)
                     && pcmk__is_fencing_action(op->action)
                     && pcmk__str_eq(device_list->data, STONITH_WATCHDOG_ID,
                                     pcmk__str_none)
                     && node_does_watchdog_fencing(op->target)) {
                     total_timeout += stonith_watchdog_timeout_ms / 1000;
                     continue;
                 }
 
                 for (iter = op->query_results; iter != NULL; iter = iter->next) {
                     const peer_device_info_t *peer = iter->data;
 
                     if (auto_list) {
                         GList *match = g_list_find_custom(auto_list, device_list->data,
                                         sort_strings);
                         if (match) {
                             auto_list = g_list_remove(auto_list, match->data);
                         }
                     }
 
                     if (find_peer_device(op, peer, device_list->data,
                                          fenced_support_flag(op->action))) {
                         total_timeout += get_device_timeout(op, peer,
-                                                            device_list->data);
+                                                            device_list->data,
+                                                            true);
                         break;
                     }
                 }               /* End Loop3: match device with peer that owns device, find device's timeout period */
             }                   /* End Loop2: iterate through devices at a specific level */
         }                       /*End Loop1: iterate through fencing levels */
 
         //Add only exists automatic_list device timeout
         if (auto_list) {
             for (iter = auto_list; iter != NULL; iter = iter->next) {
                 GList *iter2 = NULL;
 
                 for (iter2 = op->query_results; iter2 != NULL; iter = iter2->next) {
                     peer_device_info_t *peer = iter2->data;
                     if (find_peer_device(op, peer, iter->data, st_device_supports_on)) {
-                        total_timeout += get_device_timeout(op, peer, iter->data);
+                        total_timeout += get_device_timeout(op, peer,
+                                                            iter->data, true);
                         break;
                     }
                 }
             }
         }
 
         g_list_free(auto_list);
 
     } else if (chosen_peer) {
         total_timeout = get_peer_timeout(op, chosen_peer);
     } else {
         total_timeout = op->base_timeout;
     }
 
-    return total_timeout ? total_timeout : op->base_timeout;
+    /* Take any requested fencing delay into account to prevent it from eating
+     * up the total timeout.
+     */
+    return ((total_timeout ? total_timeout : op->base_timeout)
+            + (op->delay > 0 ? op->delay : 0));
 }
 
 static void
 report_timeout_period(remote_fencing_op_t * op, int op_timeout)
 {
     GList *iter = NULL;
     xmlNode *update = NULL;
     const char *client_node = NULL;
     const char *client_id = NULL;
     const char *call_id = NULL;
 
     if (op->call_options & st_opt_sync_call) {
         /* There is no reason to report the timeout for a synchronous call. It
          * is impossible to use the reported timeout to do anything when the client
          * is blocking for the response.  This update is only important for
          * async calls that require a callback to report the results in. */
         return;
     } else if (!op->request) {
         return;
     }
 
     crm_trace("Reporting timeout for %s (id=%.8s)", op->client_name, op->id);
     client_node = crm_element_value(op->request, F_STONITH_CLIENTNODE);
     call_id = crm_element_value(op->request, F_STONITH_CALLID);
     client_id = crm_element_value(op->request, F_STONITH_CLIENTID);
     if (!client_node || !call_id || !client_id) {
         return;
     }
 
     if (pcmk__str_eq(client_node, stonith_our_uname, pcmk__str_casei)) {
         // Client is connected to this node, so send update directly to them
         do_stonith_async_timeout_update(client_id, call_id, op_timeout);
         return;
     }
 
     /* The client is connected to another node, relay this update to them */
     update = stonith_create_op(op->client_callid, op->id, STONITH_OP_TIMEOUT_UPDATE, NULL, 0);
     crm_xml_add(update, F_STONITH_REMOTE_OP_ID, op->id);
     crm_xml_add(update, F_STONITH_CLIENTID, client_id);
     crm_xml_add(update, F_STONITH_CALLID, call_id);
     crm_xml_add_int(update, F_STONITH_TIMEOUT, op_timeout);
 
     send_cluster_message(crm_get_peer(0, client_node), crm_msg_stonith_ng, update, FALSE);
 
     free_xml(update);
 
     for (iter = op->duplicates; iter != NULL; iter = iter->next) {
         remote_fencing_op_t *dup = iter->data;
 
         crm_trace("Reporting timeout for duplicate %.8s to client %s",
                   dup->id, dup->client_name);
         report_timeout_period(iter->data, op_timeout);
     }
 }
 
 /*!
  * \internal
  * \brief Advance an operation to the next device in its topology
  *
  * \param[in,out] op      Fencer operation to advance
  * \param[in]     device  ID of device that just completed
  * \param[in,out] msg     If not NULL, XML reply of last delegated operation
  */
 static void
 advance_topology_device_in_level(remote_fencing_op_t *op, const char *device,
                                  xmlNode *msg)
 {
     /* Advance to the next device at this topology level, if any */
     if (op->devices) {
         op->devices = op->devices->next;
     }
 
     /* Handle automatic unfencing if an "on" action was requested */
     if ((op->phase == st_phase_requested)
         && pcmk__str_eq(op->action, "on", pcmk__str_none)) {
         /* If the device we just executed was required, it's not anymore */
         remove_required_device(op, device);
 
         /* If there are no more devices at this topology level, run through any
          * remaining devices with automatic unfencing
          */
         if (op->devices == NULL) {
             op->devices = op->automatic_list;
         }
     }
 
     if ((op->devices == NULL) && (op->phase == st_phase_off)) {
         /* We're done with this level and with required devices, but we had
          * remapped "reboot" to "off", so start over with "on". If any devices
          * need to be turned back on, op->devices will be non-NULL after this.
          */
         op_phase_on(op);
     }
 
     // This function is only called if the previous device succeeded
     pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
 
     if (op->devices) {
         /* Necessary devices remain, so execute the next one */
         crm_trace("Next targeting %s on behalf of %s@%s",
                   op->target, op->client_name, op->originator);
 
         // The requested delay has been applied for the first device
         if (op->delay > 0) {
             op->delay = 0;
         }
 
         request_peer_fencing(op, NULL);
     } else {
         /* We're done with all devices and phases, so finalize operation */
         crm_trace("Marking complex fencing op targeting %s as complete",
                   op->target);
         op->state = st_done;
         finalize_op(op, msg, false);
     }
 }
 
 static gboolean
 check_watchdog_fencing_and_wait(remote_fencing_op_t * op)
 {
     if (node_does_watchdog_fencing(op->target)) {
 
         crm_notice("Waiting %lds for %s to self-fence (%s) for "
                    "client %s " CRM_XS " id=%.8s",
                    (stonith_watchdog_timeout_ms / 1000),
                    op->target, op->action, op->client_name, op->id);
 
         if (op->op_timer_one) {
             g_source_remove(op->op_timer_one);
         }
         op->op_timer_one = g_timeout_add(stonith_watchdog_timeout_ms,
                                          remote_op_watchdog_done, op);
         return TRUE;
     } else {
         crm_debug("Skipping fallback to watchdog-fencing as %s is "
                  "not in host-list", op->target);
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Ask a peer to execute a fencing operation
  *
  * \param[in,out] op      Fencing operation to be executed
  * \param[in,out] peer    If NULL or topology is in use, choose best peer to
  *                        execute the fencing, otherwise use this peer
  */
 static void
 request_peer_fencing(remote_fencing_op_t *op, peer_device_info_t *peer)
 {
     const char *device = NULL;
     int timeout;
 
     CRM_CHECK(op != NULL, return);
 
     crm_trace("Action %.8s targeting %s for %s is %s",
               op->id, op->target, op->client_name,
               stonith_op_state_str(op->state));
 
     if ((op->phase == st_phase_on) && (op->devices != NULL)) {
         /* We are in the "on" phase of a remapped topology reboot. If this
          * device has pcmk_reboot_action="off", or doesn't support the "on"
          * action, skip it.
          *
          * We can't check device properties at this point because we haven't
          * chosen a peer for this stage yet. Instead, we check the local node's
          * knowledge about the device. If different versions of the fence agent
          * are installed on different nodes, there's a chance this could be
          * mistaken, but the worst that could happen is we don't try turning the
          * node back on when we should.
          */
         device = op->devices->data;
         if (pcmk__str_eq(fenced_device_reboot_action(device), "off",
                          pcmk__str_none)) {
             crm_info("Not turning %s back on using %s because the device is "
                      "configured to stay off (pcmk_reboot_action='off')",
                      op->target, device);
             advance_topology_device_in_level(op, device, NULL);
             return;
         }
         if (!fenced_device_supports_on(device)) {
             crm_info("Not turning %s back on using %s because the agent "
                      "doesn't support 'on'", op->target, device);
             advance_topology_device_in_level(op, device, NULL);
             return;
         }
     }
 
     timeout = op->base_timeout;
     if ((peer == NULL) && !pcmk_is_set(op->call_options, st_opt_topology)) {
         peer = stonith_choose_peer(op);
     }
 
     if (!op->op_timer_total) {
         op->total_timeout = TIMEOUT_MULTIPLY_FACTOR * get_op_total_timeout(op, peer);
         op->op_timer_total = g_timeout_add(1000 * op->total_timeout, remote_op_timeout, op);
         report_timeout_period(op, op->total_timeout);
         crm_info("Total timeout set to %d for peer's fencing targeting %s for %s"
                  CRM_XS "id=%.8s",
                  op->total_timeout, op->target, op->client_name, op->id);
     }
 
     if (pcmk_is_set(op->call_options, st_opt_topology) && op->devices) {
         /* Ignore the caller's peer preference if topology is in use, because
          * that peer might not have access to the required device. With
          * topology, stonith_choose_peer() removes the device from further
          * consideration, so the timeout must be calculated beforehand.
          *
          * @TODO Basing the total timeout on the caller's preferred peer (above)
          *       is less than ideal.
          */
         peer = stonith_choose_peer(op);
 
         device = op->devices->data;
-        timeout = get_device_timeout(op, peer, device);
+        /* Fencing timeout sent to peer takes no delay into account.
+         * The peer will add a dedicated timer for any delay upon
+         * schedule_stonith_command().
+         */
+        timeout = get_device_timeout(op, peer, device, false);
     }
 
     if (peer) {
-        int timeout_one = 0;
+       /* Take any requested fencing delay into account to prevent it from eating
+        * up the timeout.
+        */
+        int timeout_one = (op->delay > 0 ?
+                           TIMEOUT_MULTIPLY_FACTOR * op->delay : 0);
         xmlNode *remote_op = stonith_create_op(op->client_callid, op->id, STONITH_OP_FENCE, NULL, 0);
 
         crm_xml_add(remote_op, F_STONITH_REMOTE_OP_ID, op->id);
         crm_xml_add(remote_op, F_STONITH_TARGET, op->target);
         crm_xml_add(remote_op, F_STONITH_ACTION, op->action);
         crm_xml_add(remote_op, F_STONITH_ORIGIN, op->originator);
         crm_xml_add(remote_op, F_STONITH_CLIENTID, op->client_id);
         crm_xml_add(remote_op, F_STONITH_CLIENTNAME, op->client_name);
         crm_xml_add_int(remote_op, F_STONITH_TIMEOUT, timeout);
         crm_xml_add_int(remote_op, F_STONITH_CALLOPTS, op->call_options);
         crm_xml_add_int(remote_op, F_STONITH_DELAY, op->delay);
 
         if (device) {
-            timeout_one = TIMEOUT_MULTIPLY_FACTOR *
-                          get_device_timeout(op, peer, device);
+            timeout_one += TIMEOUT_MULTIPLY_FACTOR *
+                           get_device_timeout(op, peer, device, true);
             crm_notice("Requesting that %s perform '%s' action targeting %s "
                        "using %s " CRM_XS " for client %s (%ds)",
                        peer->host, op->action, op->target, device,
                        op->client_name, timeout_one);
             crm_xml_add(remote_op, F_STONITH_DEVICE, device);
 
         } else {
-            timeout_one = TIMEOUT_MULTIPLY_FACTOR * get_peer_timeout(op, peer);
+            timeout_one += TIMEOUT_MULTIPLY_FACTOR * get_peer_timeout(op, peer);
             crm_notice("Requesting that %s perform '%s' action targeting %s "
                        CRM_XS " for client %s (%ds, %lds)",
                        peer->host, op->action, op->target, op->client_name,
                        timeout_one, stonith_watchdog_timeout_ms);
         }
 
         op->state = st_exec;
         if (op->op_timer_one) {
             g_source_remove(op->op_timer_one);
             op->op_timer_one = 0;
         }
 
         if (!((stonith_watchdog_timeout_ms > 0)
               && (pcmk__str_eq(device, STONITH_WATCHDOG_ID, pcmk__str_none)
                   || (pcmk__str_eq(peer->host, op->target, pcmk__str_casei)
                       && pcmk__is_fencing_action(op->action)))
               && check_watchdog_fencing_and_wait(op))) {
 
             /* Some thoughts about self-fencing cases reaching this point:
                - Actually check in check_watchdog_fencing_and_wait
                  shouldn't fail if STONITH_WATCHDOG_ID is
                  chosen as fencing-device and it being present implies
                  watchdog-fencing is enabled anyway
                - If watchdog-fencing is disabled either in general or for
                  a specific target - detected in check_watchdog_fencing_and_wait -
                  for some other kind of self-fencing we can't expect
                  a success answer but timeout is fine if the node doesn't
                  come back in between
                - Delicate might be the case where we have watchdog-fencing
                  enabled for a node but the watchdog-fencing-device isn't
                  explicitly chosen for suicide. Local pe-execution in sbd
                  may detect the node as unclean and lead to timely suicide.
                  Otherwise the selection of stonith-watchdog-timeout at
                  least is questionable.
              */
 
             /* coming here we're not waiting for watchdog timeout -
                thus engage timer with timout evaluated before */
             op->op_timer_one = g_timeout_add((1000 * timeout_one), remote_op_timeout_one, op);
         }
 
         send_cluster_message(crm_get_peer(0, peer->host), crm_msg_stonith_ng, remote_op, FALSE);
         peer->tried = TRUE;
         free_xml(remote_op);
         return;
 
     } else if (op->phase == st_phase_on) {
         /* A remapped "on" cannot be executed, but the node was already
          * turned off successfully, so ignore the error and continue.
          */
         crm_warn("Ignoring %s 'on' failure (no capable peers) targeting %s "
                  "after successful 'off'", device, op->target);
         advance_topology_device_in_level(op, device, NULL);
         return;
 
     } else if (op->owner == FALSE) {
         crm_err("Fencing (%s) targeting %s for client %s is not ours to control",
                 op->action, op->target, op->client_name);
 
     } else if (op->query_timer == 0) {
         /* We've exhausted all available peers */
         crm_info("No remaining peers capable of fencing (%s) %s for client %s "
                  CRM_XS " state=%s", op->action, op->target, op->client_name,
                  stonith_op_state_str(op->state));
         CRM_CHECK(op->state < st_done, return);
         finalize_timed_out_op(op, "All nodes failed, or are unable, to "
                                   "fence target");
 
     } else if(op->replies >= op->replies_expected || op->replies >= fencing_active_peers()) {
         /* if the operation never left the query state,
          * but we have all the expected replies, then no devices
          * are available to execute the fencing operation. */
 
         if(stonith_watchdog_timeout_ms > 0 && pcmk__str_eq(device,
            STONITH_WATCHDOG_ID, pcmk__str_null_matches)) {
             if (check_watchdog_fencing_and_wait(op)) {
                 return;
             }
         }
 
         if (op->state == st_query) {
             crm_info("No peers (out of %d) have devices capable of fencing "
                      "(%s) %s for client %s " CRM_XS " state=%s",
                      op->replies, op->action, op->target, op->client_name,
                      stonith_op_state_str(op->state));
 
             pcmk__reset_result(&op->result);
             pcmk__set_result(&op->result, CRM_EX_ERROR,
                              PCMK_EXEC_NO_FENCE_DEVICE, NULL);
         } else {
             if (pcmk_is_set(op->call_options, st_opt_topology)) {
                 pcmk__reset_result(&op->result);
                 pcmk__set_result(&op->result, CRM_EX_ERROR,
                                  PCMK_EXEC_NO_FENCE_DEVICE, NULL);
             }
             /* ... else use existing result from previous failed attempt
              * (topology is not in use, and no devices remain to be attempted).
              * Overwriting the result with PCMK_EXEC_NO_FENCE_DEVICE would
              * prevent finalize_op() from setting the correct delegate if
              * needed.
              */
 
             crm_info("No peers (out of %d) are capable of fencing (%s) %s "
                      "for client %s " CRM_XS " state=%s",
                      op->replies, op->action, op->target, op->client_name,
                      stonith_op_state_str(op->state));
         }
 
         op->state = st_failed;
         finalize_op(op, NULL, false);
 
     } else {
         crm_info("Waiting for additional peers capable of fencing (%s) %s%s%s "
                  "for client %s " CRM_XS " id=%.8s",
                  op->action, op->target, (device? " using " : ""),
                  (device? device : ""), op->client_name, op->id);
     }
 }
 
 /*!
  * \internal
  * \brief Comparison function for sorting query results
  *
  * \param[in] a  GList item to compare
  * \param[in] b  GList item to compare
  *
  * \return Per the glib documentation, "a negative integer if the first value
  *         comes before the second, 0 if they are equal, or a positive integer
  *         if the first value comes after the second."
  */
 static gint
 sort_peers(gconstpointer a, gconstpointer b)
 {
     const peer_device_info_t *peer_a = a;
     const peer_device_info_t *peer_b = b;
 
     return (peer_b->ndevices - peer_a->ndevices);
 }
 
 /*!
  * \internal
  * \brief Determine if all the devices in the topology are found or not
  *
  * \param[in] op  Fencing operation with topology to check
  */
 static gboolean
 all_topology_devices_found(const remote_fencing_op_t *op)
 {
     GList *device = NULL;
     GList *iter = NULL;
     device_properties_t *match = NULL;
     stonith_topology_t *tp = NULL;
     gboolean skip_target = FALSE;
     int i;
 
     tp = find_topology_for_host(op->target);
     if (!tp) {
         return FALSE;
     }
     if (pcmk__is_fencing_action(op->action)) {
         /* Don't count the devices on the target node if we are killing
          * the target node. */
         skip_target = TRUE;
     }
 
     for (i = 0; i < ST_LEVEL_MAX; i++) {
         for (device = tp->levels[i]; device; device = device->next) {
             match = NULL;
             for (iter = op->query_results; iter && !match; iter = iter->next) {
                 peer_device_info_t *peer = iter->data;
 
                 if (skip_target && pcmk__str_eq(peer->host, op->target, pcmk__str_casei)) {
                     continue;
                 }
                 match = find_peer_device(op, peer, device->data, st_device_supports_none);
             }
             if (!match) {
                 return FALSE;
             }
         }
     }
 
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Parse action-specific device properties from XML
  *
  * \param[in]     xml     XML element containing the properties
  * \param[in]     peer    Name of peer that sent XML (for logs)
  * \param[in]     device  Device ID (for logs)
  * \param[in]     action  Action the properties relate to (for logs)
  * \param[in,out] op      Fencing operation that properties are being parsed for
  * \param[in]     phase   Phase the properties relate to
  * \param[in,out] props   Device properties to update
  */
 static void
 parse_action_specific(const xmlNode *xml, const char *peer, const char *device,
                       const char *action, remote_fencing_op_t *op,
                       enum st_remap_phase phase, device_properties_t *props)
 {
     props->custom_action_timeout[phase] = 0;
     crm_element_value_int(xml, F_STONITH_ACTION_TIMEOUT,
                           &props->custom_action_timeout[phase]);
     if (props->custom_action_timeout[phase]) {
         crm_trace("Peer %s with device %s returned %s action timeout %d",
                   peer, device, action, props->custom_action_timeout[phase]);
     }
 
     props->delay_max[phase] = 0;
     crm_element_value_int(xml, F_STONITH_DELAY_MAX, &props->delay_max[phase]);
     if (props->delay_max[phase]) {
         crm_trace("Peer %s with device %s returned maximum of random delay %d for %s",
                   peer, device, props->delay_max[phase], action);
     }
 
     props->delay_base[phase] = 0;
     crm_element_value_int(xml, F_STONITH_DELAY_BASE, &props->delay_base[phase]);
     if (props->delay_base[phase]) {
         crm_trace("Peer %s with device %s returned base delay %d for %s",
                   peer, device, props->delay_base[phase], action);
     }
 
     /* Handle devices with automatic unfencing */
     if (pcmk__str_eq(action, "on", pcmk__str_none)) {
         int required = 0;
 
         crm_element_value_int(xml, F_STONITH_DEVICE_REQUIRED, &required);
         if (required) {
             crm_trace("Peer %s requires device %s to execute for action %s",
                       peer, device, action);
             add_required_device(op, device);
         }
     }
 
     /* If a reboot is remapped to off+on, it's possible that a node is allowed
      * to perform one action but not another.
      */
     if (pcmk__xe_attr_is_true(xml, F_STONITH_ACTION_DISALLOWED)) {
         props->disallowed[phase] = TRUE;
         crm_trace("Peer %s is disallowed from executing %s for device %s",
                   peer, action, device);
     }
 }
 
 /*!
  * \internal
  * \brief Parse one device's properties from peer's XML query reply
  *
  * \param[in]     xml       XML node containing device properties
  * \param[in,out] op        Operation that query and reply relate to
  * \param[in,out] peer      Peer's device information
  * \param[in]     device    ID of device being parsed
  */
 static void
 add_device_properties(const xmlNode *xml, remote_fencing_op_t *op,
                       peer_device_info_t *peer, const char *device)
 {
     xmlNode *child;
     int verified = 0;
     device_properties_t *props = calloc(1, sizeof(device_properties_t));
     int flags = st_device_supports_on; /* Old nodes that don't set the flag assume they support the on action */
 
     /* Add a new entry to this peer's devices list */
     CRM_ASSERT(props != NULL);
     g_hash_table_insert(peer->devices, strdup(device), props);
 
     /* Peers with verified (monitored) access will be preferred */
     crm_element_value_int(xml, F_STONITH_DEVICE_VERIFIED, &verified);
     if (verified) {
         crm_trace("Peer %s has confirmed a verified device %s",
                   peer->host, device);
         props->verified = TRUE;
     }
 
     crm_element_value_int(xml, F_STONITH_DEVICE_SUPPORT_FLAGS, &flags);
     props->device_support_flags = flags;
 
     /* Parse action-specific device properties */
     parse_action_specific(xml, peer->host, device, op_requested_action(op),
                           op, st_phase_requested, props);
     for (child = pcmk__xml_first_child(xml); child != NULL;
          child = pcmk__xml_next(child)) {
         /* Replies for "reboot" operations will include the action-specific
          * values for "off" and "on" in child elements, just in case the reboot
          * winds up getting remapped.
          */
         if (pcmk__str_eq(ID(child), "off", pcmk__str_none)) {
             parse_action_specific(child, peer->host, device, "off",
                                   op, st_phase_off, props);
         } else if (pcmk__str_eq(ID(child), "on", pcmk__str_none)) {
             parse_action_specific(child, peer->host, device, "on",
                                   op, st_phase_on, props);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Parse a peer's XML query reply and add it to operation's results
  *
  * \param[in,out] op        Operation that query and reply relate to
  * \param[in]     host      Name of peer that sent this reply
  * \param[in]     ndevices  Number of devices expected in reply
  * \param[in]     xml       XML node containing device list
  *
  * \return Newly allocated result structure with parsed reply
  */
 static peer_device_info_t *
 add_result(remote_fencing_op_t *op, const char *host, int ndevices,
            const xmlNode *xml)
 {
     peer_device_info_t *peer = calloc(1, sizeof(peer_device_info_t));
     xmlNode *child;
 
     // cppcheck seems not to understand the abort logic in CRM_CHECK
     // cppcheck-suppress memleak
     CRM_CHECK(peer != NULL, return NULL);
     peer->host = strdup(host);
     peer->devices = pcmk__strkey_table(free, free);
 
     /* Each child element describes one capable device available to the peer */
     for (child = pcmk__xml_first_child(xml); child != NULL;
          child = pcmk__xml_next(child)) {
         const char *device = ID(child);
 
         if (device) {
             add_device_properties(child, op, peer, device);
         }
     }
 
     peer->ndevices = g_hash_table_size(peer->devices);
     CRM_CHECK(ndevices == peer->ndevices,
               crm_err("Query claimed to have %d device%s but %d found",
                       ndevices, pcmk__plural_s(ndevices), peer->ndevices));
 
     op->query_results = g_list_insert_sorted(op->query_results, peer, sort_peers);
     return peer;
 }
 
 /*!
  * \internal
  * \brief Handle a peer's reply to our fencing query
  *
  * Parse a query result from XML and store it in the remote operation
  * table, and when enough replies have been received, issue a fencing request.
  *
  * \param[in] msg  XML reply received
  *
  * \return pcmk_ok on success, -errno on error
  *
  * \note See initiate_remote_stonith_op() for how the XML query was initially
  *       formed, and stonith_query() for how the peer formed its XML reply.
  */
 int
 process_remote_stonith_query(xmlNode *msg)
 {
     int ndevices = 0;
     gboolean host_is_target = FALSE;
     gboolean have_all_replies = FALSE;
     const char *id = NULL;
     const char *host = NULL;
     remote_fencing_op_t *op = NULL;
     peer_device_info_t *peer = NULL;
     uint32_t replies_expected;
     xmlNode *dev = get_xpath_object("//@" F_STONITH_REMOTE_OP_ID, msg, LOG_ERR);
 
     CRM_CHECK(dev != NULL, return -EPROTO);
 
     id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);
     CRM_CHECK(id != NULL, return -EPROTO);
 
     dev = get_xpath_object("//@" F_STONITH_AVAILABLE_DEVICES, msg, LOG_ERR);
     CRM_CHECK(dev != NULL, return -EPROTO);
     crm_element_value_int(dev, F_STONITH_AVAILABLE_DEVICES, &ndevices);
 
     op = g_hash_table_lookup(stonith_remote_op_list, id);
     if (op == NULL) {
         crm_debug("Received query reply for unknown or expired operation %s",
                   id);
         return -EOPNOTSUPP;
     }
 
     replies_expected = fencing_active_peers();
     if (op->replies_expected < replies_expected) {
         replies_expected = op->replies_expected;
     }
     if ((++op->replies >= replies_expected) && (op->state == st_query)) {
         have_all_replies = TRUE;
     }
     host = crm_element_value(msg, F_ORIG);
     host_is_target = pcmk__str_eq(host, op->target, pcmk__str_casei);
 
     crm_info("Query result %d of %d from %s for %s/%s (%d device%s) %s",
              op->replies, replies_expected, host,
              op->target, op->action, ndevices, pcmk__plural_s(ndevices), id);
     if (ndevices > 0) {
         peer = add_result(op, host, ndevices, dev);
     }
 
     pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
 
     if (pcmk_is_set(op->call_options, st_opt_topology)) {
         /* If we start the fencing before all the topology results are in,
          * it is possible fencing levels will be skipped because of the missing
          * query results. */
         if (op->state == st_query && all_topology_devices_found(op)) {
             /* All the query results are in for the topology, start the fencing ops. */
             crm_trace("All topology devices found");
             request_peer_fencing(op, peer);
 
         } else if (have_all_replies) {
             crm_info("All topology query replies have arrived, continuing (%d expected/%d received) ",
                      replies_expected, op->replies);
             request_peer_fencing(op, NULL);
         }
 
     } else if (op->state == st_query) {
         int nverified = count_peer_devices(op, peer, TRUE,
                                            fenced_support_flag(op->action));
 
         /* We have a result for a non-topology fencing op that looks promising,
          * go ahead and start fencing before query timeout */
         if ((peer != NULL) && !host_is_target && nverified) {
             /* we have a verified device living on a peer that is not the target */
             crm_trace("Found %d verified device%s",
                       nverified, pcmk__plural_s(nverified));
             request_peer_fencing(op, peer);
 
         } else if (have_all_replies) {
             crm_info("All query replies have arrived, continuing (%d expected/%d received) ",
                      replies_expected, op->replies);
             request_peer_fencing(op, NULL);
 
         } else {
             crm_trace("Waiting for more peer results before launching fencing operation");
         }
 
     } else if ((peer != NULL) && (op->state == st_done)) {
         crm_info("Discarding query result from %s (%d device%s): "
                  "Operation is %s", peer->host,
                  peer->ndevices, pcmk__plural_s(peer->ndevices),
                  stonith_op_state_str(op->state));
     }
 
     return pcmk_ok;
 }
 
 /*!
  * \internal
  * \brief Handle a peer's reply to a fencing request
  *
  * Parse a fencing reply from XML, and either finalize the operation
  * or attempt another device as appropriate.
  *
  * \param[in] msg  XML reply received
  */
 void
 fenced_process_fencing_reply(xmlNode *msg)
 {
     const char *id = NULL;
     const char *device = NULL;
     remote_fencing_op_t *op = NULL;
     xmlNode *dev = get_xpath_object("//@" F_STONITH_REMOTE_OP_ID, msg, LOG_ERR);
     pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
 
     CRM_CHECK(dev != NULL, return);
 
     id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);
     CRM_CHECK(id != NULL, return);
 
     dev = stonith__find_xe_with_result(msg);
     CRM_CHECK(dev != NULL, return);
 
     stonith__xe_get_result(dev, &result);
 
     device = crm_element_value(dev, F_STONITH_DEVICE);
 
     if (stonith_remote_op_list) {
         op = g_hash_table_lookup(stonith_remote_op_list, id);
     }
 
     if ((op == NULL) && pcmk__result_ok(&result)) {
         /* Record successful fencing operations */
         const char *client_id = crm_element_value(dev, F_STONITH_CLIENTID);
 
         op = create_remote_stonith_op(client_id, dev, TRUE);
     }
 
     if (op == NULL) {
         /* Could be for an event that began before we started */
         /* TODO: Record the op for later querying */
         crm_info("Received peer result of unknown or expired operation %s", id);
         pcmk__reset_result(&result);
         return;
     }
 
     pcmk__reset_result(&op->result);
     op->result = result; // The operation takes ownership of the result
 
     if (op->devices && device && !pcmk__str_eq(op->devices->data, device, pcmk__str_casei)) {
         crm_err("Received outdated reply for device %s (instead of %s) to "
                 "fence (%s) %s. Operation already timed out at peer level.",
                 device, (const char *) op->devices->data, op->action, op->target);
         return;
     }
 
     if (pcmk__str_eq(crm_element_value(msg, F_SUBTYPE), "broadcast", pcmk__str_casei)) {
         if (pcmk__result_ok(&op->result)) {
             op->state = st_done;
         } else {
             op->state = st_failed;
         }
         finalize_op(op, msg, false);
         return;
 
     } else if (!pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) {
         /* If this isn't a remote level broadcast, and we are not the
          * originator of the operation, we should not be receiving this msg. */
         crm_err("Received non-broadcast fencing result for operation %.8s "
                 "we do not own (device %s targeting %s)",
                 op->id, device, op->target);
         return;
     }
 
     if (pcmk_is_set(op->call_options, st_opt_topology)) {
         const char *device = NULL;
         const char *reason = op->result.exit_reason;
 
         /* We own the op, and it is complete. broadcast the result to all nodes
          * and notify our local clients. */
         if (op->state == st_done) {
             finalize_op(op, msg, false);
             return;
         }
 
         device = crm_element_value(msg, F_STONITH_DEVICE);
 
         if ((op->phase == 2) && !pcmk__result_ok(&op->result)) {
             /* A remapped "on" failed, but the node was already turned off
              * successfully, so ignore the error and continue.
              */
             crm_warn("Ignoring %s 'on' failure (%s%s%s) targeting %s "
                      "after successful 'off'",
                      device, pcmk_exec_status_str(op->result.execution_status),
                      (reason == NULL)? "" : ": ",
                      (reason == NULL)? "" : reason,
                      op->target);
             pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
         } else {
             crm_notice("Action '%s' targeting %s%s%s on behalf of %s@%s: "
                        "%s%s%s%s",
                        op->action, op->target,
                        ((device == NULL)? "" : " using "),
                        ((device == NULL)? "" : device),
                        op->client_name,
                        op->originator,
                        pcmk_exec_status_str(op->result.execution_status),
                        (reason == NULL)? "" : " (",
                        (reason == NULL)? "" : reason,
                        (reason == NULL)? "" : ")");
         }
 
         if (pcmk__result_ok(&op->result)) {
             /* An operation completed successfully. Try another device if
              * necessary, otherwise mark the operation as done. */
             advance_topology_device_in_level(op, device, msg);
             return;
         } else {
             /* This device failed, time to try another topology level. If no other
              * levels are available, mark this operation as failed and report results. */
             if (advance_topology_level(op, false) != pcmk_rc_ok) {
                 op->state = st_failed;
                 finalize_op(op, msg, false);
                 return;
             }
         }
 
     } else if (pcmk__result_ok(&op->result) && (op->devices == NULL)) {
         op->state = st_done;
         finalize_op(op, msg, false);
         return;
 
     } else if ((op->result.execution_status == PCMK_EXEC_TIMEOUT)
                && (op->devices == NULL)) {
         /* If the operation timed out don't bother retrying other peers. */
         op->state = st_failed;
         finalize_op(op, msg, false);
         return;
 
     } else {
         /* fall-through and attempt other fencing action using another peer */
     }
 
     /* Retry on failure */
     crm_trace("Next for %s on behalf of %s@%s (result was: %s)",
               op->target, op->originator, op->client_name,
               pcmk_exec_status_str(op->result.execution_status));
     request_peer_fencing(op, NULL);
 }
 
 gboolean
 stonith_check_fence_tolerance(int tolerance, const char *target, const char *action)
 {
     GHashTableIter iter;
     time_t now = time(NULL);
     remote_fencing_op_t *rop = NULL;
 
     if (tolerance <= 0 || !stonith_remote_op_list || target == NULL ||
         action == NULL) {
         return FALSE;
     }
 
     g_hash_table_iter_init(&iter, stonith_remote_op_list);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&rop)) {
         if (strcmp(rop->target, target) != 0) {
             continue;
         } else if (rop->state != st_done) {
             continue;
         /* We don't have to worry about remapped reboots here
          * because if state is done, any remapping has been undone
          */
         } else if (strcmp(rop->action, action) != 0) {
             continue;
         } else if ((rop->completed + tolerance) < now) {
             continue;
         }
 
         crm_notice("Target %s was fenced (%s) less than %ds ago by %s on behalf of %s",
                    target, action, tolerance, rop->delegate, rop->originator);
         return TRUE;
     }
     return FALSE;
 }
diff --git a/lib/pacemaker/pcmk_fence.c b/lib/pacemaker/pcmk_fence.c
index bb531c3eca..7a0490f379 100644
--- a/lib/pacemaker/pcmk_fence.c
+++ b/lib/pacemaker/pcmk_fence.c
@@ -1,625 +1,626 @@
 /*
  * Copyright 2009-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <crm/common/mainloop.h>
 #include <crm/common/results.h>
 #include <crm/common/output.h>
 #include <crm/common/output_internal.h>
 #include <crm/stonith-ng.h>
 #include <crm/fencing/internal.h>
 
 #include <glib.h>
 #include <libxml/tree.h>
 #include <pacemaker.h>
 #include <pacemaker-internal.h>
 
 static const int st_opts = st_opt_sync_call | st_opt_allow_suicide;
 
 static GMainLoop *mainloop = NULL;
 
 static struct {
     stonith_t *st;
     const char *target;
     const char *action;
     char *name;
     unsigned int timeout;
     unsigned int tolerance;
     int delay;
     pcmk__action_result_t result;
 } async_fence_data = { NULL, };
 
 static int
 handle_level(stonith_t *st, const char *target, int fence_level,
              const stonith_key_value_t *devices, bool added)
 {
     const char *node = NULL;
     const char *pattern = NULL;
     const char *name = NULL;
     char *value = NULL;
     int rc = pcmk_rc_ok;
 
     if (target == NULL) {
         // Not really possible, but makes static analysis happy
         return EINVAL;
     }
 
     /* Determine if targeting by attribute, node name pattern or node name */
     value = strchr(target, '=');
     if (value != NULL)  {
         name = target;
         *value++ = '\0';
     } else if (*target == '@') {
         pattern = target + 1;
     } else {
         node = target;
     }
 
     /* Register or unregister level as appropriate */
     if (added) {
         rc = st->cmds->register_level_full(st, st_opts, node, pattern,
                                            name, value, fence_level,
                                            devices);
     } else {
         rc = st->cmds->remove_level_full(st, st_opts, node, pattern,
                                          name, value, fence_level);
     }
 
     return pcmk_legacy2rc(rc);
 }
 
 static stonith_history_t *
 reduce_fence_history(stonith_history_t *history)
 {
     stonith_history_t *new, *hp, *np;
 
     if (!history) {
         return history;
     }
 
     new = history;
     hp = new->next;
     new->next = NULL;
 
     while (hp) {
         stonith_history_t *hp_next = hp->next;
 
         hp->next = NULL;
 
         for (np = new; ; np = np->next) {
             if ((hp->state == st_done) || (hp->state == st_failed)) {
                 /* action not in progress */
                 if (pcmk__str_eq(hp->target, np->target, pcmk__str_casei) &&
                     pcmk__str_eq(hp->action, np->action, pcmk__str_none) &&
                     (hp->state == np->state) &&
                     ((hp->state == st_done) ||
                      pcmk__str_eq(hp->delegate, np->delegate, pcmk__str_casei))) {
                         /* purge older hp */
                         stonith_history_free(hp);
                         break;
                 }
             }
 
             if (!np->next) {
                 np->next = hp;
                 break;
             }
         }
         hp = hp_next;
     }
 
     return new;
 }
 
 static void
 notify_callback(stonith_t * st, stonith_event_t * e)
 {
     if (pcmk__str_eq(async_fence_data.target, e->target, pcmk__str_casei)
         && pcmk__str_eq(async_fence_data.action, e->action, pcmk__str_none)) {
 
         pcmk__set_result(&async_fence_data.result,
                          stonith__event_exit_status(e),
                          stonith__event_execution_status(e),
                          stonith__event_exit_reason(e));
         g_main_loop_quit(mainloop);
     }
 }
 
 static void
 fence_callback(stonith_t * stonith, stonith_callback_data_t * data)
 {
     pcmk__set_result(&async_fence_data.result, stonith__exit_status(data),
                      stonith__execution_status(data),
                      stonith__exit_reason(data));
     g_main_loop_quit(mainloop);
 }
 
 static gboolean
 async_fence_helper(gpointer user_data)
 {
     stonith_t *st = async_fence_data.st;
     int call_id = 0;
     int rc = stonith_api_connect_retry(st, async_fence_data.name, 10);
 
     if (rc != pcmk_ok) {
         g_main_loop_quit(mainloop);
         pcmk__set_result(&async_fence_data.result, CRM_EX_ERROR,
                          PCMK_EXEC_NOT_CONNECTED, pcmk_strerror(rc));
         return TRUE;
     }
 
     st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, notify_callback);
 
     call_id = st->cmds->fence_with_delay(st,
                                          st_opt_allow_suicide,
                                          async_fence_data.target,
                                          async_fence_data.action,
                                          async_fence_data.timeout/1000,
                                          async_fence_data.tolerance/1000,
                                          async_fence_data.delay);
 
     if (call_id < 0) {
         g_main_loop_quit(mainloop);
         pcmk__set_result(&async_fence_data.result, CRM_EX_ERROR,
                          PCMK_EXEC_ERROR, pcmk_strerror(call_id));
         return TRUE;
     }
 
     st->cmds->register_callback(st,
                                 call_id,
-                                async_fence_data.timeout/1000,
+                                (async_fence_data.timeout/1000
+                                + (async_fence_data.delay > 0 ? async_fence_data.delay : 0)),
                                 st_opt_timeout_updates, NULL, "callback", fence_callback);
 
     return TRUE;
 }
 
 int
 pcmk__request_fencing(stonith_t *st, const char *target, const char *action,
                       const char *name, unsigned int timeout,
                       unsigned int tolerance, int delay, char **reason)
 {
     crm_trigger_t *trig;
     int rc = pcmk_rc_ok;
 
     async_fence_data.st = st;
     async_fence_data.name = strdup(name);
     async_fence_data.target = target;
     async_fence_data.action = action;
     async_fence_data.timeout = timeout;
     async_fence_data.tolerance = tolerance;
     async_fence_data.delay = delay;
     pcmk__set_result(&async_fence_data.result, CRM_EX_ERROR, PCMK_EXEC_UNKNOWN,
                      NULL);
 
     trig = mainloop_add_trigger(G_PRIORITY_HIGH, async_fence_helper, NULL);
     mainloop_set_trigger(trig);
 
     mainloop = g_main_loop_new(NULL, FALSE);
     g_main_loop_run(mainloop);
 
     free(async_fence_data.name);
 
     if (reason != NULL) {
         // Give the caller ownership of the exit reason
         *reason = async_fence_data.result.exit_reason;
         async_fence_data.result.exit_reason = NULL;
     }
     rc = stonith__result2rc(&async_fence_data.result);
     pcmk__reset_result(&async_fence_data.result);
     return rc;
 }
 
 #ifdef BUILD_PUBLIC_LIBPACEMAKER
 int
 pcmk_request_fencing(stonith_t *st, const char *target, const char *action,
                      const char *name, unsigned int timeout,
                      unsigned int tolerance, int delay, char **reason)
 {
     return pcmk__request_fencing(st, target, action, name, timeout, tolerance,
                                  delay, reason);
 }
 #endif
 
 int
 pcmk__fence_history(pcmk__output_t *out, stonith_t *st, const char *target,
                     unsigned int timeout, int verbose, bool broadcast,
                     bool cleanup)
 {
     stonith_history_t *history = NULL, *hp, *latest = NULL;
     int rc = pcmk_rc_ok;
     int opts = 0;
 
     if (cleanup) {
         out->info(out, "cleaning up fencing-history%s%s",
                   target ? " for node " : "", target ? target : "");
     }
     if (broadcast) {
         out->info(out, "gather fencing-history from all nodes");
     }
 
     stonith__set_call_options(opts, target, st_opts);
     if (cleanup) {
         stonith__set_call_options(opts, target, st_opt_cleanup);
     }
     if (broadcast) {
         stonith__set_call_options(opts, target, st_opt_broadcast);
     }
     rc = st->cmds->history(st, opts,
                            pcmk__str_eq(target, "*", pcmk__str_none)? NULL : target,
                            &history, timeout/1000);
 
     if (cleanup) {
         // Cleanup doesn't return a history list
         stonith_history_free(history);
         return pcmk_legacy2rc(rc);
     }
 
     out->begin_list(out, "event", "events", "Fencing history");
 
     history = stonith__sort_history(history);
     for (hp = history; hp; hp = hp->next) {
         if (hp->state == st_done) {
             latest = hp;
         }
 
         if (out->is_quiet(out) || !verbose) {
             continue;
         }
 
         out->message(out, "stonith-event", hp, true, false,
                      stonith__later_succeeded(hp, history),
                      (uint32_t) pcmk_show_failed_detail);
         out->increment_list(out);
     }
 
     if (latest) {
         if (out->is_quiet(out)) {
             out->message(out, "stonith-event", latest, false, true, NULL,
                          (uint32_t) pcmk_show_failed_detail);
         } else if (!verbose) { // already printed if verbose
             out->message(out, "stonith-event", latest, false, false, NULL,
                          (uint32_t) pcmk_show_failed_detail);
             out->increment_list(out);
         }
     }
 
     out->end_list(out);
 
     stonith_history_free(history);
     return pcmk_legacy2rc(rc);
 }
 
 #ifdef BUILD_PUBLIC_LIBPACEMAKER
 int
 pcmk_fence_history(xmlNodePtr *xml, stonith_t *st, const char *target,
                    unsigned int timeout, bool quiet, int verbose,
                    bool broadcast, bool cleanup)
 {
     pcmk__output_t *out = NULL;
     int rc = pcmk_rc_ok;
 
     rc = pcmk__xml_output_new(&out, xml);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     stonith__register_messages(out);
 
     out->quiet = quiet;
 
     rc = pcmk__fence_history(out, st, target, timeout, verbose, broadcast, cleanup);
     pcmk__xml_output_finish(out, xml);
     return rc;
 }
 #endif
 
 int
 pcmk__fence_installed(pcmk__output_t *out, stonith_t *st, unsigned int timeout)
 {
     stonith_key_value_t *devices = NULL;
     int rc = pcmk_rc_ok;
 
     rc = st->cmds->list_agents(st, st_opt_sync_call, NULL, &devices, timeout/1000);
     /* list_agents returns a negative error code or a positive number of agents. */
     if (rc < 0) {
         return pcmk_legacy2rc(rc);
     }
 
     out->begin_list(out, "fence device", "fence devices", "Installed fence devices");
     for (stonith_key_value_t *dIter = devices; dIter; dIter = dIter->next) {
         out->list_item(out, "device", "%s", dIter->value);
     }
     out->end_list(out);
 
     stonith_key_value_freeall(devices, 1, 1);
     return pcmk_rc_ok;
 }
 
 #ifdef BUILD_PUBLIC_LIBPACEMAKER
 int
 pcmk_fence_installed(xmlNodePtr *xml, stonith_t *st, unsigned int timeout)
 {
     pcmk__output_t *out = NULL;
     int rc = pcmk_rc_ok;
 
     rc = pcmk__xml_output_new(&out, xml);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     stonith__register_messages(out);
 
     rc = pcmk__fence_installed(out, st, timeout);
     pcmk__xml_output_finish(out, xml);
     return rc;
 }
 #endif
 
 int
 pcmk__fence_last(pcmk__output_t *out, const char *target, bool as_nodeid)
 {
     time_t when = 0;
 
     if (target == NULL) {
         return pcmk_rc_ok;
     }
 
     if (as_nodeid) {
         when = stonith_api_time(atol(target), NULL, FALSE);
     } else {
         when = stonith_api_time(0, target, FALSE);
     }
 
     return out->message(out, "last-fenced", target, when);
 }
 
 #ifdef BUILD_PUBLIC_LIBPACEMAKER
 int
 pcmk_fence_last(xmlNodePtr *xml, const char *target, bool as_nodeid)
 {
     pcmk__output_t *out = NULL;
     int rc = pcmk_rc_ok;
 
     rc = pcmk__xml_output_new(&out, xml);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     stonith__register_messages(out);
 
     rc = pcmk__fence_last(out, target, as_nodeid);
     pcmk__xml_output_finish(out, xml);
     return rc;
 }
 #endif
 
 int
 pcmk__fence_list_targets(pcmk__output_t *out, stonith_t *st,
                          const char *device_id, unsigned int timeout)
 {
     GList *targets = NULL;
     char *lists = NULL;
     int rc = pcmk_rc_ok;
 
     rc = st->cmds->list(st, st_opts, device_id, &lists, timeout/1000);
     if (rc != pcmk_rc_ok) {
         return pcmk_legacy2rc(rc);
     }
 
     targets = stonith__parse_targets(lists);
 
     out->begin_list(out, "fence target", "fence targets", "Fence Targets");
     while (targets != NULL) {
         out->list_item(out, NULL, "%s", (const char *) targets->data);
         targets = targets->next;
     }
     out->end_list(out);
 
     free(lists);
     return rc;
 }
 
 #ifdef BUILD_PUBLIC_LIBPACEMAKER
 int
 pcmk_fence_list_targets(xmlNodePtr *xml, stonith_t *st, const char *device_id,
                         unsigned int timeout)
 {
     pcmk__output_t *out = NULL;
     int rc = pcmk_rc_ok;
 
     rc = pcmk__xml_output_new(&out, xml);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     stonith__register_messages(out);
 
     rc = pcmk__fence_list_targets(out, st, device_id, timeout);
     pcmk__xml_output_finish(out, xml);
     return rc;
 }
 #endif
 
 int
 pcmk__fence_metadata(pcmk__output_t *out, stonith_t *st, const char *agent,
                      unsigned int timeout)
 {
     char *buffer = NULL;
     int rc = st->cmds->metadata(st, st_opt_sync_call, agent, NULL, &buffer,
                                 timeout/1000);
 
     if (rc != pcmk_rc_ok) {
         return pcmk_legacy2rc(rc);
     }
 
     out->output_xml(out, "metadata", buffer);
     free(buffer);
     return rc;
 }
 
 #ifdef BUILD_PUBLIC_LIBPACEMAKER
 int
 pcmk_fence_metadata(xmlNodePtr *xml, stonith_t *st, const char *agent,
                     unsigned int timeout)
 {
     pcmk__output_t *out = NULL;
     int rc = pcmk_rc_ok;
 
     rc = pcmk__xml_output_new(&out, xml);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     stonith__register_messages(out);
 
     rc = pcmk__fence_metadata(out, st, agent, timeout);
     pcmk__xml_output_finish(out, xml);
     return rc;
 }
 #endif
 
 int
 pcmk__fence_registered(pcmk__output_t *out, stonith_t *st, const char *target,
                        unsigned int timeout)
 {
     stonith_key_value_t *devices = NULL;
     int rc = pcmk_rc_ok;
 
     rc = st->cmds->query(st, st_opts, target, &devices, timeout/1000);
     /* query returns a negative error code or a positive number of results. */
     if (rc < 0) {
         return pcmk_legacy2rc(rc);
     }
 
     out->begin_list(out, "fence device", "fence devices", "Registered fence devices");
     for (stonith_key_value_t *dIter = devices; dIter; dIter = dIter->next) {
         out->list_item(out, "device", "%s", dIter->value);
     }
     out->end_list(out);
 
     stonith_key_value_freeall(devices, 1, 1);
 
     /* Return pcmk_rc_ok here, not the number of results.  Callers probably
      * don't care.
      */
     return pcmk_rc_ok;
 }
 
 #ifdef BUILD_PUBLIC_LIBPACEMAKER
 int
 pcmk_fence_registered(xmlNodePtr *xml, stonith_t *st, const char *target,
                       unsigned int timeout)
 {
     pcmk__output_t *out = NULL;
     int rc = pcmk_rc_ok;
 
     rc = pcmk__xml_output_new(&out, xml);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     stonith__register_messages(out);
 
     rc = pcmk__fence_registered(out, st, target, timeout);
     pcmk__xml_output_finish(out, xml);
     return rc;
 }
 #endif
 
 int
 pcmk__fence_register_level(stonith_t *st, const char *target, int fence_level,
                            const stonith_key_value_t *devices)
 {
     return handle_level(st, target, fence_level, devices, true);
 }
 
 #ifdef BUILD_PUBLIC_LIBPACEMAKER
 int
 pcmk_fence_register_level(stonith_t *st, const char *target, int fence_level,
                           const stonith_key_value_t *devices)
 {
     return pcmk__fence_register_level(st, target, fence_level, devices);
 }
 #endif
 
 int
 pcmk__fence_unregister_level(stonith_t *st, const char *target, int fence_level)
 {
     return handle_level(st, target, fence_level, NULL, false);
 }
 
 #ifdef BUILD_PUBLIC_LIBPACEMAKER
 int
 pcmk_fence_unregister_level(stonith_t *st, const char *target, int fence_level)
 {
     return pcmk__fence_unregister_level(st, target, fence_level);
 }
 #endif
 
 int
 pcmk__fence_validate(pcmk__output_t *out, stonith_t *st, const char *agent,
                      const char *id, const stonith_key_value_t *params,
                      unsigned int timeout)
 {
     char *output = NULL;
     char *error_output = NULL;
     int rc;
 
     rc  = st->cmds->validate(st, st_opt_sync_call, id, NULL, agent, params,
                              timeout/1000, &output, &error_output);
     out->message(out, "validate", agent, id, output, error_output, rc);
     return pcmk_legacy2rc(rc);
 }
 
 #ifdef BUILD_PUBLIC_LIBPACEMAKER
 int
 pcmk_fence_validate(xmlNodePtr *xml, stonith_t *st, const char *agent,
                     const char *id, const stonith_key_value_t *params,
                     unsigned int timeout)
 {
     pcmk__output_t *out = NULL;
     int rc = pcmk_rc_ok;
 
     rc = pcmk__xml_output_new(&out, xml);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     stonith__register_messages(out);
 
     rc = pcmk__fence_validate(out, st, agent, id, params, timeout);
     pcmk__xml_output_finish(out, xml);
     return rc;
 }
 #endif
 
 int
 pcmk__get_fencing_history(stonith_t *st, stonith_history_t **stonith_history,
                           enum pcmk__fence_history fence_history)
 {
     int rc = pcmk_rc_ok;
 
     if ((st == NULL) || (st->state == stonith_disconnected)) {
         rc = ENOTCONN;
     } else if (fence_history != pcmk__fence_history_none) {
         rc = st->cmds->history(st, st_opt_sync_call, NULL, stonith_history, 120);
 
         rc = pcmk_legacy2rc(rc);
         if (rc != pcmk_rc_ok) {
             return rc;
         }
 
         *stonith_history = stonith__sort_history(*stonith_history);
         if (fence_history == pcmk__fence_history_reduced) {
             *stonith_history = reduce_fence_history(*stonith_history);
         }
     }
 
     return rc;
 }
diff --git a/python/pacemaker/_cts/patterns.py b/python/pacemaker/_cts/patterns.py
index 2e4c23d232..880477aa6a 100644
--- a/python/pacemaker/_cts/patterns.py
+++ b/python/pacemaker/_cts/patterns.py
@@ -1,408 +1,408 @@
 """ Pattern-holding classes for Pacemaker's Cluster Test Suite (CTS) """
 
 __all__ = ["PatternSelector"]
 __copyright__ = "Copyright 2008-2023 the Pacemaker project contributors"
 __license__ = "GNU General Public License version 2 or later (GPLv2+)"
 
 import argparse
 
 from pacemaker.buildoptions import BuildOptions
 
 class BasePatterns:
     """ The base class for holding a stack-specific set of command and log
         file/stdout patterns.  Stack-specific classes need to be built on top
         of this one.
     """
 
     def __init__(self):
         """ Create a new BasePatterns instance which holds a very minimal set of
             basic patterns.
         """
 
         self._bad_news = []
         self._components = {}
         self._name = "crm-base"
 
         self._ignore = [
             "avoid confusing Valgrind",
 
             # Logging bug in some versions of libvirtd
             r"libvirtd.*: internal error: Failed to parse PCI config address",
 
             # pcs can log this when node is fenced, but fencing is OK in some
             # tests (and we will catch it in pacemaker logs when not OK)
             r"pcs.daemon:No response from: .* request: get_configs, error:",
         ]
 
         self._commands = {
             "StatusCmd"      : "crmadmin -t 60 -S %s 2>/dev/null",
             "CibQuery"       : "cibadmin -Ql",
             "CibAddXml"      : "cibadmin --modify -c --xml-text %s",
             "CibDelXpath"    : "cibadmin --delete --xpath %s",
             "RscRunning"     : BuildOptions.DAEMON_DIR + "/cts-exec-helper -R -r %s",
             "CIBfile"        : "%s:" + BuildOptions.CIB_DIR + "/cib.xml",
             "TmpDir"         : "/tmp",
 
             "BreakCommCmd"   : "iptables -A INPUT -s %s -j DROP >/dev/null 2>&1",
             "FixCommCmd"     : "iptables -D INPUT -s %s -j DROP >/dev/null 2>&1",
 
             "MaintenanceModeOn"    : "cibadmin --modify -c --xml-text '<cluster_property_set id=\"cib-bootstrap-options\"><nvpair id=\"cts-maintenance-mode-setting\" name=\"maintenance-mode\" value=\"true\"/></cluster_property_set>'",
             "MaintenanceModeOff"    : "cibadmin --delete --xpath \"//nvpair[@name='maintenance-mode']\"",
 
             "StandbyCmd"      : "crm_attribute -Vq  -U %s -n standby -l forever -v %s 2>/dev/null",
             "StandbyQueryCmd" : "crm_attribute -qG -U %s -n standby -l forever -d off 2>/dev/null",
         }
 
         self._search = {
             "Pat:DC_IDLE"      : r"pacemaker-controld.*State transition.*-> S_IDLE",
 
             # This won't work if we have multiple partitions
             "Pat:Local_started" : r"%s\W.*controller successfully started",
             "Pat:NonDC_started" : r"%s\W.*State transition.*-> S_NOT_DC",
             "Pat:DC_started"    : r"%s\W.*State transition.*-> S_IDLE",
             "Pat:We_stopped"    : r"%s\W.*OVERRIDE THIS PATTERN",
             "Pat:They_stopped"  : r"%s\W.*LOST:.* %s ",
             "Pat:They_dead"     : r"node %s.*: is dead",
             "Pat:They_up"       : r"%s %s\W.*OVERRIDE THIS PATTERN",
             "Pat:TransitionComplete" : "Transition status: Complete: complete",
 
             "Pat:Fencing_start"   : r"Requesting peer fencing .* targeting %s",
             "Pat:Fencing_ok"      : r"pacemaker-fenced.*:\s*Operation .* targeting %s by .* for .*@.*: OK",
             "Pat:Fencing_recover" : r"pacemaker-schedulerd.*: Recover\s+%s",
             "Pat:Fencing_active"  : r"stonith resource .* is active on 2 nodes (attempting recovery)",
             "Pat:Fencing_probe"   : r"pacemaker-controld.* Result of probe operation for %s on .*: Error",
 
             "Pat:RscOpOK"       : r"pacemaker-controld.*:\s+Result of %s operation for %s.*: (0 \()?ok",
             "Pat:RscOpFail"     : r"pacemaker-schedulerd.*:.*Unexpected result .* recorded for %s of %s ",
             "Pat:CloneOpFail"   : r"pacemaker-schedulerd.*:.*Unexpected result .* recorded for %s of (%s|%s) ",
             "Pat:RscRemoteOpOK" : r"pacemaker-controld.*:\s+Result of %s operation for %s on %s: (0 \()?ok",
             "Pat:NodeFenced"    : r"pacemaker-controld.*:\s* Peer %s was terminated \(.*\) by .* on behalf of .*: OK",
         }
 
     def get_component(self, key):
         """ Return the patterns for a single component as a list, given by key.
             This is typically the name of some subprogram (pacemaker-based,
             pacemaker-fenced, etc.) or various special purpose keys.  If key is
             unknown, return an empty list.
         """
 
         if key in self._components:
             return self._components[key]
 
         print("Unknown component '%s' for %s" % (key, self._name))
         return []
 
     def get_patterns(self, key):
         """ Return various patterns supported by this object, given by key.
             Depending on the key, this could either be a list or a hash.  If key
             is unknown, return None.
         """
 
         if key == "BadNews":
             return self._bad_news
         if key == "BadNewsIgnore":
             return self._ignore
         if key == "Commands":
             return self._commands
         if key == "Search":
             return self._search
         if key == "Components":
             return self._components
 
         print("Unknown pattern '%s' for %s" % (key, self._name))
         return None
 
     def __getitem__(self, key):
         if key == "Name":
             return self._name
         if key in self._commands:
             return self._commands[key]
         if key in self._search:
             return self._search[key]
 
         print("Unknown template '%s' for %s" % (key, self._name))
         return None
 
 
 class Corosync2Patterns(BasePatterns):
     """ Patterns for Corosync version 2 cluster manager class """
 
     def __init__(self):
         BasePatterns.__init__(self)
         self._name = "crm-corosync"
 
         self._commands.update({
             "StartCmd"       : "service corosync start && service pacemaker start",
             "StopCmd"        : "service pacemaker stop; [ ! -e /usr/sbin/pacemaker-remoted ] || service pacemaker_remote stop; service corosync stop",
 
             "EpochCmd"       : "crm_node -e",
             "QuorumCmd"      : "crm_node -q",
             "PartitionCmd"   : "crm_node -p",
         })
 
         self._search.update({
             # Close enough ... "Corosync Cluster Engine exiting normally" isn't
             # printed reliably.
             "Pat:We_stopped"   : r"%s\W.*Unloading all Corosync service engines",
             "Pat:They_stopped" : r"%s\W.*pacemaker-controld.*Node %s(\[|\s).*state is now lost",
             "Pat:They_dead"    : r"pacemaker-controld.*Node %s(\[|\s).*state is now lost",
             "Pat:They_up"      : r"\W%s\W.*pacemaker-controld.*Node %s state is now member",
 
             "Pat:ChildExit"    : r"\[[0-9]+\] exited with status [0-9]+ \(",
             # "with signal 9" == pcmk_child_exit(), "$" == check_active_before_startup_processes()
             "Pat:ChildKilled"  : r"%s\W.*pacemakerd.*%s\[[0-9]+\] terminated( with signal 9|$)",
             "Pat:ChildRespawn" : r"%s\W.*pacemakerd.*Respawning %s subdaemon after unexpected exit",
 
             "Pat:InfraUp"      : r"%s\W.*corosync.*Initializing transport",
             "Pat:PacemakerUp"  : r"%s\W.*pacemakerd.*Starting Pacemaker",
         })
 
         self._ignore += [
             r"crm_mon:",
             r"crmadmin:",
             r"update_trace_data",
             r"async_notify:.*strange, client not found",
             r"Parse error: Ignoring unknown option .*nodename",
             r"error.*: Operation 'reboot' .* using FencingFail returned ",
             r"getinfo response error: 1$",
             r"sbd.* error: inquisitor_child: DEBUG MODE IS ACTIVE",
             r"sbd.* pcmk:\s*error:.*Connection to cib_ro.* (failed|closed)",
         ]
 
         self._bad_news = [
             r"[^(]error:",
             r"crit:",
             r"ERROR:",
             r"CRIT:",
             r"Shutting down...NOW",
             r"Timer I_TERMINATE just popped",
             r"input=I_ERROR",
             r"input=I_FAIL",
             r"input=I_INTEGRATED cause=C_TIMER_POPPED",
             r"input=I_FINALIZED cause=C_TIMER_POPPED",
             r"input=I_ERROR",
             r"(pacemakerd|pacemaker-execd|pacemaker-controld):.*, exiting",
             r"schedulerd.*Attempting recovery of resource",
             r"is taking more than 2x its timeout",
             r"Confirm not received from",
             r"Welcome reply not received from",
             r"Attempting to schedule .* after a stop",
             r"Resource .* was active at shutdown",
             r"duplicate entries for call_id",
             r"Search terminated:",
             r":global_timer_callback",
             r"Faking parameter digest creation",
             r"Parameters to .* action changed:",
             r"Parameters to .* changed",
             r"pacemakerd.*\[[0-9]+\] terminated( with signal| as IPC server|$)",
             r"pacemaker-schedulerd.*Recover\s+.*\(.* -\> .*\)",
             r"rsyslogd.* imuxsock lost .* messages from pid .* due to rate-limiting",
             r"Peer is not part of our cluster",
             r"We appear to be in an election loop",
             r"Unknown node -> we will not deliver message",
             r"(Blackbox dump requested|Problem detected)",
             r"pacemakerd.*Could not connect to Cluster Configuration Database API",
             r"Receiving messages from a node we think is dead",
             r"share the same cluster nodeid",
             r"share the same name",
 
             r"pacemaker-controld:.*Transition failed: terminated",
             r"Local CIB .* differs from .*:",
             r"warn.*:\s*Continuing but .* will NOT be used",
             r"warn.*:\s*Cluster configuration file .* is corrupt",
             r"Election storm",
             r"stalled the FSA with pending inputs",
         ]
 
         self._components["common-ignore"] = [
             r"Pending action:",
             r"resource( was|s were) active at shutdown",
             r"pending LRM operations at shutdown",
             r"Lost connection to the CIB manager",
             r"pacemaker-controld.*:\s*Action A_RECOVER .* not supported",
             r"pacemaker-controld.*:\s*Performing A_EXIT_1 - forcefully exiting ",
-            r".*:\s*Requesting fencing \([^)]+\) of node ",
+            r".*:\s*Requesting fencing \([^)]+\) targeting node ",
             r"(Blackbox dump requested|Problem detected)",
         ]
 
         self._components["corosync-ignore"] = [
             r"Could not connect to Corosync CFG: CS_ERR_LIBRARY",
             r"error:.*Connection to the CPG API failed: Library error",
             r"\[[0-9]+\] exited with status [0-9]+ \(",
             r"\[[0-9]+\] terminated with signal 15",
             r"pacemaker-based.*error:.*Corosync connection lost",
             r"pacemaker-fenced.*error:.*Corosync connection terminated",
             r"pacemaker-controld.*State transition .* S_RECOVERY",
             r"pacemaker-controld.*error:.*Input (I_ERROR|I_TERMINATE ) .*received in state",
             r"pacemaker-controld.*error:.*Could not recover from internal error",
             r"error:.*Connection to cib_(shm|rw).* (failed|closed)",
             r"error:.*cib_(shm|rw) IPC provider disconnected while waiting",
             r"error:.*Connection to (fencer|stonith-ng).* (closed|failed|lost)",
             r"crit: Fencing daemon connection failed",
             # This is overbroad, but we don't have a way to say that only
             # certain transition errors are acceptable (if the fencer respawns,
             # fence devices may appear multiply active). We have to rely on
             # other causes of a transition error logging their own error
             # message, which is the usual practice.
             r"pacemaker-schedulerd.* Calculated transition .*/pe-error",
             ]
 
         self._components["corosync"] = [
             # We expect each daemon to lose its cluster connection.
             # However, if the CIB manager loses its connection first,
             # it's possible for another daemon to lose that connection and
             # exit before losing the cluster connection.
             r"pacemakerd.*:\s*warning:.*Lost connection to cluster layer",
             r"pacemaker-attrd.*:\s*(crit|error):.*Lost connection to (cluster layer|the CIB manager)",
             r"pacemaker-based.*:\s*(crit|error):.*Lost connection to cluster layer",
             r"pacemaker-controld.*:\s*(crit|error):.*Lost connection to (cluster layer|the CIB manager)",
             r"pacemaker-fenced.*:\s*(crit|error):.*Lost connection to (cluster layer|the CIB manager)",
             r"schedulerd.*Scheduling node .* for fencing",
             r"pacemaker-controld.*:\s*Peer .* was terminated \(.*\) by .* on behalf of .*:\s*OK",
         ]
 
         self._components["pacemaker-based"] = [
             r"pacemakerd.* pacemaker-attrd\[[0-9]+\] exited with status 102",
             r"pacemakerd.* pacemaker-controld\[[0-9]+\] exited with status 1",
             r"pacemakerd.* Respawning pacemaker-attrd subdaemon after unexpected exit",
             r"pacemakerd.* Respawning pacemaker-based subdaemon after unexpected exit",
             r"pacemakerd.* Respawning pacemaker-controld subdaemon after unexpected exit",
             r"pacemakerd.* Respawning pacemaker-fenced subdaemon after unexpected exit",
             r"pacemaker-.* Connection to cib_.* (failed|closed)",
             r"pacemaker-attrd.*:.*Lost connection to the CIB manager",
             r"pacemaker-controld.*:.*Lost connection to the CIB manager",
             r"pacemaker-controld.*I_ERROR.*handle_cib_disconnect",
             r"pacemaker-controld.* State transition .* S_RECOVERY",
             r"pacemaker-controld.*: Input I_TERMINATE .*from do_recover",
             r"pacemaker-controld.*Could not recover from internal error",
         ]
 
         self._components["pacemaker-based-ignore"] = [
             r"pacemaker-execd.*Connection to (fencer|stonith-ng).* (closed|failed|lost)",
             r"pacemaker-controld.*:\s+Result of .* operation for Fencing.*Error \(Lost connection to fencer\)",
             r"pacemaker-controld.*:Could not connect to attrd: Connection refused",
             # This is overbroad, but we don't have a way to say that only
             # certain transition errors are acceptable (if the fencer respawns,
             # fence devices may appear multiply active). We have to rely on
             # other causes of a transition error logging their own error
             # message, which is the usual practice.
             r"pacemaker-schedulerd.* Calculated transition .*/pe-error",
         ]
 
         self._components["pacemaker-execd"] = [
             r"pacemaker-controld.*Connection to executor failed",
             r"pacemaker-controld.*I_ERROR.*lrm_connection_destroy",
             r"pacemaker-controld.*State transition .* S_RECOVERY",
             r"pacemaker-controld.*: Input I_TERMINATE .*from do_recover",
             r"pacemaker-controld.*Could not recover from internal error",
             r"pacemakerd.*pacemaker-controld\[[0-9]+\] exited with status 1",
             r"pacemakerd.* Respawning pacemaker-execd subdaemon after unexpected exit",
             r"pacemakerd.* Respawning pacemaker-controld subdaemon after unexpected exit",
         ]
 
         self._components["pacemaker-execd-ignore"] = [
             r"pacemaker-(attrd|controld).*Connection to lrmd.* (failed|closed)",
             r"pacemaker-(attrd|controld).*Could not execute alert",
         ]
 
         self._components["pacemaker-controld"] = [
             r"State transition .* -> S_IDLE",
         ]
 
         self._components["pacemaker-controld-ignore"] = []
         self._components["pacemaker-attrd"] = []
         self._components["pacemaker-attrd-ignore"] = []
 
         self._components["pacemaker-schedulerd"] = [
             r"State transition .* S_RECOVERY",
             r"pacemakerd.* Respawning pacemaker-controld subdaemon after unexpected exit",
             r"pacemaker-controld\[[0-9]+\] exited with status 1 \(",
             r"Connection to the scheduler failed",
             r"pacemaker-controld.*I_ERROR.*save_cib_contents",
             r"pacemaker-controld.*: Input I_TERMINATE .*from do_recover",
             r"pacemaker-controld.*Could not recover from internal error",
         ]
 
         self._components["pacemaker-schedulerd-ignore"] = [
             r"Connection to pengine.* (failed|closed)",
         ]
 
         self._components["pacemaker-fenced"] = [
             r"error:.*Connection to (fencer|stonith-ng).* (closed|failed|lost)",
             r"Fencing daemon connection failed",
             r"pacemaker-controld.*Fencer successfully connected",
         ]
 
         self._components["pacemaker-fenced-ignore"] = [
             r"(error|warning):.*Connection to (fencer|stonith-ng).* (closed|failed|lost)",
             r"crit:.*Fencing daemon connection failed",
             r"error:.*Fencer connection failed \(will retry\)",
             r"pacemaker-controld.*:\s+Result of .* operation for Fencing.*Error \(Lost connection to fencer\)",
             # This is overbroad, but we don't have a way to say that only
             # certain transition errors are acceptable (if the fencer respawns,
             # fence devices may appear multiply active). We have to rely on
             # other causes of a transition error logging their own error
             # message, which is the usual practice.
             r"pacemaker-schedulerd.* Calculated transition .*/pe-error",
         ]
 
         self._components["pacemaker-fenced-ignore"].extend(self._components["common-ignore"])
 
 
 patternVariants = {
     "crm-base": BasePatterns,
     "crm-corosync": Corosync2Patterns
 }
 
 
 class PatternSelector:
     """ A class for choosing one of several Pattern objects and then extracting
         various pieces of information from that object
     """
 
     def __init__(self, name="crm-corosync"):
         """ Create a new PatternSelector object by instantiating whatever class
             is given by name.  Defaults to Corosync2Patterns for "crm-corosync" or
             None.  While other objects could be supported in the future, only this
             and the base object are supported at this time.
         """
 
         self._name = name
 
         # If no name was given, use the default.  Otherwise, look up the appropriate
         # class in patternVariants, instantiate it, and use that.
         if not name:
             self._base = Corosync2Patterns()
         else:
             self._base = patternVariants[name]()
 
     def get_patterns(self, kind):
         """ Call get_patterns on the previously instantiated pattern object """
 
         return self._base.get_patterns(kind)
 
     def get_template(self, key):
         """ Return a single pattern from the previously instantiated pattern
             object as a string, or None if no pattern exists for the given key.
         """
 
         return self._base[key]
 
     def get_component(self, kind):
         """ Call get_component on the previously instantiated pattern object """
 
         return self._base.get_component(kind)
 
     def __getitem__(self, key):
         return self.get_template(key)
 
 
 # PYTHONPATH=python python python/pacemaker/_cts/patterns.py -k crm-corosync -t StartCmd
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument("-k", "--kind", metavar="KIND")
     parser.add_argument("-t", "--template", metavar="TEMPLATE")
 
     args = parser.parse_args()
 
     print(PatternSelector(args.kind)[args.template])