diff --git a/.gitignore b/.gitignore
index 83a1e2e20d..8cc2c0c3fb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,183 +1,185 @@
 # Common
 \#*
 .\#*
 GPATH
 GRTAGS
 GTAGS
 TAGS
 Makefile
 Makefile.in
 .deps
 .libs
 *.pc
 *.pyc
 *.bz2
 *.tar.gz
 *.rpm
 *.la
 *.lo
 *.o
 *~
 *.gcda
 *.gcno
 
 
 # Autobuild
 aclocal.m4
 autoconf
 autoheader
 autom4te.cache/
 automake
 build.counter
 compile
 config.guess
 config.log
 config.status
 config.sub
 configure
 depcomp
 install-sh
 include/stamp-*
 libtool
 libtool.m4
 ltdl.m4
 libltdl
 ltmain.sh
 missing
 py-compile
 /m4/argz.m4
 /m4/ltargz.m4
 /m4/ltoptions.m4
 /m4/ltsugar.m4
 /m4/ltversion.m4
 /m4/lt~obsolete.m4
 test-driver
 ylwrap
 
 # Configure targets
 Doxyfile
 cts/CTSlab.py
 cts/CTSvars.py
 cts/LSBDummy
 cts/OCFIPraTest.py
 cts/benchmark/clubench
 cts/cts-coverage
 cts/cts-lrmd
+cts/cts-pengine
 cts/cts-stonithd
 cts/fence_dummy
 cts/lxc_autogen.sh
 extra/logrotate/pacemaker
 include/config.h
 include/config.h.in
 include/crm_config.h
 lrmd/pacemaker_remote
 lrmd/pacemaker_remoted
 lrmd/pacemaker_remote.service
 mcp/pacemaker
 mcp/pacemaker.combined.upstart
 mcp/pacemaker.service
 mcp/pacemaker.upstart
-pengine/regression.core.sh
 publican.cfg
 tools/cibsecret
 tools/crm_error
 tools/crm_mon.service
 tools/crm_mon.upstart
 tools/crm_report
 tools/report.common
 
 # Build targets
 *.7
 *.7.xml
 *.7.html
 *.8
 *.8.xml
 *.8.html
 attrd/attrd
 doc/*/en-US/images/*.png
 doc/*/tmp/**
 doc/*/publish
 cib/cib
 cib/cibmon
 cib/cibpipe
 crmd/atest
 crmd/crmd
 doc/api/*
 doc/Clusters_from_Scratch.txt
 doc/Pacemaker_Explained.txt
 doc/acls.html
 doc/crm_fencing.html
 doc/publican-catalog*
 fencing/stonith-test
 fencing/stonith_admin
 fencing/stonithd
 fencing/stonithd.xml
 lrmd/lrmd
 lrmd/lrmd_internal_ctl
 lrmd/lrmd_test
 mcp/pacemakerd
 pengine/pengine
 pengine/pengine.xml
 pengine/ptest
 scratch
 tools/attrd_updater
 tools/cibadmin
 tools/crm_attribute
 tools/crm_diff
 tools/crm_mon
 tools/crm_node
 tools/crm_resource
 tools/crm_shadow
 tools/crm_simulate
 tools/crm_verify
 tools/crmadmin
 tools/iso8601
 tools/crm_ticket
 tools/report.collector.1
 xml/crm.dtd
 xml/pacemaker*.rng
 xml/versions.rng
 doc/shared/en-US/*.xml
 doc/Clusters_from_Scratch.build
 doc/Clusters_from_Scratch/en-US/Ap-*.xml
 doc/Clusters_from_Scratch/en-US/Ch-*.xml
 doc/Pacemaker_Administration.build
 doc/Pacemaker_Administration/en-US/Ch-*.xml
 doc/Pacemaker_Development.build
 doc/Pacemaker_Development/en-US/Ch-*.xml
 doc/Pacemaker_Explained.build
 doc/Pacemaker_Explained/en-US/Ch-*.xml
 doc/Pacemaker_Explained/en-US/Ap-*.xml
 doc/Pacemaker_Remote.build
 doc/Pacemaker_Remote/en-US/Ch-*.xml
 lib/gnu/libgnu.a
 lib/gnu/stdalign.h
 *.coverity
 
 # Test detritus
+cts/.regression.failed.diff
 cts/pengine/shadow.*
 
 # Formerly built files (helps when jumping back and forth in checkout)
 /coverage.sh
 /fencing/regression.py
 /lrmd/regression.py
+/pengine/.regression.failed.diff
+/pengine/regression.core.sh
 /pengine/test10/shadow.*
 
 #Other 
 mock
 HTML
 pacemaker*.spec
-pengine/.regression.failed.diff
 coverity-*
 
 compat_reports
 .ABI-build
 abi_dumps
 logs
 
 *.patch
 *.diff
 *.sed
 *.orig
 *.rej
 *.swp
diff --git a/configure.ac b/configure.ac
index c04b110c86..0a11e90a56 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,1746 +1,1746 @@
 dnl
 dnl autoconf for Pacemaker
 dnl
 dnl License: GNU General Public License (GPL)
 
 dnl ===============================================
 dnl Bootstrap
 dnl ===============================================
 AC_PREREQ(2.64)
 
 AC_CONFIG_MACRO_DIR([m4])
 
 dnl Suggested structure:
 dnl     information on the package
 dnl     checks for programs
 dnl     checks for libraries
 dnl     checks for header files
 dnl     checks for types
 dnl     checks for structures
 dnl     checks for compiler characteristics
 dnl     checks for library functions
 dnl     checks for system services
 
 m4_include([version.m4])
 AC_INIT([pacemaker], VERSION_NUMBER, [users@clusterlabs.org], [pacemaker],
         PCMK_URL)
 
 PCMK_FEATURES=""
 
 AC_CONFIG_AUX_DIR(.)
 AC_CANONICAL_HOST
 
 dnl Where #defines go (e.g. `AC_CHECK_HEADERS' below)
 dnl
 dnl Internal header: include/config.h
 dnl   - Contains ALL defines
 dnl   - include/config.h.in is generated automatically by autoheader
 dnl   - NOT to be included in any header files except crm_internal.h
 dnl     (which is also not to be included in any other header files)
 dnl
 dnl External header: include/crm_config.h
 dnl   - Contains a subset of defines checked here
 dnl   - Manually edit include/crm_config.h.in to have configure include
 dnl     new defines
 dnl   - Should not include HAVE_* defines
 dnl   - Safe to include anywhere
 AM_CONFIG_HEADER(include/config.h include/crm_config.h)
 
 AC_ARG_WITH(version,
     [  --with-version=version   Override package version (if you are a packager needing to pretend) ],
     [ PACKAGE_VERSION="$withval" ])
 
 AC_ARG_WITH(pkg-name,
     [  --with-pkg-name=name     Override package name (if you are a packager needing to pretend) ],
     [ PACKAGE_NAME="$withval" ])
 
 dnl 1.11:         minimum automake version required
 dnl foreign:      don't require GNU-standard top-level files
 dnl silent-rules: allow "--enable-silent-rules" (no-op in 1.13+)
 AM_INIT_AUTOMAKE([1.11 foreign silent-rules])
 
 dnl Example 2.4. Silent Custom Rule to Generate a File
 dnl %-bar.pc: %.pc
 dnl	$(AM_V_GEN)$(LN_S) $(notdir $^) $@
 
 AC_DEFINE_UNQUOTED(PACEMAKER_VERSION, "$PACKAGE_VERSION",
                    [Current pacemaker version])
 
 dnl Versioned attributes implementation is not yet production-ready
 AC_DEFINE_UNQUOTED(ENABLE_VERSIONED_ATTRS, 0, [Enable versioned attributes])
 
 PACKAGE_SERIES=`echo $PACKAGE_VERSION | awk -F. '{ print $1"."$2 }'`
 AC_SUBST(PACKAGE_SERIES)
 AC_SUBST(PACKAGE_VERSION)
 
 CC_IN_CONFIGURE=yes
 export CC_IN_CONFIGURE
 
 LDD=ldd
 
 dnl ========================================================================
 dnl Compiler characteristics
 dnl ========================================================================
 
 AC_PROG_CC dnl Can force other with environment variable "CC".
 AM_PROG_CC_C_O
 AC_PROG_CC_STDC
 gl_EARLY
 gl_INIT
 
 LT_INIT([dlopen])
 LTDL_INIT([convenience])
 
 AC_PROG_YACC
 AM_PROG_LEX
 
 AC_C_STRINGIZE
 AC_TYPE_SIZE_T
 AC_CHECK_SIZEOF(char)
 AC_CHECK_SIZEOF(short)
 AC_CHECK_SIZEOF(int)
 AC_CHECK_SIZEOF(long)
 AC_CHECK_SIZEOF(long long)
 AC_STRUCT_TIMEZONE
 
 dnl ===============================================
 dnl Helpers
 dnl ===============================================
 cc_supports_flag() {
     local CFLAGS="-Werror $@"
     AC_MSG_CHECKING(whether $CC supports "$@")
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ ]])],
                       [RC=0; AC_MSG_RESULT(yes)],
                       [RC=1; AC_MSG_RESULT(no)])
     return $RC
 }
 
 dnl ===============================================
 dnl Configure Options
 dnl ===============================================
 
 dnl Some systems, like Solaris require a custom package name
 AC_ARG_WITH(pkgname,
     [  --with-pkgname=name     name for pkg (typically for Solaris) ],
     [ PKGNAME="$withval" ],
     [ PKGNAME="LXHAhb" ],
   )
 AC_SUBST(PKGNAME)
 
 AC_ARG_ENABLE([ansi],
     [  --enable-ansi  Force GCC to compile to ANSI standard for older compilers. [default=no] ])
 
 AC_ARG_ENABLE([fatal-warnings],
     [  --enable-fatal-warnings  Enable pedantic and fatal warnings for gcc [default=yes] ])
 
 AC_ARG_ENABLE([quiet],
     [  --enable-quiet  Suppress make output unless there is an error [default=no] ])
 
 AC_ARG_ENABLE([no-stack],
     [  --enable-no-stack  Build only the Policy Engine and its requirements [default=no] ])
 
 AC_ARG_ENABLE([upstart],
     [  --enable-upstart  Enable support for managing resources via Upstart [default=try] ],
     [],
     [enable_upstart=try],
 )
 
 AC_ARG_ENABLE([systemd],
     [  --enable-systemd  Enable support for managing resources via systemd [default=try]],
     [],
     [enable_systemd=try],
 )
 
 AC_ARG_ENABLE(hardening,
     [  --with-hardening  Harden the resulting executables/libraries (best effort by default) ],
     [ HARDENING="${enableval}" ],
     [ HARDENING=try ],
 )
 
 AC_ARG_WITH(corosync,
     [  --with-corosync  Support the Corosync messaging and membership layer ],
     [ SUPPORT_CS=$withval ],
     [ SUPPORT_CS=try ],
 )
 
 AC_ARG_WITH(nagios,
     [  --with-nagios  Support nagios remote monitoring ],
     [ SUPPORT_NAGIOS=$withval ],
     [ SUPPORT_NAGIOS=try ],
 )
 
 AC_ARG_WITH(nagios-plugin-dir,
     [  --with-nagios-plugin-dir=DIR  Directory for nagios plugins [${NAGIOS_PLUGIN_DIR}] ],
     [ NAGIOS_PLUGIN_DIR="$withval" ]
 )
 
 AC_ARG_WITH(nagios-metadata-dir,
     [  --with-nagios-metadata-dir=DIR  Directory for nagios plugins metadata [${NAGIOS_METADATA_DIR}] ],
     [ NAGIOS_METADATA_DIR="$withval" ]
 )
 
 AC_ARG_WITH(acl,
     [  --with-acl  Support CIB ACL ],
     [ SUPPORT_ACL=$withval ],
     [ SUPPORT_ACL=yes ],
 )
 
 AC_ARG_WITH(cibsecrets,
     [  --with-cibsecrets  Support separate file for CIB secrets ],
     [ SUPPORT_CIBSECRETS=$withval ],
     [ SUPPORT_CIBSECRETS=no ],
 )
 
 INITDIR=""
 AC_ARG_WITH(initdir,
     [  --with-initdir=DIR  Directory for init (rc) scripts [${INITDIR}] ],
     [ INITDIR="$withval" ])
 
 SUPPORT_PROFILING=0
 AC_ARG_WITH(profiling,
     [  --with-profiling  Disable optimizations for effective profiling ],
     [ SUPPORT_PROFILING=$withval ])
 
 AC_ARG_WITH(coverage,
     [  --with-coverage   Disable optimizations for effective profiling ],
     [ SUPPORT_COVERAGE=$withval ])
 
 PUBLICAN_BRAND="common"
 AC_ARG_WITH(brand,
     [  --with-brand=brand  Brand to use for generated documentation (set empty for no docs) [$PUBLICAN_BRAND] ],
     [ test x"$withval" = x"no" || PUBLICAN_BRAND="$withval" ])
 AC_SUBST(PUBLICAN_BRAND)
 
 CONFIGDIR=""
 AC_ARG_WITH(configdir,
     [  --with-configdir=DIR  Directory for Pacemaker configuration file [${CONFIGDIR}]],
     [ CONFIGDIR="$withval" ]
 )
 
 CRM_LOG_DIR=""
 AC_ARG_WITH(logdir,
     [  --with-logdir=DIR  Directory for Pacemaker log file @<:@LOCALSTATEDIR/log/pacemaker@:>@ ],
     [ CRM_LOG_DIR="$withval" ]
 )
 
 CRM_BUNDLE_DIR=""
 AC_ARG_WITH(bundledir,
     [  --with-bundledir=DIR  Directory for Pacemaker bundle logs @<:@LOCALSTATEDIR/log/pacemaker/bundles@:>@ ],
     [ CRM_BUNDLE_DIR="$withval" ]
 )
 
 dnl ===============================================
 dnl General Processing
 dnl ===============================================
 
 INIT_EXT=""
 echo Our Host OS: $host_os/$host
 
 AC_MSG_NOTICE(Sanitizing prefix: ${prefix})
 case $prefix in
     NONE)
         prefix=/usr
         dnl Fix default variables - "prefix" variable if not specified
         if test "$localstatedir" = "\${prefix}/var"; then
             localstatedir="/var"
         fi
         if test "$sysconfdir" = "\${prefix}/etc"; then
             sysconfdir="/etc"
         fi
         ;;
 esac
 
 AC_MSG_NOTICE(Sanitizing exec_prefix: ${exec_prefix})
 case $exec_prefix in
     prefix|NONE)
         exec_prefix=$prefix
         ;;
 esac
 
 AC_MSG_NOTICE(Sanitizing INITDIR: ${INITDIR})
 case $INITDIR in
     prefix) INITDIR=$prefix;;
     "")
         AC_MSG_CHECKING(which init (rc) directory to use)
         for initdir in /etc/init.d /etc/rc.d/init.d /sbin/init.d \
             /usr/local/etc/rc.d /etc/rc.d
         do
             if
                 test -d $initdir
             then
                 INITDIR=$initdir
                 break
             fi
         done
         AC_MSG_RESULT($INITDIR)
         ;;
 esac
 AC_SUBST(INITDIR)
 
 AC_MSG_NOTICE(Sanitizing libdir: ${libdir})
 case $libdir in
     prefix|NONE)
         AC_MSG_CHECKING(which lib directory to use)
         for aDir in lib64 lib
         do
             trydir="${exec_prefix}/${aDir}"
             if
                 test -d ${trydir}
             then
                 libdir=${trydir}
                 break
             fi
         done
         AC_MSG_RESULT($libdir);
         ;;
 esac
 
 dnl Expand autoconf variables so that we don't end up with '${prefix}'
 dnl in #defines and python scripts
 dnl NOTE: Autoconf deliberately leaves them unexpanded to allow
 dnl    make exec_prefix=/foo install
 dnl No longer being able to do this seems like no great loss to me...
 
 eval prefix="`eval echo ${prefix}`"
 eval exec_prefix="`eval echo ${exec_prefix}`"
 eval bindir="`eval echo ${bindir}`"
 eval sbindir="`eval echo ${sbindir}`"
 eval libexecdir="`eval echo ${libexecdir}`"
 eval datadir="`eval echo ${datadir}`"
 eval sysconfdir="`eval echo ${sysconfdir}`"
 eval sharedstatedir="`eval echo ${sharedstatedir}`"
 eval localstatedir="`eval echo ${localstatedir}`"
 eval libdir="`eval echo ${libdir}`"
 eval includedir="`eval echo ${includedir}`"
 eval oldincludedir="`eval echo ${oldincludedir}`"
 eval infodir="`eval echo ${infodir}`"
 eval mandir="`eval echo ${mandir}`"
 
 dnl Home-grown variables
 eval INITDIR="${INITDIR}"
 eval docdir="`eval echo ${docdir}`"
 if test x"${docdir}" = x""; then
     docdir=${datadir}/doc/${PACKAGE}-${VERSION}
 fi
 AC_SUBST(docdir)
 if test x"${CONFIGDIR}" = x""; then
     CONFIGDIR="${sysconfdir}/sysconfig"
 fi
 AC_SUBST(CONFIGDIR)
 
 if test x"${CRM_LOG_DIR}" = x""; then
     CRM_LOG_DIR="${localstatedir}/log/pacemaker"
 fi
 AC_DEFINE_UNQUOTED(CRM_LOG_DIR,"$CRM_LOG_DIR", Location for Pacemaker log file)
 AC_SUBST(CRM_LOG_DIR)
 
 if test x"${CRM_BUNDLE_DIR}" = x""; then
     CRM_BUNDLE_DIR="${localstatedir}/log/pacemaker/bundles"
 fi
 AC_DEFINE_UNQUOTED(CRM_BUNDLE_DIR,"$CRM_BUNDLE_DIR", Location for Pacemaker bundle logs)
 AC_SUBST(CRM_BUNDLE_DIR)
 
 for j in prefix exec_prefix bindir sbindir libexecdir datadir sysconfdir \
     sharedstatedir localstatedir libdir includedir oldincludedir infodir \
     mandir INITDIR docdir CONFIGDIR
 do
     dirname=`eval echo '${'${j}'}'`
     if
         test ! -d "$dirname"
     then
         AC_MSG_WARN([$j directory ($dirname) does not exist!])
     fi
 done
 
 dnl This OS-based decision-making is poor autotools practice;
 dnl feature-based mechanisms are strongly preferred.
 dnl
 dnl So keep this section to a bare minimum; regard as a "necessary evil".
 
 case "$host_os" in
     *bsd*)
         AC_DEFINE_UNQUOTED(ON_BSD, 1, Compiling for BSD platform)
         LIBS="-L/usr/local/lib"
         CPPFLAGS="$CPPFLAGS -I/usr/local/include"
         INIT_EXT=".sh"
         ;;
     *solaris*)
         AC_DEFINE_UNQUOTED(ON_SOLARIS, 1, Compiling for Solaris platform)
         ;;
     *linux*)
         AC_DEFINE_UNQUOTED(ON_LINUX, 1, Compiling for Linux platform)
         ;;
     darwin*)
         AC_DEFINE_UNQUOTED(ON_DARWIN, 1, Compiling for Darwin platform)
         LIBS="$LIBS -L${prefix}/lib"
         CFLAGS="$CFLAGS -I${prefix}/include"
         ;;
 esac
 
 AC_SUBST(INIT_EXT)
 AC_MSG_NOTICE(Host CPU: $host_cpu)
 
 case "$host_cpu" in
     ppc64|powerpc64)
         case $CFLAGS in
          *powerpc64*)
              ;;
          *)
              if test "$GCC" = yes; then
                  CFLAGS="$CFLAGS -m64"
              fi
              ;;
         esac
         ;;
 esac
 
 AC_MSG_CHECKING(which format is needed to print uint64_t)
 
 ac_save_CFLAGS=$CFLAGS
 CFLAGS="-Wall -Werror"
 
 AC_COMPILE_IFELSE(
     [AC_LANG_PROGRAM(
         [
 #include <stdio.h>
 #include <stdint.h>
 #include <stdlib.h>
         ],
         [
 int max = 512;
 uint64_t bignum = 42;
 char *buffer = malloc(max);
 const char *random = "random";
 snprintf(buffer, max-1, "<quorum id=%lu quorate=%s/>", bignum, random);
 fprintf(stderr, "Result: %s\n", buffer);
         ]
     )],
     [U64T="%lu"],
     [U64T="%llu"]
 )
 CFLAGS=$ac_save_CFLAGS
 
 AC_MSG_RESULT($U64T)
 AC_DEFINE_UNQUOTED(U64T, "$U64T", Correct printf format for logging uint64_t)
 
 dnl ===============================================
 dnl Program Paths
 dnl ===============================================
 
 PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin"
 export PATH
 
 dnl Replacing AC_PROG_LIBTOOL with AC_CHECK_PROG because LIBTOOL
 dnl was NOT being expanded all the time thus causing things to fail.
 AC_CHECK_PROGS(LIBTOOL, glibtool libtool libtool15 libtool13)
 
 dnl Pacemaker's executable python scripts will invoke the python specified by
 dnl configure's PYTHON variable. If not specified, AM_PATH_PYTHON will check a
 dnl built-in list with (unversioned) "python" having precedence. To configure
 dnl Pacemaker to use a specific python interpreter version, define PYTHON
 dnl when calling configure, for example: ./configure PYTHON=/usr/bin/python3.6
 case "x$PYTHON" in
     x*python3*)
         dnl When used with Python 3, Pacemaker requires a minimum of 3.2
         AM_PATH_PYTHON([3.2])
         ;;
     *)
         dnl Otherwise, Pacemaker requires a minimum of 2.7
         AM_PATH_PYTHON([2.7])
         ;;
 esac
 
 AC_CHECK_PROGS(MAKE, gmake make)
 AC_PATH_PROGS(HTML2TXT, lynx w3m)
 AC_PATH_PROGS(HELP2MAN, help2man)
 AC_PATH_PROGS(POD2MAN, pod2man, pod2man)
 AC_PATH_PROGS(ASCIIDOC, asciidoc)
 AC_PATH_PROGS(PUBLICAN, publican)
 AC_PATH_PROGS(INKSCAPE, inkscape)
 AC_PATH_PROGS(XSLTPROC, xsltproc)
 AC_PATH_PROGS(XMLCATALOG, xmlcatalog)
 AC_PATH_PROGS(FOP, fop)
 AC_PATH_PROGS(SSH, ssh, /usr/bin/ssh)
 AC_PATH_PROGS(SCP, scp, /usr/bin/scp)
 AC_PATH_PROGS(TAR, tar)
 AC_PATH_PROGS(MD5, md5)
 AC_PATH_PROGS(TEST, test)
 PKG_PROG_PKG_CONFIG
 AC_PATH_PROGS(XML2CONFIG, xml2-config)
 AC_PATH_PROGS(VALGRIND_BIN, valgrind, /usr/bin/valgrind)
 AC_DEFINE_UNQUOTED(VALGRIND_BIN, "$VALGRIND_BIN", Valgrind command)
 
 if test x"${LIBTOOL}" = x""; then
     AC_MSG_ERROR(You need (g)libtool installed in order to build ${PACKAGE})
 fi
 if test x"${MAKE}" = x""; then
     AC_MSG_ERROR(You need (g)make installed in order to build ${PACKAGE})
 fi
 
 AM_CONDITIONAL(BUILD_HELP, test x"${HELP2MAN}" != x"")
 if test x"${HELP2MAN}" != x""; then
     PCMK_FEATURES="$PCMK_FEATURES generated-manpages"
 fi
 
 MANPAGE_XSLT=""
 if test x"${XSLTPROC}" != x""; then
     AC_MSG_CHECKING(docbook to manpage transform)
     # first try to figure out correct template using xmlcatalog query,
     # resort to extensive (semi-deterministic) file search if that fails
     DOCBOOK_XSL_URI='http://docbook.sourceforge.net/release/xsl/current'
     DOCBOOK_XSL_PATH='manpages/docbook.xsl'
     MANPAGE_XSLT=$(${XMLCATALOG} "" ${DOCBOOK_XSL_URI}/${DOCBOOK_XSL_PATH} \
                    | sed -n 's|^file://||p;q')
     if test x"${MANPAGE_XSLT}" = x""; then
         DIRS=$(find "${datadir}" -name $(basename $(dirname ${DOCBOOK_XSL_PATH})) \
                -type d | LC_ALL=C sort)
         XSLT=$(basename ${DOCBOOK_XSL_PATH})
         for d in ${DIRS}; do
             if test -f "${d}/${XSLT}"; then
                  MANPAGE_XSLT="${d}/${XSLT}"
                  break
             fi
         done
     fi
 fi
 AC_MSG_RESULT($MANPAGE_XSLT)
 AC_SUBST(MANPAGE_XSLT)
 
 AM_CONDITIONAL(BUILD_XML_HELP, test x"${MANPAGE_XSLT}" != x"")
 if test x"${MANPAGE_XSLT}" != x""; then
     PCMK_FEATURES="$PCMK_FEATURES agent-manpages"
 fi
 
 AM_CONDITIONAL(BUILD_ASCIIDOC, test x"${ASCIIDOC}" != x"")
 if test x"${ASCIIDOC}" != x""; then
     PCMK_FEATURES="$PCMK_FEATURES ascii-docs"
 fi
 
 publican_intree_brand=no
 if test x"${PUBLICAN_BRAND}" != x"" \
    && test x"${PUBLICAN}" != x"" \
    && test x"${INKSCAPE}" != x""; then
 
     dnl special handling for clusterlabs brand (possibly in-tree version used)
     test "${PUBLICAN_BRAND}" != "clusterlabs" \
         || test -d /usr/share/publican/Common_Content/clusterlabs
     if test $? -ne 0; then
         dnl Unknown option: brand_dir vs. Option brand_dir requires an argument
         if ${PUBLICAN} build --brand_dir 2>&1 | grep -Eq 'brand_dir$'; then
             AC_MSG_WARN([Cannot use in-tree clusterlabs brand, resorting to common])
             PUBLICAN_BRAND=common
         else
             publican_intree_brand=yes
         fi
     fi
     AC_MSG_NOTICE([Enabling Publican-generated documentation using ${PUBLICAN_BRAND} brand])
     PCMK_FEATURES="$PCMK_FEATURES publican-docs"
 fi
 AM_CONDITIONAL([BUILD_DOCBOOK],
                [test x"${PUBLICAN_BRAND}" != x"" \
                 && test x"${PUBLICAN}" != x"" \
                 && test x"${INKSCAPE}" != x""])
 AM_CONDITIONAL([PUBLICAN_INTREE_BRAND],
                [test x"${publican_intree_brand}" = x"yes"])
 
 dnl ========================================================================
 dnl checks for library functions to replace them
 dnl
 dnl     NoSuchFunctionName:
 dnl             is a dummy function which no system supplies.  It is here to make
 dnl             the system compile semi-correctly on OpenBSD which doesn't know
 dnl             how to create an empty archive
 dnl
 dnl     scandir: Only on BSD.
 dnl             System-V systems may have it, but hidden and/or deprecated.
 dnl             A replacement function is supplied for it.
 dnl
 dnl     setenv: is some bsdish function that should also be avoided (use
 dnl             putenv instead)
 dnl             On the other hand, putenv doesn't provide the right API for the
 dnl             code and has memory leaks designed in (sigh...)  Fortunately this
 dnl             A replacement function is supplied for it.
 dnl
 dnl     strerror: returns a string that corresponds to an errno.
 dnl             A replacement function is supplied for it.
 dnl
 dnl     strnlen: is a gnu function similar to strlen, but safer.
 dnl            We wrote a tolearably-fast replacement function for it.
 dnl
 dnl     strndup: is a gnu function similar to strdup, but safer.
 dnl            We wrote a tolearably-fast replacement function for it.
 
 AC_REPLACE_FUNCS(alphasort NoSuchFunctionName scandir setenv strerror strchrnul unsetenv strnlen strndup)
 
 dnl ===============================================
 dnl Libraries
 dnl ===============================================
 AC_CHECK_LIB(socket, socket)                    dnl -lsocket
 AC_CHECK_LIB(c, dlopen)                         dnl if dlopen is in libc...
 AC_CHECK_LIB(dl, dlopen)                        dnl -ldl (for Linux)
 AC_CHECK_LIB(rt, sched_getscheduler)            dnl -lrt (for Tru64)
 AC_CHECK_LIB(gnugetopt, getopt_long)            dnl -lgnugetopt ( if available )
 AC_CHECK_LIB(pam, pam_start)                    dnl -lpam (if available)
 
 AC_CHECK_FUNCS([sched_setscheduler])
 
 AC_CHECK_LIB(uuid, uuid_parse)                  dnl load the library if necessary
 AC_CHECK_FUNCS(uuid_unparse)                    dnl OSX ships uuid_* as standard functions
 
 AC_CHECK_HEADERS(uuid/uuid.h)
 
 if test "x$ac_cv_func_uuid_unparse" != xyes; then
     AC_MSG_ERROR(You do not have the libuuid development package installed)
 fi
 
 if test x"${PKG_CONFIG}" = x""; then
     AC_MSG_ERROR(You need pkgconfig installed in order to build ${PACKAGE})
 fi
 
 if
     $PKG_CONFIG --exists glib-2.0
 then
     GLIBCONFIG="$PKG_CONFIG glib-2.0"
 else
     set -x
     echo PKG_CONFIG_PATH=$PKG_CONFIG_PATH
     $PKG_CONFIG --exists glib-2.0; echo $?
     $PKG_CONFIG --cflags glib-2.0; echo $?
     $PKG_CONFIG glib-2.0; echo $?
     set +x
     AC_MSG_ERROR(You need glib2-devel installed in order to build ${PACKAGE})
 fi
 AC_MSG_RESULT(using $GLIBCONFIG)
 
 #
 # Where is dlopen?
 #
 if test "$ac_cv_lib_c_dlopen" = yes; then
     LIBADD_DL=""
 elif test "$ac_cv_lib_dl_dlopen" = yes; then
     LIBADD_DL=-ldl
 else
     LIBADD_DL=${lt_cv_dlopen_libs}
 fi
 
 if test "X$GLIBCONFIG" != X; then
     AC_MSG_CHECKING(for special glib includes: )
     GLIBHEAD=`$GLIBCONFIG --cflags`
     AC_MSG_RESULT($GLIBHEAD)
     CPPFLAGS="$CPPFLAGS $GLIBHEAD"
 
     AC_MSG_CHECKING(for glib library flags)
     GLIBLIB=`$GLIBCONFIG --libs`
     AC_MSG_RESULT($GLIBLIB)
     LIBS="$LIBS $GLIBLIB"
 fi
 
 dnl FreeBSD needs -lcompat for ftime() used by lrmd.c
 AC_CHECK_LIB([compat], [ftime], [COMPAT_LIBS='-lcompat'])
 AC_SUBST(COMPAT_LIBS)
 
 dnl ========================================================================
 dnl Headers
 dnl ========================================================================
 
 AC_HEADER_STDC
 AC_CHECK_HEADERS(arpa/inet.h)
 AC_CHECK_HEADERS(ctype.h)
 AC_CHECK_HEADERS(dirent.h)
 AC_CHECK_HEADERS(errno.h)
 AC_CHECK_HEADERS(getopt.h)
 AC_CHECK_HEADERS(glib.h)
 AC_CHECK_HEADERS(grp.h)
 AC_CHECK_HEADERS(limits.h)
 AC_CHECK_HEADERS(linux/swab.h)
 AC_CHECK_HEADERS(malloc.h)
 AC_CHECK_HEADERS(netdb.h)
 AC_CHECK_HEADERS(netinet/in.h)
 AC_CHECK_HEADERS(netinet/ip.h)
 AC_CHECK_HEADERS(pwd.h)
 AC_CHECK_HEADERS(sgtty.h)
 AC_CHECK_HEADERS(signal.h)
 AC_CHECK_HEADERS(stdarg.h)
 AC_CHECK_HEADERS(stddef.h)
 AC_CHECK_HEADERS(stdio.h)
 AC_CHECK_HEADERS(stdlib.h)
 AC_CHECK_HEADERS(string.h)
 AC_CHECK_HEADERS(strings.h)
 AC_CHECK_HEADERS(sys/dir.h)
 AC_CHECK_HEADERS(sys/ioctl.h)
 AC_CHECK_HEADERS(sys/param.h)
 AC_CHECK_HEADERS(sys/reboot.h)
 AC_CHECK_HEADERS(sys/resource.h)
 AC_CHECK_HEADERS(sys/socket.h)
 AC_CHECK_HEADERS(sys/signalfd.h)
 AC_CHECK_HEADERS(sys/sockio.h)
 AC_CHECK_HEADERS(sys/stat.h)
 AC_CHECK_HEADERS(sys/time.h)
 AC_CHECK_HEADERS(sys/timeb.h)
 AC_CHECK_HEADERS(sys/types.h)
 AC_CHECK_HEADERS(sys/utsname.h)
 AC_CHECK_HEADERS(sys/wait.h)
 AC_CHECK_HEADERS(time.h)
 AC_CHECK_HEADERS(unistd.h)
 
 dnl These headers need prerequisites before the tests will pass
 dnl AC_CHECK_HEADERS(net/if.h)
 
 AC_MSG_CHECKING(for special libxml2 includes)
 if test "x$XML2CONFIG" = "x"; then
     AC_MSG_ERROR(libxml2 config not found)
 else
     XML2HEAD="`$XML2CONFIG --cflags`"
     AC_MSG_RESULT($XML2HEAD)
     AC_CHECK_LIB(xml2, xmlReadMemory)
     AC_CHECK_LIB(xslt, xsltApplyStylesheet)
 fi
 
 CPPFLAGS="$CPPFLAGS $XML2HEAD"
 
 AC_CHECK_HEADERS(libxml/xpath.h)
 AC_CHECK_HEADERS(libxslt/xslt.h)
 if test "$ac_cv_header_libxml_xpath_h" != "yes"; then
     AC_MSG_ERROR(The libxml developement headers were not found)
 fi
 if test "$ac_cv_header_libxslt_xslt_h" != "yes"; then
     AC_MSG_ERROR(The libxslt developement headers were not found)
 fi
 
 AC_CACHE_CHECK(whether __progname and __progname_full are available,
                pf_cv_var_progname,
                AC_TRY_LINK([extern char *__progname, *__progname_full;],
                            [__progname = "foo"; __progname_full = "foo bar";],
                            pf_cv_var_progname="yes", pf_cv_var_progname="no"))
 
 if test "$pf_cv_var_progname" = "yes"; then
     AC_DEFINE(HAVE___PROGNAME,1,[ ])
 fi
 
 dnl ========================================================================
 dnl Structures
 dnl ========================================================================
 
 AC_CHECK_MEMBERS([struct tm.tm_gmtoff],,,[[#include <time.h>]])
 AC_CHECK_MEMBERS([lrm_op_t.rsc_deleted],,,[[#include <lrm/lrm_api.h>]])
 AC_CHECK_MEMBER([struct dirent.d_type],
     AC_DEFINE(HAVE_STRUCT_DIRENT_D_TYPE,1,[Define this if struct dirent has d_type]),,
     [#include <dirent.h>])
 
 dnl ========================================================================
 dnl Functions
 dnl ========================================================================
 
 AC_CHECK_FUNCS(getopt, AC_DEFINE(HAVE_DECL_GETOPT,  1, [Have getopt function]))
 AC_CHECK_FUNCS(nanosleep, AC_DEFINE(HAVE_DECL_NANOSLEEP,  1, [Have nanosleep function]))
 
 dnl ========================================================================
 dnl   bzip2
 dnl ========================================================================
 AC_CHECK_HEADERS(bzlib.h)
 AC_CHECK_LIB(bz2, BZ2_bzBuffToBuffCompress)
 
 if test x$ac_cv_lib_bz2_BZ2_bzBuffToBuffCompress != xyes ; then
     AC_MSG_ERROR(BZ2 libraries not found)
 fi
 
 if test x$ac_cv_header_bzlib_h != xyes; then
     AC_MSG_ERROR(BZ2 Development headers not found)
 fi
 
 dnl ========================================================================
 dnl sighandler_t is missing from Illumos, Solaris11 systems
 dnl ========================================================================
 
 AC_MSG_CHECKING([for sighandler_t])
 AC_TRY_COMPILE([#include <signal.h>],[sighandler_t *f;],
 has_sighandler_t=yes,has_sighandler_t=no)
 AC_MSG_RESULT($has_sighandler_t)
 if test "$has_sighandler_t" = "yes" ; then
     AC_DEFINE( HAVE_SIGHANDLER_T, 1, [Define if sighandler_t available] )
 fi
 
 dnl ========================================================================
 dnl   ncurses
 dnl ========================================================================
 dnl
 dnl A few OSes (e.g. Linux) deliver a default "ncurses" alongside "curses".
 dnl Many non-Linux deliver "curses"; sites may add "ncurses".
 dnl
 dnl However, the source-code recommendation for both is to #include "curses.h"
 dnl (i.e. "ncurses" still wants the include to be simple, no-'n', "curses.h").
 dnl
 dnl ncurse takes precedence.
 dnl
 AC_CHECK_HEADERS(curses.h)
 AC_CHECK_HEADERS(curses/curses.h)
 AC_CHECK_HEADERS(ncurses.h)
 AC_CHECK_HEADERS(ncurses/ncurses.h)
 
 dnl Although n-library is preferred, only look for it if the n-header was found.
 CURSESLIBS=''
 if test "$ac_cv_header_ncurses_h" = "yes"; then
     AC_CHECK_LIB(ncurses, printw,
                  [AC_DEFINE(HAVE_LIBNCURSES,1, have ncurses library)])
     CURSESLIBS=`$PKG_CONFIG --libs ncurses` || CURSESLIBS='-lncurses'
 fi
 
 if test "$ac_cv_header_ncurses_ncurses_h" = "yes"; then
     AC_CHECK_LIB(ncurses, printw,
                  [AC_DEFINE(HAVE_LIBNCURSES,1, have ncurses library)])
     CURSESLIBS=`$PKG_CONFIG --libs ncurses` || CURSESLIBS='-lncurses'
 fi
 
 dnl Only look for non-n-library if there was no n-library.
 if test X"$CURSESLIBS" = X"" -a "$ac_cv_header_curses_h" = "yes"; then
     AC_CHECK_LIB(curses, printw,
                  [CURSESLIBS='-lcurses'; AC_DEFINE(HAVE_LIBCURSES,1, have curses library)])
 fi
 
 dnl Only look for non-n-library if there was no n-library.
 if test X"$CURSESLIBS" = X"" -a "$ac_cv_header_curses_curses_h" = "yes"; then
     AC_CHECK_LIB(curses, printw,
                  [CURSESLIBS='-lcurses'; AC_DEFINE(HAVE_LIBCURSES,1, have curses library)])
 fi
 
 if test "x$CURSESLIBS" != "x"; then
     PCMK_FEATURES="$PCMK_FEATURES ncurses"
 fi
 
 dnl Check for printw() prototype compatibility
 if test X"$CURSESLIBS" != X"" && cc_supports_flag -Wcast-qual && cc_supports_flag -Werror; then
     ac_save_LIBS=$LIBS
     LIBS="$CURSESLIBS"
     ac_save_CFLAGS=$CFLAGS
     CFLAGS="-Wcast-qual -Werror"
     # avoid broken test because of hardened build environment in Fedora 23+
     # - https://fedoraproject.org/wiki/Changes/Harden_All_Packages
     # - https://bugzilla.redhat.com/1297985
     if cc_supports_flag -fPIC; then
         CFLAGS="$CFLAGS -fPIC"
     fi
 
     AC_MSG_CHECKING(whether printw() requires argument of "const char *")
     AC_LINK_IFELSE(
         [AC_LANG_PROGRAM([
 #if defined(HAVE_NCURSES_H)
 #  include <ncurses.h>
 #elif defined(HAVE_NCURSES_NCURSES_H)
 #  include <ncurses/ncurses.h>
 #elif defined(HAVE_CURSES_H)
 #  include <curses.h>
 #endif
                          ],
                          [printw((const char *)"Test");]
         )],
         [ac_cv_compatible_printw=yes],
         [ac_cv_compatible_printw=no]
     )
 
     LIBS=$ac_save_LIBS
     CFLAGS=$ac_save_CFLAGS
 
     AC_MSG_RESULT([$ac_cv_compatible_printw])
 
     if test "$ac_cv_compatible_printw" = no; then
         AC_MSG_WARN([The printw() function of your ncurses or curses library is old, we will disable usage of the library. If you want to use this library anyway, please update to newer version of the library, ncurses 5.4 or later is recommended. You can get the library from http://www.gnu.org/software/ncurses/.])
         AC_MSG_NOTICE([Disabling curses])
         AC_DEFINE(HAVE_INCOMPATIBLE_PRINTW, 1, [Do we have incompatible printw() in curses library?])
     fi
 fi
 
 AC_SUBST(CURSESLIBS)
 
 dnl ========================================================================
 dnl    Profiling and GProf
 dnl ========================================================================
 
 AC_MSG_NOTICE(Old CFLAGS: $CFLAGS)
 case $SUPPORT_COVERAGE in
     1|yes|true)
         SUPPORT_PROFILING=1
         PCMK_FEATURES="$PCMK_FEATURES coverage"
         CFLAGS="$CFLAGS -fprofile-arcs -ftest-coverage"
         dnl During linking, make sure to specify -lgcov or -coverage
         ;;
 esac
 
 case $SUPPORT_PROFILING in
     1|yes|true)
         SUPPORT_PROFILING=1
 
         dnl Disable various compiler optimizations
         CFLAGS="$CFLAGS -fno-omit-frame-pointer -fno-inline -fno-builtin "
         dnl CFLAGS="$CFLAGS -fno-inline-functions -fno-default-inline -fno-inline-functions-called-once -fno-optimize-sibling-calls"
 
         dnl Turn off optimization so tools can get accurate line numbers
         CFLAGS=`echo $CFLAGS | sed -e 's/-O.\ //g' -e 's/-Wp,-D_FORTIFY_SOURCE=.\ //g' -e 's/-D_FORTIFY_SOURCE=.\ //g'`
         CFLAGS="$CFLAGS -O0 -g3 -gdwarf-2"
 
         dnl Update features
         PCMK_FEATURES="$PCMK_FEATURES profile"
         ;;
      *)
         SUPPORT_PROFILING=0
         ;;
 esac
 AC_MSG_NOTICE(New CFLAGS: $CFLAGS)
 AC_DEFINE_UNQUOTED(SUPPORT_PROFILING, $SUPPORT_PROFILING, Support for profiling)
 
 dnl ========================================================================
 dnl    Cluster infrastructure - LibQB
 dnl ========================================================================
 
 if test x${enable_no_stack} = xyes; then
     SUPPORT_CS=no
 fi
 
 PKG_CHECK_MODULES(libqb, libqb >= 0.13)
 CPPFLAGS="$libqb_CFLAGS $CPPFLAGS"
 LIBS="$libqb_LIBS $LIBS"
 
 dnl libqb 0.14.0+ (2012-06)
 AC_CHECK_LIB(qb, qb_ipcs_connection_auth_set)
 
 PCMK_FEATURES="$PCMK_FEATURES libqb-logging libqb-ipc"
 
 dnl libqb 0.17.0+ (2014-02)
 AC_CHECK_FUNCS(qb_ipcs_connection_get_buffer_size,
                AC_DEFINE(HAVE_IPCS_GET_BUFFER_SIZE, 1,
                          [Have qb_ipcc_get_buffer_size function]))
 
 dnl Support Linux-HA fence agents if available
 if test "$cross_compiling" != "yes"; then
     CPPFLAGS="$CPPFLAGS -I${prefix}/include/heartbeat"
 fi
 AC_CHECK_HEADERS(stonith/stonith.h)
 if test "$ac_cv_header_stonith_stonith_h" = "yes";  then
     dnl On Debian, AC_CHECK_LIBS fail if a library has any unresolved symbols
     dnl So check for all the dependencies (so they're added to LIBS) before checking for -lplumb
     AC_CHECK_LIB(pils, PILLoadPlugin)
     AC_CHECK_LIB(plumb, G_main_add_IPC_Channel)
     PCMK_FEATURES="$PCMK_FEATURES lha-fencing"
 fi
 
 dnl ===============================================
 dnl Variables needed for substitution
 dnl ===============================================
 CRM_SCHEMA_DIRECTORY="${datadir}/pacemaker"
 AC_DEFINE_UNQUOTED(CRM_SCHEMA_DIRECTORY,"$CRM_SCHEMA_DIRECTORY", Location for the Pacemaker Relax-NG Schema)
 AC_SUBST(CRM_SCHEMA_DIRECTORY)
 
 CRM_CORE_DIR="${localstatedir}/lib/pacemaker/cores"
 AC_DEFINE_UNQUOTED(CRM_CORE_DIR,"$CRM_CORE_DIR", Location to store core files produced by Pacemaker daemons)
 AC_SUBST(CRM_CORE_DIR)
 
 CRM_DAEMON_USER="hacluster"
 AC_DEFINE_UNQUOTED(CRM_DAEMON_USER,"$CRM_DAEMON_USER", User to run Pacemaker daemons as)
 AC_SUBST(CRM_DAEMON_USER)
 
 CRM_DAEMON_GROUP="haclient"
 AC_DEFINE_UNQUOTED(CRM_DAEMON_GROUP,"$CRM_DAEMON_GROUP", Group to run Pacemaker daemons as)
 AC_SUBST(CRM_DAEMON_GROUP)
 
 CRM_STATE_DIR=${localstatedir}/run/crm
 AC_DEFINE_UNQUOTED(CRM_STATE_DIR,"$CRM_STATE_DIR", Where to keep state files and sockets)
 AC_SUBST(CRM_STATE_DIR)
 
 CRM_PACEMAKER_DIR=${localstatedir}/lib/pacemaker
 AC_DEFINE_UNQUOTED(CRM_PACEMAKER_DIR,"$CRM_PACEMAKER_DIR", Location to store directory produced by Pacemaker daemons)
 AC_SUBST(CRM_PACEMAKER_DIR)
 
 CRM_BLACKBOX_DIR=${localstatedir}/lib/pacemaker/blackbox
 AC_DEFINE_UNQUOTED(CRM_BLACKBOX_DIR,"$CRM_BLACKBOX_DIR", Where to keep blackbox dumps)
 AC_SUBST(CRM_BLACKBOX_DIR)
 
 PE_STATE_DIR="${localstatedir}/lib/pacemaker/pengine"
 AC_DEFINE_UNQUOTED(PE_STATE_DIR,"$PE_STATE_DIR", Where to keep PEngine outputs)
 AC_SUBST(PE_STATE_DIR)
 
 CRM_CONFIG_DIR="${localstatedir}/lib/pacemaker/cib"
 AC_DEFINE_UNQUOTED(CRM_CONFIG_DIR,"$CRM_CONFIG_DIR", Where to keep configuration files)
 AC_SUBST(CRM_CONFIG_DIR)
 
 CRM_CONFIG_CTS="${localstatedir}/lib/pacemaker/cts"
 AC_DEFINE_UNQUOTED(CRM_CONFIG_CTS,"$CRM_CONFIG_CTS", Where to keep cts stateful data)
 AC_SUBST(CRM_CONFIG_CTS)
 
 CRM_DAEMON_DIR="${libexecdir}/pacemaker"
 AC_DEFINE_UNQUOTED(CRM_DAEMON_DIR,"$CRM_DAEMON_DIR", Location for Pacemaker daemons)
 AC_SUBST(CRM_DAEMON_DIR)
 
 HA_STATE_DIR="${localstatedir}/run"
 AC_DEFINE_UNQUOTED(HA_STATE_DIR,"$HA_STATE_DIR", Where sbd keeps its PID file)
 AC_SUBST(HA_STATE_DIR)
 
 CRM_RSCTMP_DIR="${localstatedir}/run/resource-agents"
 AC_DEFINE_UNQUOTED(CRM_RSCTMP_DIR,"$CRM_RSCTMP_DIR", Where resource agents should keep state files)
 AC_SUBST(CRM_RSCTMP_DIR)
 
 OCF_ROOT_DIR="/usr/lib/ocf"
 if test "X$OCF_ROOT_DIR" = X; then
     AC_MSG_ERROR(Could not locate OCF directory)
 fi
 AC_SUBST(OCF_ROOT_DIR)
 
 OCF_RA_DIR="$OCF_ROOT_DIR/resource.d"
 AC_DEFINE_UNQUOTED(OCF_RA_DIR,"$OCF_RA_DIR", Location for OCF RAs)
 AC_SUBST(OCF_RA_DIR)
 
 RH_STONITH_DIR="$sbindir"
 AC_DEFINE_UNQUOTED(RH_STONITH_DIR,"$RH_STONITH_DIR", Location for Red Hat Stonith agents)
 AC_DEFINE_UNQUOTED(SBIN_DIR,"$sbindir", Location for system binaries)
 
 RH_STONITH_PREFIX="fence_"
 AC_DEFINE_UNQUOTED(RH_STONITH_PREFIX,"$RH_STONITH_PREFIX", Prefix for Red Hat Stonith agents)
 
 AC_PATH_PROGS(GIT, git false)
 AC_MSG_CHECKING(build version)
 
 BUILD_VERSION=$Format:%h$
 if test $BUILD_VERSION != ":%h$"; then
     AC_MSG_RESULT(archive hash: $BUILD_VERSION)
 elif test -x $GIT -a -d .git; then
     BUILD_VERSION=`$GIT log --pretty="format:%h" -n 1`
     AC_MSG_RESULT(git hash: $BUILD_VERSION)
 else
     # The current directory name make a reasonable default
     # Most generated archives will include the hash or tag
     BASE=`basename $PWD`
     BUILD_VERSION=`echo $BASE | sed s:.*[[Pp]]acemaker-::`
     AC_MSG_RESULT(directory based hash: $BUILD_VERSION)
 fi
 
 AC_DEFINE_UNQUOTED(BUILD_VERSION, "$BUILD_VERSION", Build version)
 AC_SUBST(BUILD_VERSION)
 
 HAVE_dbus=1
 HAVE_upstart=0
 HAVE_systemd=0
 PKG_CHECK_MODULES(DBUS, dbus-1, ,HAVE_dbus=0)
 
 AC_DEFINE_UNQUOTED(SUPPORT_DBUS, $HAVE_dbus, Support dbus)
 AM_CONDITIONAL(BUILD_DBUS, test $HAVE_dbus = 1)
 
 if test $HAVE_dbus = 1; then
     CFLAGS="$CFLAGS `$PKG_CONFIG --cflags dbus-1`"
 fi
 
 DBUS_LIBS="$CFLAGS `$PKG_CONFIG --libs dbus-1`"
 AC_SUBST(DBUS_LIBS)
 
 AC_CHECK_TYPES([DBusBasicValue],,,[[#include <dbus/dbus.h>]])
 
 if test "x${enable_systemd}" != xno; then
     if test $HAVE_dbus = 0; then
         if test "x${enable_systemd}" = xyes; then
             AC_MSG_FAILURE([cannot enable systemd without DBus])
         else
             enable_systemd=no
         fi
     fi
     if test "x${enable_systemd}" = xtry; then
         AC_MSG_CHECKING([for systemd version query result via dbus-send])
         ret=$({ dbus-send --system --print-reply \
                     --dest=org.freedesktop.systemd1 \
                     /org/freedesktop/systemd1 \
                     org.freedesktop.DBus.Properties.Get \
                     string:org.freedesktop.systemd1.Manager \
                     string:Version 2>/dev/null \
                 || echo "this borked"; } | tail -n1)
         # sanitize output a bit (interested just in value, not type),
         # ret is intentionally unenquoted so as to normalize whitespace
         ret=$(echo ${ret} | cut -d' ' -f2-)
         AC_MSG_RESULT([${ret}])
         if test "x${ret}" != xborked \
            || systemctl --version 2>/dev/null | grep -q systemd; then
             enable_systemd=yes
         else
             enable_systemd=no
         fi
     fi
 fi
 
 AC_MSG_CHECKING([whether to enable support for managing resources via systemd])
 AC_MSG_RESULT([${enable_systemd}])
 if test "x${enable_systemd}" = xyes; then
     HAVE_systemd=1
     PCMK_FEATURES="$PCMK_FEATURES systemd"
 
     AC_MSG_CHECKING([for systemd path for system unit files])
     systemdunitdir="${systemdunitdir-}"
     PKG_CHECK_VAR([systemdunitdir], [systemd],
                   [systemdsystemunitdir], [], [systemdunitdir=no])
     AC_MSG_RESULT([${systemdunitdir}])
     if test "x${systemdunitdir}" = xno; then
         AC_MSG_FAILURE([cannot enable systemd when systemdunitdir unresolved])
     fi
 fi
 
 AC_DEFINE_UNQUOTED(SUPPORT_SYSTEMD, $HAVE_systemd, Support systemd based system services)
 AM_CONDITIONAL(BUILD_SYSTEMD, test $HAVE_systemd = 1)
 AC_SUBST(SUPPORT_SYSTEMD)
 
 if test "x${enable_upstart}" != xno; then
     if test $HAVE_dbus = 0; then
         if test "x${enable_upstart}" = xyes; then
             AC_MSG_FAILURE([cannot enable Upstart without DBus])
         else
             enable_upstart=no
         fi
     fi
     if test "x${enable_upstart}" = xtry; then
         AC_MSG_CHECKING([for Upstart version query result via dbus-send])
         ret=$({ dbus-send --system --print-reply --dest=com.ubuntu.Upstart \
                     /com/ubuntu/Upstart org.freedesktop.DBus.Properties.Get \
                     string:com.ubuntu.Upstart0_6 string:version 2>/dev/null \
                 || echo "this borked"; } | tail -n1)
         # sanitize output a bit (interested just in value, not type),
         # ret is intentionally unenquoted so as to normalize whitespace
         ret=$(echo ${ret} | cut -d' ' -f2-)
         AC_MSG_RESULT([${ret}])
         if test "x${ret}" != xborked \
            || initctl --version 2>/dev/null | grep -q upstart; then
             enable_upstart=yes
         else
             enable_upstart=no
         fi
     fi
 fi
 AC_MSG_CHECKING([whether to enable support for managing resources via Upstart])
 AC_MSG_RESULT([${enable_upstart}])
 if test "x${enable_upstart}" = xyes; then
     HAVE_upstart=1
     PCMK_FEATURES="$PCMK_FEATURES upstart"
 fi
 
 AC_DEFINE_UNQUOTED(SUPPORT_UPSTART, $HAVE_upstart, Support upstart based system services)
 AM_CONDITIONAL(BUILD_UPSTART, test $HAVE_upstart = 1)
 AC_SUBST(SUPPORT_UPSTART)
 
 case $SUPPORT_NAGIOS in
     1|yes|true|try)
         SUPPORT_NAGIOS=1
         ;;
     *)
         SUPPORT_NAGIOS=0
         ;;
 esac
 
 if test $SUPPORT_NAGIOS = 1; then
     PCMK_FEATURES="$PCMK_FEATURES nagios"
 fi
 
 AC_DEFINE_UNQUOTED(SUPPORT_NAGIOS, $SUPPORT_NAGIOS, Support nagios plugins)
 AM_CONDITIONAL(BUILD_NAGIOS, test $SUPPORT_NAGIOS = 1)
 
 if test x"$NAGIOS_PLUGIN_DIR" = x""; then
     NAGIOS_PLUGIN_DIR="${libexecdir}/nagios/plugins"
 fi
 
 AC_DEFINE_UNQUOTED(NAGIOS_PLUGIN_DIR, "$NAGIOS_PLUGIN_DIR", Directory for nagios plugins)
 AC_SUBST(NAGIOS_PLUGIN_DIR)
 
 if test x"$NAGIOS_METADATA_DIR" = x""; then
     NAGIOS_METADATA_DIR="${datadir}/nagios/plugins-metadata"
 fi
 
 AC_DEFINE_UNQUOTED(NAGIOS_METADATA_DIR, "$NAGIOS_METADATA_DIR", Directory for nagios plugins metadata)
 AC_SUBST(NAGIOS_METADATA_DIR)
 
 STACKS=""
 CLUSTERLIBS=""
 
 dnl ========================================================================
 dnl    Cluster stack - Corosync
 dnl ========================================================================
 
 dnl Normalize the values
 case $SUPPORT_CS in
     1|yes|true)
         SUPPORT_CS=yes
         missingisfatal=1
         ;;
     try)
         missingisfatal=0
         ;;
     *)
         SUPPORT_CS=no
         ;;
 esac
 
 AC_MSG_CHECKING(for native corosync)
 COROSYNC_LIBS=""
 
 if test $SUPPORT_CS = no; then
     AC_MSG_RESULT(no (disabled))
     SUPPORT_CS=0
 else
     AC_MSG_RESULT($SUPPORT_CS)
     SUPPORT_CS=1
     PKG_CHECK_MODULES(cpg,    libcpg) dnl Fatal
     PKG_CHECK_MODULES(cfg,    libcfg) dnl Fatal
     PKG_CHECK_MODULES(cmap,   libcmap) dnl Fatal
     PKG_CHECK_MODULES(quorum, libquorum) dnl Fatal
 
     CFLAGS="$CFLAGS $libqb_FLAGS $cpg_FLAGS $cfg_FLAGS $cmap_CFLAGS $quorum_CFLAGS"
     COROSYNC_LIBS="$COROSYNC_LIBS $libqb_LIBS $cpg_LIBS $cfg_LIBS $cmap_LIBS $quorum_LIBS"
     CLUSTERLIBS="$CLUSTERLIBS $COROSYNC_LIBS"
     AC_CHECK_LIB(corosync_common, cs_strerror)
     STACKS="$STACKS corosync-native"
 fi
 
 AC_DEFINE_UNQUOTED(SUPPORT_COROSYNC, $SUPPORT_CS,    Support the Corosync messaging and membership layer)
 AM_CONDITIONAL(BUILD_CS_SUPPORT, test $SUPPORT_CS = 1)
 AC_SUBST(SUPPORT_COROSYNC)
 
 dnl
 dnl    Cluster stack - Sanity
 dnl
 
 if test x${enable_no_stack} = xyes; then
     AC_MSG_NOTICE(No cluster stack supported.  Just building the Policy Engine)
     PCMK_FEATURES="$PCMK_FEATURES no-cluster-stack"
 else
     AC_MSG_CHECKING(for supported stacks)
     if test x"$STACKS" = x; then
         AC_MSG_FAILURE(You must support at least one cluster stack)
     fi
     AC_MSG_RESULT($STACKS)
     PCMK_FEATURES="$PCMK_FEATURES $STACKS"
 fi
 
 PCMK_FEATURES="$PCMK_FEATURES atomic-attrd"
 AC_SUBST(CLUSTERLIBS)
 
 dnl ========================================================================
 dnl    ACL
 dnl ========================================================================
 
 case $SUPPORT_ACL in
     1|yes|true)
         missingisfatal=1
         ;;
     try)
         missingisfatal=0
         ;;
     *)
         SUPPORT_ACL=no
         ;;
 esac
 
 AC_MSG_CHECKING(for acl support)
 if test $SUPPORT_ACL = no; then
     AC_MSG_RESULT(no (disabled))
     SUPPORT_ACL=0
 else
     AC_MSG_RESULT($SUPPORT_ACL)
 
     SUPPORT_ACL=1
     AC_CHECK_LIB(qb, qb_ipcs_connection_auth_set)
     if test $ac_cv_lib_qb_qb_ipcs_connection_auth_set != yes; then
         SUPPORT_ACL=0
     fi
 
     if test $SUPPORT_ACL = 0; then
         if test $missingisfatal = 0; then
             AC_MSG_WARN(Unable to support ACL. You need to use libqb > 0.13.0)
         else
             AC_MSG_FAILURE(Unable to support ACL. You need to use libqb > 0.13.0)
         fi
     fi
 fi
 
 if test $SUPPORT_ACL = 1; then
     PCMK_FEATURES="$PCMK_FEATURES acls"
 fi
 
 AM_CONDITIONAL(ENABLE_ACL, test "$SUPPORT_ACL" = "1")
 AC_DEFINE_UNQUOTED(ENABLE_ACL, $SUPPORT_ACL, Build in support for CIB ACL)
 
 dnl ========================================================================
 dnl    CIB secrets
 dnl ========================================================================
 
 case $SUPPORT_CIBSECRETS in
     1|yes|true|try)
         SUPPORT_CIBSECRETS=1
         ;;
     *)
         SUPPORT_CIBSECRETS=0
         ;;
 esac
 
 AC_DEFINE_UNQUOTED(SUPPORT_CIBSECRETS, $SUPPORT_CIBSECRETS, Support CIB secrets)
 AM_CONDITIONAL(BUILD_CIBSECRETS, test $SUPPORT_CIBSECRETS = 1)
 
 if test $SUPPORT_CIBSECRETS = 1; then
     PCMK_FEATURES="$PCMK_FEATURES cibsecrets"
 
     LRM_CIBSECRETS_DIR="${localstatedir}/lib/pacemaker/lrm/secrets"
     AC_DEFINE_UNQUOTED(LRM_CIBSECRETS_DIR,"$LRM_CIBSECRETS_DIR", Location for CIB secrets)
     AC_SUBST(LRM_CIBSECRETS_DIR)
 fi
 
 dnl ========================================================================
 dnl    GnuTLS
 dnl ========================================================================
 
 AC_CHECK_HEADERS(gnutls/gnutls.h)
 AC_CHECK_HEADERS(security/pam_appl.h pam/pam_appl.h)
 
 dnl GNUTLS library: Attempt to determine by 'libgnutls-config' program.
 dnl If no 'libgnutls-config', try traditional autoconf means.
 AC_PATH_PROGS(LIBGNUTLS_CONFIG, libgnutls-config)
 
 if test -n "$LIBGNUTLS_CONFIG"; then
     AC_MSG_CHECKING(for gnutls header flags)
     GNUTLSHEAD="`$LIBGNUTLS_CONFIG --cflags`";
     AC_MSG_RESULT($GNUTLSHEAD)
     AC_MSG_CHECKING(for gnutls library flags)
     GNUTLSLIBS="`$LIBGNUTLS_CONFIG --libs`";
     AC_MSG_RESULT($GNUTLSLIBS)
 fi
 AC_CHECK_LIB(gnutls, gnutls_init)
 AC_CHECK_FUNCS(gnutls_priority_set_direct)
 
 AC_SUBST(GNUTLSHEAD)
 AC_SUBST(GNUTLSLIBS)
 
 
 dnl ========================================================================
 dnl    System Health
 dnl ========================================================================
 
 dnl Check if servicelog development package is installed
 SERVICELOG=servicelog-1
 SERVICELOG_EXISTS="no"
 AC_MSG_CHECKING(for $SERVICELOG packages)
 if
     $PKG_CONFIG --exists $SERVICELOG
 then
     PKG_CHECK_MODULES([SERVICELOG], [servicelog-1])
     SERVICELOG_EXISTS="yes"
 fi
 AC_MSG_RESULT($SERVICELOG_EXISTS)
 AM_CONDITIONAL(BUILD_SERVICELOG, test "$SERVICELOG_EXISTS" = "yes")
 
 dnl Check if OpenIMPI packages and servicelog are installed
 OPENIPMI="OpenIPMI OpenIPMIposix"
 OPENIPMI_SERVICELOG_EXISTS="no"
 AC_MSG_CHECKING(for $SERVICELOG $OPENIPMI packages)
 if
     $PKG_CONFIG --exists $OPENIPMI $SERVICELOG
 then
     PKG_CHECK_MODULES([OPENIPMI_SERVICELOG],[OpenIPMI OpenIPMIposix])
     OPENIPMI_SERVICELOG_EXISTS="yes"
 fi
 AC_MSG_RESULT($OPENIPMI_SERVICELOG_EXISTS)
 AM_CONDITIONAL(BUILD_OPENIPMI_SERVICELOG, test "$OPENIPMI_SERVICELOG_EXISTS" = "yes")
 
 dnl ========================================================================
 dnl Compiler flags
 dnl ========================================================================
 
 dnl Make sure that CFLAGS is not exported. If the user did
 dnl not have CFLAGS in their environment then this should have
 dnl no effect. However if CFLAGS was exported from the user's
 dnl environment, then the new CFLAGS will also be exported
 dnl to sub processes.
 if export | fgrep " CFLAGS=" > /dev/null; then
     SAVED_CFLAGS="$CFLAGS"
     unset CFLAGS
     CFLAGS="$SAVED_CFLAGS"
     unset SAVED_CFLAGS
 fi
 
 AC_ARG_VAR([CFLAGS_HARDENED_LIB], [extra C compiler flags for hardened libraries])
 AC_ARG_VAR([LDFLAGS_HARDENED_LIB], [extra linker flags for hardened libraries])
 
 AC_ARG_VAR([CFLAGS_HARDENED_EXE], [extra C compiler flags for hardened executables])
 AC_ARG_VAR([LDFLAGS_HARDENED_EXE], [extra linker flags for hardened executables])
 
 CC_EXTRAS=""
 
 if test "$GCC" != yes; then
     CFLAGS="$CFLAGS -g"
     enable_fatal_warnings=no
 else
     CFLAGS="$CFLAGS -ggdb"
 
 dnl When we don't have diagnostic push / pull, we can't explicitly disable
 dnl checking for nonliteral formats in the places where they occur on purpose
 dnl thus we disable nonliteral format checking globally as we are aborting
 dnl on warnings. 
 dnl what makes the things really ugly is that nonliteral format checking is 
 dnl obviously available as an extra switch in very modern gcc but for older
 dnl gcc this is part of -Wformat=2 
 dnl so if we have push/pull we can enable -Wformat=2 -Wformat-nonliteral
 dnl if we don't have push/pull but -Wformat-nonliteral we can enable -Wformat=2
 dnl otherwise none of both
 
     gcc_diagnostic_push_pull=no
     SAVED_CFLAGS="$CFLAGS"
     CFLAGS="$CFLAGS -Werror"
     AC_MSG_CHECKING([for gcc diagnostic push / pull])
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #pragma GCC diagnostic push
 #pragma GCC diagnostic pop
                       ]])],
                       [
                           AC_MSG_RESULT([yes])
                           gcc_diagnostic_push_pull=yes
                       ], AC_MSG_RESULT([no]))
     CFLAGS="$SAVED_CFLAGS"
 
     if cc_supports_flag "-Wformat-nonliteral"; then
         gcc_format_nonliteral=yes
     else
         gcc_format_nonliteral=no
     fi
         
     # We had to eliminate -Wnested-externs because of libtool changes
     # Make sure to order options so that the former stand for prerequisites
     # of the latter (e.g., -Wformat-nonliteral requires -Wformat).
     EXTRA_FLAGS="-fgnu89-inline
                 -Wall
                 -Waggregate-return
                 -Wbad-function-cast
                 -Wcast-align
                 -Wdeclaration-after-statement
                 -Wendif-labels
                 -Wfloat-equal
                 -Wformat-security
                 -Wmissing-prototypes
                 -Wmissing-declarations
                 -Wnested-externs
                 -Wno-long-long
                 -Wno-strict-aliasing
                 -Wpointer-arith
                 -Wstrict-prototypes
                 -Wwrite-strings
                 -Wunused-but-set-variable
                 -Wunsigned-char"
 
     if test "x$gcc_diagnostic_push_pull" = "xyes"; then
         AC_DEFINE([GCC_FORMAT_NONLITERAL_CHECKING_ENABLED], [],
                   [gcc can complain about nonliterals in format])
         EXTRA_FLAGS="$EXTRA_FLAGS
                     -Wformat=2 
                     -Wformat-nonliteral"
     else
         if test "x$gcc_format_nonliteral" = "xyes"; then
             EXTRA_FLAGS="$EXTRA_FLAGS -Wformat=2"
         fi
     fi
 
 # Additional warnings it might be nice to enable one day
 #                -Wshadow
 #                -Wunreachable-code
     for j in $EXTRA_FLAGS
     do
         if
             cc_supports_flag $CC_EXTRAS $j
         then
             CC_EXTRAS="$CC_EXTRAS $j"
         fi
     done
 
 dnl System specific options
 
     case "$host_os" in
         *linux*|*bsd*)
             if test "${enable_fatal_warnings}" = "unknown"; then
                 enable_fatal_warnings=yes
             fi
             ;;
     esac
 
     if test "x${enable_fatal_warnings}" != xno && cc_supports_flag -Werror ; then
         enable_fatal_warnings=yes
     else
         enable_fatal_warnings=no
     fi
 
     if test "x${enable_ansi}" = xyes && cc_supports_flag -std=iso9899:199409 ; then
         AC_MSG_NOTICE(Enabling ANSI Compatibility)
         CC_EXTRAS="$CC_EXTRAS -ansi -D_GNU_SOURCE -DANSI_ONLY"
     fi
 
     AC_MSG_NOTICE(Activated additional gcc flags: ${CC_EXTRAS})
 fi
 
 dnl
 dnl Hardening flags
 dnl
 dnl The prime control of whether to apply (targeted) hardening build flags and
 dnl which ones is --{enable,disable}-hardening option passed to ./configure:
 dnl
 dnl --enable-hardening=try (default):
 dnl     depending on whether any of CFLAGS_HARDENED_EXE, LDFLAGS_HARDENED_EXE,
 dnl     CFLAGS_HARDENED_LIB or LDFLAGS_HARDENED_LIB environment variables
 dnl     (see below) is set and non-null, all these custom flags (even if not
 dnl     set) are used as are, otherwise the best effort is made to offer
 dnl     reasonably strong hardening in several categories (RELRO, PIE,
 dnl     "bind now", stack protector) according to what the selected toolchain
 dnl     can offer
 dnl
 dnl --enable-hardening:
 dnl     same effect as --enable-hardening=try when the environment variables
 dnl     in question are suppressed
 dnl
 dnl --disable-hardening:
 dnl     do not apply any targeted hardening measures at all
 dnl
 dnl The user-injected environment variables that regulate the hardening in
 dnl default case are as follows:
 dnl
 dnl * CFLAGS_HARDENED_EXE, LDFLAGS_HARDENED_EXE
 dnl    compiler and linker flags (respectively) for daemon programs
 dnl    (attrd, cib, crmd, lrmd, stonithd, pacemakerd, pacemaker_remoted,
 dnl    pengine)
 dnl
 dnl * CFLAGS_HARDENED_LIB, LDFLAGS_HARDENED_LIB
 dnl    compiler and linker flags (respectively) for libraries linked
 dnl    with the daemon programs
 dnl
 dnl Note that these are purposedly targeted variables (addressing particular
 dnl targets all over the scattered Makefiles) and have no effect outside of
 dnl the predestined scope (e.g., CLI utilities).  For a global reach,
 dnl use CFLAGS, LDFLAGS, etc. as usual.
 dnl
 dnl For guidance on the suitable flags consult, for instance:
 dnl https://fedoraproject.org/wiki/Changes/Harden_All_Packages#Detailed_Harden_Flags_Description
 dnl https://owasp.org/index.php/C-Based_Toolchain_Hardening#GCC.2FBinutils
 dnl
 
 if test "x${HARDENING}" != "xtry"; then
     unset CFLAGS_HARDENED_EXE
     unset CFLAGS_HARDENED_LIB
     unset LDFLAGS_HARDENED_EXE
     unset LDFLAGS_HARDENED_LIB
 fi
 if test "x${HARDENING}" = "xno"; then
     AC_MSG_NOTICE([Hardening: explicitly disabled])
 elif test "x${HARDENING}" = "xyes" \
      || test "$(env | grep -Ec '^(C|LD)FLAGS_HARDENED_(EXE|LIB)=.')" = 0; then
     dnl We'll figure out on our own...
     CFLAGS_HARDENED_EXE=
     CFLAGS_HARDENED_LIB=
     LDFLAGS_HARDENED_EXE=
     LDFLAGS_HARDENED_LIB=
     relro=0
     pie=0
     bindnow=0
     # daemons incl. libs: partial RELRO
     flag="-Wl,-z,relro"
     CC_CHECK_LDFLAGS(["${flag}"],
                      [LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}";
                       LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}";
                       relro=1])
     # daemons: PIE for both CFLAGS and LDFLAGS
     if cc_supports_flag -fPIE; then
         flag="-pie"
         CC_CHECK_LDFLAGS(["${flag}"],
                          [CFLAGS_HARDENED_EXE="${CFLAGS_HARDENED_EXE} -fPIE";
                           LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}";
                           pie=1])
     fi
     # daemons incl. libs: full RELRO if sensible + as-needed linking
     #                     so as to possibly mitigate startup performance
     #                     hit caused by excessive linking with unneeded
     #                     libraries
     if test "${relro}" = 1 && test "${pie}" = 1; then
         flag="-Wl,-z,now"
         CC_CHECK_LDFLAGS(["${flag}"],
                          [LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}";
                           LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}";
                           bindnow=1])
     fi
     if test "${bindnow}" = 1; then
         flag="-Wl,--as-needed"
         CC_CHECK_LDFLAGS(["${flag}"],
                          [LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}";
                           LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}"])
     fi
     # universal: prefer strong > all > default stack protector if possible
     flag=
     if cc_supports_flag -fstack-protector-strong; then
         flag="-fstack-protector-strong"
     elif cc_supports_flag -fstack-protector-all; then
         flag="-fstack-protector-all"
     elif cc_supports_flag -fstack-protector; then
         flag="-fstack-protector"
     fi
     if test -n "${flag}"; then
         CC_EXTRAS="${CC_EXTRAS} ${flag}"
         stackprot=1
     fi
     if test "${relro}" = 1 \
     || test "${pie}" = 1 \
     || test "${stackprot}" = 1; then
         AC_MSG_NOTICE([Hardening: relro=${relro} pie=${pie} bindnow=${bindnow} stackprot=${flag}])
     else
         AC_MSG_WARN([Hardening: no suitable features in the toolchain detected])
     fi
 else
     AC_MSG_NOTICE([Hardening: using custom flags])
 fi
 
 CFLAGS="$CFLAGS $CC_EXTRAS"
 
 NON_FATAL_CFLAGS="$CFLAGS"
 AC_SUBST(NON_FATAL_CFLAGS)
 
 dnl
 dnl We reset CFLAGS to include our warnings *after* all function
 dnl checking goes on, so that our warning flags don't keep the
 dnl AC_*FUNCS() calls above from working.  In particular, -Werror will
 dnl *always* cause us troubles if we set it before here.
 dnl
 dnl
 if test "x${enable_fatal_warnings}" = xyes ; then
     AC_MSG_NOTICE(Enabling Fatal Warnings)
     CFLAGS="$CFLAGS -Werror"
 fi
 AC_SUBST(CFLAGS)
 
 dnl This is useful for use in Makefiles that need to remove one specific flag
 CFLAGS_COPY="$CFLAGS"
 AC_SUBST(CFLAGS_COPY)
 
 AC_SUBST(LIBADD_DL)        dnl extra flags for dynamic linking libraries
 AC_SUBST(LIBADD_INTL)        dnl extra flags for GNU gettext stuff...
 
 AC_SUBST(LOCALE)
 
 dnl Options for cleaning up the compiler output
 QUIET_LIBTOOL_OPTS=""
 QUIET_MAKE_OPTS=""
 if test "x${enable_quiet}" = "xyes"; then
     QUIET_LIBTOOL_OPTS="--quiet"
     QUIET_MAKE_OPTS="--quiet"
 fi
 
 AC_MSG_RESULT(Supress make details: ${enable_quiet})
 
 dnl Put the above variables to use
 LIBTOOL="${LIBTOOL} --tag=CC \$(QUIET_LIBTOOL_OPTS)"
 MAKE="${MAKE} \$(QUIET_MAKE_OPTS)"
 
 AC_SUBST(CC)
 AC_SUBST(MAKE)
 AC_SUBST(LIBTOOL)
 AC_SUBST(QUIET_MAKE_OPTS)
 AC_SUBST(QUIET_LIBTOOL_OPTS)
 AC_DEFINE_UNQUOTED(CRM_FEATURES, "$PCMK_FEATURES", Set of enabled features)
 AC_SUBST(PCMK_FEATURES)
 
 dnl Files we output that need to be executable
 AC_CONFIG_FILES([cts/cts-coverage], [chmod +x cts/cts-coverage])
 AC_CONFIG_FILES([cts/cts-lrmd], [chmod +x cts/cts-lrmd])
+AC_CONFIG_FILES([cts/cts-pengine], [chmod +x cts/cts-pengine])
 AC_CONFIG_FILES([cts/cts-stonithd], [chmod +x cts/cts-stonithd])
 
 dnl Other files we output
 AC_CONFIG_FILES(Makefile                                            \
                 Doxyfile                                            \
                 cts/Makefile                                        \
                 cts/CTSlab.py                                       \
                 cts/CTSvars.py                                      \
                 cts/LSBDummy                                        \
 		cts/OCFIPraTest.py                                  \
                 cts/benchmark/Makefile                              \
                 cts/benchmark/clubench                              \
                 cts/fence_dummy                                     \
                 cts/lxc_autogen.sh                                  \
                 cib/Makefile                                        \
                 attrd/Makefile                                      \
                 crmd/Makefile                                       \
                 pengine/Makefile                                    \
-                pengine/regression.core.sh                          \
                 doc/Makefile                                        \
                 doc/Clusters_from_Scratch/publican.cfg              \
                 doc/Pacemaker_Administration/publican.cfg           \
                 doc/Pacemaker_Development/publican.cfg              \
                 doc/Pacemaker_Explained/publican.cfg                \
                 doc/Pacemaker_Remote/publican.cfg                   \
                 fencing/Makefile                                    \
                 include/Makefile                                    \
                 include/crm/Makefile                                \
                 include/crm/cib/Makefile                            \
                 include/crm/common/Makefile                         \
                 include/crm/cluster/Makefile                        \
                 include/crm/fencing/Makefile                        \
                 include/crm/pengine/Makefile                        \
                 replace/Makefile                                    \
                 lib/Makefile                                        \
                 lib/pacemaker.pc                                    \
                 lib/pacemaker-cib.pc                                \
                 lib/pacemaker-lrmd.pc                               \
                 lib/pacemaker-service.pc                            \
                 lib/pacemaker-pengine.pc                            \
                 lib/pacemaker-fencing.pc                            \
                 lib/pacemaker-cluster.pc                            \
                 lib/common/Makefile                                 \
                 lib/cluster/Makefile                                \
                 lib/cib/Makefile                                    \
                 lib/pengine/Makefile                                \
                 lib/transition/Makefile                             \
                 lib/fencing/Makefile                                \
                 lib/lrmd/Makefile                                   \
                 lib/services/Makefile                               \
                 mcp/Makefile                                        \
                 mcp/pacemaker                                       \
                 mcp/pacemaker.service                               \
                 mcp/pacemaker.upstart                               \
                 mcp/pacemaker.combined.upstart                      \
                 lrmd/Makefile                                       \
                 lrmd/pacemaker_remote.service                       \
                 lrmd/pacemaker_remote                               \
                 extra/Makefile                                      \
                 extra/alerts/Makefile                               \
                 extra/resources/Makefile                            \
                 extra/logrotate/Makefile                            \
                 extra/logrotate/pacemaker                           \
                 tools/Makefile                                      \
                 tools/crm_report                                    \
                 tools/report.common                                 \
                 tools/cibsecret                                     \
                 tools/crm_mon.service                               \
                 tools/crm_mon.upstart                               \
                 xml/Makefile                                        \
                 lib/gnu/Makefile                                    \
 )
 
 dnl Now process the entire list of files added by previous
 dnl  calls to AC_CONFIG_FILES()
 AC_OUTPUT()
 
 dnl *****************
 dnl Configure summary
 dnl *****************
 
 AC_MSG_RESULT([])
 AC_MSG_RESULT([$PACKAGE configuration:])
 AC_MSG_RESULT([  Version                  = ${VERSION} (Build: $BUILD_VERSION)])
 AC_MSG_RESULT([  Features                 =${PCMK_FEATURES}])
 AC_MSG_RESULT([])
 AC_MSG_RESULT([  Prefix                   = ${prefix}])
 AC_MSG_RESULT([  Executables              = ${sbindir}])
 AC_MSG_RESULT([  Man pages                = ${mandir}])
 AC_MSG_RESULT([  Libraries                = ${libdir}])
 AC_MSG_RESULT([  Header files             = ${includedir}])
 AC_MSG_RESULT([  Arch-independent files   = ${datadir}])
 AC_MSG_RESULT([  State information        = ${localstatedir}])
 AC_MSG_RESULT([  System configuration     = ${sysconfdir}])
 AC_MSG_RESULT([])
 AC_MSG_RESULT([  HA group name            = ${CRM_DAEMON_GROUP}])
 AC_MSG_RESULT([  HA user name             = ${CRM_DAEMON_USER}])
 AC_MSG_RESULT([])
 AC_MSG_RESULT([  CFLAGS                   = ${CFLAGS}])
 AC_MSG_RESULT([  CFLAGS_HARDENED_EXE      = ${CFLAGS_HARDENED_EXE}])
 AC_MSG_RESULT([  CFLAGS_HARDENED_LIB      = ${CFLAGS_HARDENED_LIB}])
 AC_MSG_RESULT([  LDFLAGS_HARDENED_EXE     = ${LDFLAGS_HARDENED_EXE}])
 AC_MSG_RESULT([  LDFLAGS_HARDENED_LIB     = ${LDFLAGS_HARDENED_LIB}])
 AC_MSG_RESULT([  Libraries                = ${LIBS}])
 AC_MSG_RESULT([  Stack Libraries          = ${CLUSTERLIBS}])
diff --git a/cts/Makefile.am b/cts/Makefile.am
index 5d5b28404b..b1f9b25ddc 100644
--- a/cts/Makefile.am
+++ b/cts/Makefile.am
@@ -1,80 +1,83 @@
 #
 # Copyright (C) 2001 Michael Moerz
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 MAINTAINERCLEANFILES    = Makefile.in
 
 CLEANFILES      = CTSlab.py LSBDummy OCFIPraTest.py fence_dummy
 
 EXTRA_DIST      = $(cts_SCRIPTS) $(cts_DATA)
 
 #
 # Test commands and globally applicable test files should be in $(testdir),
 # and command-specific test data should be in a command-specific subdirectory.
 #
 testdir		= $(datadir)/$(PACKAGE)/tests
-test_SCRIPTS	= cts-coverage cts-regression cts-cli cts-lrmd cts-stonithd
+test_SCRIPTS	= cts-coverage cts-regression cts-cli cts-lrmd cts-pengine \
+		  cts-stonithd
 test_DATA	= valgrind-pcmk.suppressions
 
 ctslibdir	= $(pyexecdir)/cts
 ctslib_PYTHON	=	__init__.py		\
 			CTSvars.py		\
 			CM_common.py		\
 			CM_corosync.py		\
 			CTS.py			\
 			CTSaudits.py		\
 			CTStests.py		\
 			CTSscenarios.py		\
 			CIB.py			\
 			cib_xml.py		\
 			environment.py		\
 			logging.py		\
 			patterns.py		\
 			remote.py		\
 			watcher.py
 
 ctsdir		= 	$(testdir)/cts
 cts_DATA	=	README.md cts.supp
 cts_SCRIPTS	=	cts		\
 			CTSlab.py		\
 			lxc_autogen.sh	\
 			LSBDummy		\
 			fence_dummy
 
 clidir		= $(testdir)/cli
 cli_DATA	= cli/regression.dates.exp cli/regression.tools.exp \
 		  cli/regression.acls.exp cli/regression.validity.exp
 
 PE_TESTS	= $(wildcard pengine/*.scores)
 pedir		= $(testdir)/pengine
 pe_DATA		= $(PE_TESTS) \
 		  $(PE_TESTS:%.scores=%.xml) \
 		  $(PE_TESTS:%.scores=%.exp) \
 		  $(PE_TESTS:%.scores=%.dot) \
 		  $(PE_TESTS:%.scores=%.summary) \
 		  $(wildcard pengine/*.stderr)
 
 # For "make test", run a single pengine test
 TESTS = pengine/bug-rh-1097457.xml
 TEST_EXTENSIONS = .xml
+XML_LOG_COMPILER = ./cts-pengine
+AM_XML_LOG_FLAGS = -V --run
 
 pe-list:
 	echo $(shell ls -1 pengine/*.xml)
 
 clean-local:
 	rm -f pengine/*.pe.*
 
 SUBDIRS	= benchmark
diff --git a/pengine/regression.sh b/cts/cts-pengine.in
old mode 100755
new mode 100644
similarity index 87%
rename from pengine/regression.sh
rename to cts/cts-pengine.in
index b77ee7f7fe..01c40ae144
--- a/pengine/regression.sh
+++ b/cts/cts-pengine.in
@@ -1,898 +1,1204 @@
 #!/bin/bash
 
  # Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  #
  # This program is free software; you can redistribute it and/or
  # modify it under the terms of the GNU General Public
  # License as published by the Free Software Foundation; either
  # version 2 of the License, or (at your option) any later version.
  #
  # This software is distributed in the hope that it will be useful,
  # but WITHOUT ANY WARRANTY; without even the implied warranty of
  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  # General Public License for more details.
  #
  # You should have received a copy of the GNU General Public
  # License along with this library; if not, write to the Free Software
  # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  #
 
 core=`dirname $0`
-. $core/regression.core.sh || exit 1
+verbose=0
+num_failed=0
+num_tests=0
+force_local=0
+VALGRIND_CMD=""
+diff_opts="--ignore-all-space --ignore-blank-lines -u -N"
+
+if [ "x$0" = "x-bash" ]; then 
+    test_home=@abs_top_builddir@/pengine
+else
+    test_home=`dirname $0`
+    test_name=`basename $0`
+fi
+
+function info() {
+    printf "$*\n"
+}
+
+function error() {
+    printf "      * ERROR:   $*\n"
+}
+
+function failed() {
+    printf "      * FAILED:  $*\n"
+}
+
+function show_test() {
+    name=$1; shift
+    printf "  Test %-25s $*\n" "$name:"
+}
+
+info "Test home is:\t$test_home"
+io_dir=$test_home/test10
+test_binary=@abs_top_builddir@/tools/crm_simulate
+PCMK_schema_directory=@abs_top_builddir@/xml
+
+
+failed=$test_home/.regression.failed.diff
+single_test=
+
+while true ; do
+    case "$1" in
+	-V|--verbose) verbose=1; shift;;
+	-v|--valgrind)
+	    export G_SLICE=always-malloc
+	    VALGRIND_CMD="valgrind -q --gen-suppressions=all --log-file=%q{valgrind_output} --show-reachable=no --leak-check=full --trace-children=no --time-stamp=yes --num-callers=20 --suppressions=@datadir@/@PACKAGE@/tests/valgrind-pcmk.suppressions"
+	    test_binary=
+	    shift;;
+	--valgrind-dhat)
+	    VALGRIND_CMD="valgrind --log-file=%q{valgrind_output} --show-top-n=100 --num-callers=4 --time-stamp=yes --trace-children=no --tool=exp-dhat"
+	    test_binary=
+	    shift;;
+	--valgrind-skip-output)
+	    VALGRIND_SKIP_OUTPUT=1
+	    shift;;
+	--run) single_test="$2"; shift; shift; break;;
+	-b|--binary) test_binary=$2; PCMK_schema_directory=""; shift; shift;;
+	-i|--io-dir) io_dir=$2; shift; shift;;
+	-?|--help) echo "$0 [--binary name] [--force-local]"; shift; exit 0;;
+	--) shift ; break ;;
+	"") break;;
+	*) echo "unknown option: $1"; exit 1;;
+    esac
+done
+
+if [ "x$test_binary" = "x" ]; then
+    test_binary=@sbindir@/crm_simulate
+    PCMK_schema_directory=@CRM_DTD_DIRECTORY@
+elif [ ! -x $test_binary ]; then
+    test_binary=@sbindir@/crm_simulate
+    PCMK_schema_directory=@CRM_DTD_DIRECTORY@
+fi
+
+export PCMK_schema_directory
+
+if [ "x$test_binary" = "x" ]; then
+    info "crm_simulate not installed. Aborting."
+    exit 1
+fi
+
+info "Test binary is:\t$test_binary"
+info "Schema home is:\t$PCMK_schema_directory"
+if [ "x$VALGRIND_CMD" != "x" ]; then
+    info "Activating memory testing with valgrind";
+fi
+
+info " "
+
+test_cmd="$VALGRIND_CMD $test_binary"
+#echo $test_cmd
+
+if [ `whoami` != root ]; then
+    declare -x CIB_shadow_dir=/tmp
+fi
+
+function do_test {
+
+    did_fail=0
+    expected_rc=0
+    num_tests=`expr $num_tests + 1`
+
+    base=$1; shift
+    name=$1; shift
+
+    input=$io_dir/${base}.xml
+    output=$io_dir/${base}.out
+    expected=$io_dir/${base}.exp
+
+    dot_png=$io_dir/${base}.png
+    dot_expected=$io_dir/${base}.dot
+    dot_output=$io_dir/${base}.pe.dot
+
+    scores=$io_dir/${base}.scores
+    score_output=$io_dir/${base}.scores.pe
+
+    stderr_expected=$io_dir/${base}.stderr
+    stderr_output=$io_dir/${base}.stderr.pe
+
+    summary=$io_dir/${base}.summary
+    summary_output=$io_dir/${base}.summary.pe
+
+    valgrind_output=$io_dir/${base}.valgrind
+    export valgrind_output
+
+    if [ "x$1" = "x--rc" ]; then
+	expected_rc=$2
+	shift; shift;
+    fi
+
+    show_test "$base" "$name"
+
+    if [ ! -f $input ]; then
+	error "No input";
+	did_fail=1
+	num_failed=`expr $num_failed + $did_fail`
+	return;
+    fi
+
+    if [ "$create_mode" != "true" -a ! -f $expected ]; then
+	error "no stored output";
+#	return;
+    fi
+
+#    ../admin/crm_verify -X $input
+    if [ ! -z $single_test ]; then
+	echo CIB_shadow_dir=$io_dir $test_cmd -x $input -D $dot_output -G $output -S $*
+	CIB_shadow_dir=$io_dir $test_cmd -x $input -D $dot_output -G $output -S $* 2>&1 | tee $summary_output
+    else
+	CIB_shadow_dir=$io_dir $test_cmd -x $input -S &> $summary_output
+    fi
+
+    CIB_shadow_dir=$io_dir $test_cmd -x $input -D $dot_output -G $output -SQ -s $* 2> $stderr_output > $score_output
+    rc=$?
+
+    if [ $rc != $expected_rc ]; then
+	failed "Test returned: $rc";
+	did_fail=1
+	echo "CIB_shadow_dir=$io_dir $test_cmd -x $input -D $dot_output -G $output -SQ -s $*"
+    fi
+
+    if [ -z "$VALGRIND_SKIP_OUTPUT" ]; then
+	if [ -s ${valgrind_output} ]; then
+	    error "Valgrind reported errors";
+	    did_fail=1
+	    cat ${valgrind_output}
+	fi
+	rm -f ${valgrind_output}
+    fi
+
+    if [ -s core ]; then
+	error "Core-file detected: core.${base}";
+	did_fail=1
+	rm -f $test_home/core.$base
+	mv core $test_home/core.$base
+    fi
+
+    if [ -e $stderr_expected ]; then
+
+	diff $diff_opts $stderr_expected $stderr_output >/dev/null
+	rc2=$?
+	if [ $rc2 != 0 ]; then
+	    failed "stderr changed";
+	    diff $diff_opts $stderr_expected $stderr_output 2>/dev/null >> $failed
+	    echo "" >> $failed
+	    did_fail=1
+	fi
+
+    elif [ -s $stderr_output ]; then
+	error "Output was written to stderr"
+	did_fail=1
+	cat $stderr_output
+    fi
+    rm -f $stderr_output
+
+    if [ ! -s $output ]; then
+	error "No graph produced";
+	did_fail=1
+	num_failed=`expr $num_failed + $did_fail`
+	rm -f $output
+	return;
+#    else
+#	mv $output $output.sed
+#	cat $output.sed | sed 's/id=.[0-9]*.\ //g' >> $output
+    fi
+
+    if [ ! -s $dot_output ]; then
+	error "No dot-file summary produced";
+	did_fail=1
+	num_failed=`expr $num_failed + $did_fail`
+	rm -f $output
+	return;
+    else
+	echo "digraph \"g\" {" > $dot_output.sort
+	LC_ALL=POSIX sort -u $dot_output | grep -v -e ^}$ -e digraph >> $dot_output.sort
+	echo "}" >> $dot_output.sort
+	mv -f $dot_output.sort $dot_output
+    fi
+
+    if [ ! -s $score_output ]; then
+	error "No allocation scores produced";
+	did_fail=1
+	num_failed=`expr $num_failed + $did_fail`
+	rm $output
+	return;
+    else
+	LC_ALL=POSIX sort $score_output > $score_output.sorted
+	mv -f $score_output.sorted $score_output
+    fi
+
+    if [ "$create_mode" = "true" ]; then
+	cp "$output" "$expected"
+	cp "$dot_output" "$dot_expected"
+	cp "$score_output" "$scores"
+	cp "$summary_output" "$summary"
+	info "	Created expected outputs"
+    fi
+
+    diff $diff_opts $summary $summary_output >/dev/null
+    rc2=$?
+    if [ $rc2 != 0 ]; then
+	failed "summary changed";
+	diff $diff_opts $summary $summary_output 2>/dev/null >> $failed
+	echo "" >> $failed
+	did_fail=1
+    fi
+
+    diff $diff_opts $dot_expected $dot_output >/dev/null
+    rc=$?
+    if [ $rc != 0 ]; then
+	failed "dot-file summary changed";
+	diff $diff_opts $dot_expected $dot_output 2>/dev/null >> $failed
+	echo "" >> $failed
+	did_fail=1
+    else
+	rm -f $dot_output
+    fi
+
+    sed -i -e 's/crm_feature_set="[^"]*"//' -e 's/batch-limit="[0-9]*"//' $expected $output
+    diff $diff_opts $expected $output >/dev/null
+    rc2=$?
+    if [ $rc2 != 0 ]; then
+	failed "xml-file changed";
+	diff $diff_opts $expected $output 2>/dev/null >> $failed
+	echo "" >> $failed
+	did_fail=1
+    fi
+
+    diff $diff_opts $scores $score_output >/dev/null
+    rc=$?
+    if [ $rc != 0 ]; then
+	failed "scores-file changed";
+	diff $diff_opts $scores $score_output 2>/dev/null >> $failed
+	echo "" >> $failed
+	did_fail=1
+    fi
+    rm -f $output $score_output $summary_output
+    num_failed=`expr $num_failed + $did_fail`
+}
+
+function test_results {
+    if [ $num_failed != 0 ]; then
+	if [ -s $failed ]; then
+	    if [ "$verbose" = "1" ]; then
+		error "Results of $num_failed failed tests (out of $num_tests)...."
+		cat $failed
+	    else
+		error "Results of $num_failed failed tests (out of $num_tests) are in $failed...."
+		error "Use $0 -V to display them automatically."
+	    fi
+	else
+	    error "$num_failed (of $num_tests) tests failed (no diff results)"
+	    rm $failed
+	fi
+    fi
+    exit $num_failed
+}
+
+if [ -z $single_test ]; then
+# zero out the error log
+    > $failed
+else
+    failed=.single
+    > $failed
+    do_test $single_test "Single shot" $*
+    cat $failed
+    exit $?
+fi
 
 DO_VERSIONED_TESTS=0
 
 create_mode="true"
 info Generating test outputs for these tests...
 # do_test file description
 info Done.
 echo ""
 
 info Performing the following tests from $io_dir
 create_mode="false"
 echo ""
 
 do_test simple1 "Offline     "
 do_test simple2 "Start       "
 do_test simple3 "Start 2     "
 do_test simple4 "Start Failed"
 do_test simple6 "Stop Start  "
 do_test simple7 "Shutdown    "
 #do_test simple8 "Stonith	"
 #do_test simple9 "Lower version"
 #do_test simple10 "Higher version"
 do_test simple11 "Priority (ne)"
 do_test simple12 "Priority (eq)"
 do_test simple8 "Stickiness"
 
 echo ""
 do_test group1 "Group		"
 do_test group2 "Group + Native	"
 do_test group3 "Group + Group	"
 do_test group4 "Group + Native (nothing)"
 do_test group5 "Group + Native (move)   "
 do_test group6 "Group + Group (move)    "
 do_test group7 "Group colocation"
 do_test group13 "Group colocation (cant run)"
 do_test group8 "Group anti-colocation"
 do_test group9 "Group recovery"
 do_test group10 "Group partial recovery"
 do_test group11 "Group target_role"
 do_test group14 "Group stop (graph terminated)"
 do_test group15 "-ve group colocation"
 do_test bug-1573 "Partial stop of a group with two children"
 do_test bug-1718 "Mandatory group ordering - Stop group_FUN"
 do_test bug-lf-2613 "Move group on failure"
 do_test bug-lf-2619 "Move group on clone failure"
 do_test group-fail "Ensure stop order is preserved for partially active groups"
 do_test group-unmanaged "No need to restart r115 because r114 is unmanaged"
 do_test group-unmanaged-stopped "Make sure r115 is stopped when r114 fails"
 do_test group-dependents "Account for the location preferences of things colocated with a group"
 
 echo ""
 do_test rsc_dep1 "Must not     "
 do_test rsc_dep3 "Must         "
 do_test rsc_dep5 "Must not 3   "
 do_test rsc_dep7 "Must 3       "
 do_test rsc_dep10 "Must (but cant)"
 do_test rsc_dep2  "Must (running) "
 do_test rsc_dep8  "Must (running : alt) "
 do_test rsc_dep4  "Must (running + move)"
 do_test asymmetric "Asymmetric - require explicit location constraints"
 
 echo ""
 do_test orphan-0 "Orphan ignore"
 do_test orphan-1 "Orphan stop"
 do_test orphan-2 "Orphan stop, remove failcount"
 
 echo ""
 do_test params-0 "Params: No change"
 do_test params-1 "Params: Changed"
 do_test params-2 "Params: Resource definition"
 do_test params-4 "Params: Reload"
 do_test params-5 "Params: Restart based on probe digest"
 do_test novell-251689 "Resource definition change + target_role=stopped"
 do_test bug-lf-2106 "Restart all anonymous clone instances after config change"
 do_test params-6 "Params: Detect reload in previously migrated resource"
 do_test nvpair-id-ref "Support id-ref in nvpair with optional name"
 do_test not-reschedule-unneeded-monitor "Do not reschedule unneeded monitors while resource definitions have changed"
 do_test reload-becomes-restart "Cancel reload if restart becomes required"
 
 echo ""
 do_test target-0 "Target Role : baseline"
 do_test target-1 "Target Role : master"
 do_test target-2 "Target Role : invalid"
 
 echo ""
 do_test base-score "Set a node's default score for all nodes"
 
 echo ""
 do_test date-1 "Dates" -t "2005-020"
 do_test date-2 "Date Spec - Pass" -t "2005-020T12:30"
 do_test date-3 "Date Spec - Fail" -t "2005-020T11:30"
 do_test origin "Timing of recurring operations" -t "2014-05-07 00:28:00" 
 do_test probe-0 "Probe (anon clone)"
 do_test probe-1 "Pending Probe"
 do_test probe-2 "Correctly re-probe cloned groups"
 do_test probe-3 "Probe (pending node)"
 do_test probe-4 "Probe (pending node + stopped resource)"
 do_test standby "Standby"
 do_test comments "Comments"
 
 echo ""
 do_test one-or-more-0 "Everything starts"
 do_test one-or-more-1 "Nothing starts because of A"
 do_test one-or-more-2 "D can start because of C"
 do_test one-or-more-3 "D cannot start because of B and C"
 do_test one-or-more-4 "D cannot start because of target-role"
 do_test one-or-more-5 "Start A and F even though C and D are stopped"
 do_test one-or-more-6 "Leave A running even though B is stopped"
 do_test one-or-more-7 "Leave A running even though C is stopped"
 do_test bug-5140-require-all-false "Allow basegrp:0 to stop"
 do_test clone-require-all-1 "clone B starts node 3 and 4"
 do_test clone-require-all-2 "clone B remains stopped everywhere"
 do_test clone-require-all-3 "clone B stops everywhere because A stops everywhere"
 do_test clone-require-all-4 "clone B remains on node 3 and 4 with only one instance of A remaining."
 do_test clone-require-all-5 "clone B starts on node 1 3 and 4"
 do_test clone-require-all-6 "clone B remains active after shutting down instances of A"
 do_test clone-require-all-7 "clone A and B both start at the same time. all instances of A start before B."
 do_test clone-require-all-no-interleave-1 "C starts everywhere after A and B"
 do_test clone-require-all-no-interleave-2 "C starts on nodes 1, 2, and 4 with only one active instance of B"
 do_test clone-require-all-no-interleave-3 "C remains active when instance of B is stopped on one node and started on another."
 do_test one-or-more-unrunnable-instances "Avoid dependencies on instances that won't ever be started"
 
 echo ""
 do_test order1 "Order start 1     "
 do_test order2 "Order start 2     "
 do_test order3 "Order stop	  "
 do_test order4 "Order (multiple)  "
 do_test order5 "Order (move)  "
 do_test order6 "Order (move w/ restart)  "
 do_test order7 "Order (mandatory)  "
 do_test order-optional "Order (score=0)  "
 do_test order-required "Order (score=INFINITY)  "
 do_test bug-lf-2171 "Prevent group start when clone is stopped"
 do_test order-clone "Clone ordering should be able to prevent startup of dependent clones"
 do_test order-sets "Ordering for resource sets"
 do_test order-serialize "Serialize resources without inhibiting migration"
 do_test order-serialize-set "Serialize a set of resources without inhibiting migration"
 do_test clone-order-primitive "Order clone start after a primitive"
 do_test clone-order-16instances "Verify ordering of 16 cloned resources"
 do_test order-optional-keyword "Order (optional keyword)"
 do_test order-mandatory "Order (mandatory keyword)"
 do_test bug-lf-2493 "Don't imply colocation requirements when applying ordering constraints with clones"
 do_test ordered-set-basic-startup "Constraint set with default order settings."
 do_test ordered-set-natural "Allow natural set ordering"
 do_test order-wrong-kind "Order (error)"
 
 echo ""
 do_test coloc-loop "Colocation - loop"
 do_test coloc-many-one "Colocation - many-to-one"
 do_test coloc-list "Colocation - many-to-one with list"
 do_test coloc-group "Colocation - groups"
 do_test coloc-slave-anti "Anti-colocation with slave shouldn't prevent master colocation"
 do_test coloc-attr "Colocation based on node attributes"
 do_test coloc-negative-group "Negative colocation with a group"
 do_test coloc-intra-set "Intra-set colocation"
 do_test bug-lf-2435 "Colocation sets with a negative score"
 do_test coloc-clone-stays-active "Ensure clones don't get stopped/demoted because a dependent must stop"
 do_test coloc_fp_logic "Verify floating point calculations in colocation are working"
 do_test colo_master_w_native "cl#5070 - Verify promotion order is affected when colocating master to native rsc."
 do_test colo_slave_w_native  "cl#5070 - Verify promotion order is affected when colocating slave to native rsc."
 do_test anti-colocation-order "cl#5187 - Prevent resources in an anti-colocation from even temporarily running on a same node"
 do_test anti-colocation-master "Organize order of actions for master resources in anti-colocations"
 do_test anti-colocation-slave "Organize order of actions for slave resources in anti-colocations"
 do_test enforce-colo1 "Always enforce B with A INFINITY."
 do_test complex_enforce_colo "Always enforce B with A INFINITY. (make sure heat-engine stops)"
 
 echo ""
 do_test rsc-sets-seq-true "Resource Sets - sequential=false"
 do_test rsc-sets-seq-false "Resource Sets - sequential=true"
 do_test rsc-sets-clone "Resource Sets - Clone"
 do_test rsc-sets-master "Resource Sets - Master"
 do_test rsc-sets-clone-1 "Resource Sets - Clone (lf#2404)"
 
 #echo ""
 #do_test agent1 "version: lt (empty)"
 #do_test agent2 "version: eq	"
 #do_test agent3 "version: gt	"
 
 echo ""
 do_test attrs1 "string: eq (and)     "
 do_test attrs2 "string: lt / gt (and)"
 do_test attrs3 "string: ne (or)      "
 do_test attrs4 "string: exists       "
 do_test attrs5 "string: not_exists   "
 do_test attrs6 "is_dc: true          "
 do_test attrs7 "is_dc: false         "
 do_test attrs8 "score_attribute      "
 do_test per-node-attrs "Per node resource parameters"
 
 echo ""
 do_test mon-rsc-1 "Schedule Monitor - start"
 do_test mon-rsc-2 "Schedule Monitor - move "
 do_test mon-rsc-3 "Schedule Monitor - pending start     "
 do_test mon-rsc-4 "Schedule Monitor - move/pending start"
 
 echo ""
 do_test rec-rsc-0 "Resource Recover - no start     "
 do_test rec-rsc-1 "Resource Recover - start        "
 do_test rec-rsc-2 "Resource Recover - monitor      "
 do_test rec-rsc-3 "Resource Recover - stop - ignore"
 do_test rec-rsc-4 "Resource Recover - stop - block "
 do_test rec-rsc-5 "Resource Recover - stop - fence "
 do_test rec-rsc-6 "Resource Recover - multiple - restart"
 do_test rec-rsc-7 "Resource Recover - multiple - stop   "
 do_test rec-rsc-8 "Resource Recover - multiple - block  "
 do_test rec-rsc-9 "Resource Recover - group/group"
 do_test monitor-recovery "on-fail=block + resource recovery detected by recurring monitor"
 do_test stop-failure-no-quorum "Stop failure without quorum"
 do_test stop-failure-no-fencing "Stop failure without fencing available"
 do_test stop-failure-with-fencing "Stop failure with fencing available"
 do_test multiple-active-block-group "Support of multiple-active=block for resource groups"
 do_test multiple-monitor-one-failed "Consider resource failed if any of the configured monitor operations failed"
 
 echo ""
 do_test quorum-1 "No quorum - ignore"
 do_test quorum-2 "No quorum - freeze"
 do_test quorum-3 "No quorum - stop  "
 do_test quorum-4 "No quorum - start anyway"
 do_test quorum-5 "No quorum - start anyway (group)"
 do_test quorum-6 "No quorum - start anyway (clone)"
 do_test bug-cl-5212 "No promotion with no-quorum-policy=freeze"
 do_test suicide-needed-inquorate "no-quorum-policy=suicide: suicide necessary"
 do_test suicide-not-needed-initial-quorum "no-quorum-policy=suicide: suicide not necessary at initial quorum"
 do_test suicide-not-needed-never-quorate "no-quorum-policy=suicide: suicide not necessary if never quorate"
 do_test suicide-not-needed-quorate "no-quorum-policy=suicide: suicide necessary if quorate"
 
 echo ""
 do_test rec-node-1 "Node Recover - Startup   - no fence"
 do_test rec-node-2 "Node Recover - Startup   - fence   "
 do_test rec-node-3 "Node Recover - HA down   - no fence"
 do_test rec-node-4 "Node Recover - HA down   - fence   "
 do_test rec-node-5 "Node Recover - CRM down  - no fence"
 do_test rec-node-6 "Node Recover - CRM down  - fence   "
 do_test rec-node-7 "Node Recover - no quorum - ignore  "
 do_test rec-node-8 "Node Recover - no quorum - freeze  "
 do_test rec-node-9 "Node Recover - no quorum - stop    "
 do_test rec-node-10 "Node Recover - no quorum - stop w/fence"
 do_test rec-node-11 "Node Recover - CRM down w/ group - fence   "
 do_test rec-node-12 "Node Recover - nothing active - fence   "
 do_test rec-node-13 "Node Recover - failed resource + shutdown - fence   "
 do_test rec-node-15 "Node Recover - unknown lrm section"
 do_test rec-node-14 "Serialize all stonith's"
 
 echo ""
 do_test multi1 "Multiple Active (stop/start)"
 
 echo ""
 do_test migrate-begin     "Normal migration"
 do_test migrate-success   "Completed migration"
 do_test migrate-partial-1 "Completed migration, missing stop on source"
 do_test migrate-partial-2 "Successful migrate_to only"
 do_test migrate-partial-3 "Successful migrate_to only, target down"
 do_test migrate-partial-4 "Migrate from the correct host after migrate_to+migrate_from"
 do_test bug-5186-partial-migrate "Handle partial migration when src node loses membership"
 
 do_test migrate-fail-2 "Failed migrate_from"
 do_test migrate-fail-3 "Failed migrate_from + stop on source"
 do_test migrate-fail-4 "Failed migrate_from + stop on target - ideally we wouldn't need to re-stop on target"
 do_test migrate-fail-5 "Failed migrate_from + stop on source and target"
 
 do_test migrate-fail-6 "Failed migrate_to"
 do_test migrate-fail-7 "Failed migrate_to + stop on source"
 do_test migrate-fail-8 "Failed migrate_to + stop on target - ideally we wouldn't need to re-stop on target"
 do_test migrate-fail-9 "Failed migrate_to + stop on source and target"
 
 do_test migrate-stop "Migration in a stopping stack"
 do_test migrate-start "Migration in a starting stack"
 do_test migrate-stop_start "Migration in a restarting stack"
 do_test migrate-stop-complex "Migration in a complex stopping stack"
 do_test migrate-start-complex "Migration in a complex starting stack"
 do_test migrate-stop-start-complex "Migration in a complex moving stack"
 do_test migrate-shutdown "Order the post-migration 'stop' before node shutdown"
 
 do_test migrate-1 "Migrate (migrate)"
 do_test migrate-2 "Migrate (stable)"
 do_test migrate-3 "Migrate (failed migrate_to)"
 do_test migrate-4 "Migrate (failed migrate_from)"
 do_test novell-252693 "Migration in a stopping stack"
 do_test novell-252693-2 "Migration in a starting stack"
 do_test novell-252693-3 "Non-Migration in a starting and stopping stack"
 do_test bug-1820 "Migration in a group"
 do_test bug-1820-1 "Non-migration in a group"
 do_test migrate-5 "Primitive migration with a clone"
 do_test migrate-fencing "Migration after Fencing"
 do_test migrate-both-vms "Migrate two VMs that have no colocation"
 do_test migration-behind-migrating-remote "Migrate resource behind migrating remote connection"
 
 do_test 1-a-then-bm-move-b "Advanced migrate logic. A then B. migrate B."
 do_test 2-am-then-b-move-a "Advanced migrate logic, A then B, migrate A without stopping B"
 do_test 3-am-then-bm-both-migrate "Advanced migrate logic. A then B. migrate both"
 do_test 4-am-then-bm-b-not-migratable "Advanced migrate logic, A then B, B not migratable"
 do_test 5-am-then-bm-a-not-migratable "Advanced migrate logic. A then B. move both, a not migratable"
 do_test 6-migrate-group "Advanced migrate logic, migrate a group"
 do_test 7-migrate-group-one-unmigratable "Advanced migrate logic, migrate group mixed with allow-migrate true/false"
 do_test 8-am-then-bm-a-migrating-b-stopping "Advanced migrate logic, A then B, A migrating, B stopping"
 do_test 9-am-then-bm-b-migrating-a-stopping "Advanced migrate logic, A then B, B migrate, A stopping"
 do_test 10-a-then-bm-b-move-a-clone "Advanced migrate logic, A clone then B, migrate B while stopping A"
 do_test 11-a-then-bm-b-move-a-clone-starting "Advanced migrate logic, A clone then B, B moving while A is start/stopping"
 
 do_test a-promote-then-b-migrate "A promote then B start. migrate B"
 do_test a-demote-then-b-migrate "A demote then B stop. migrate B"
 
 if [ $DO_VERSIONED_TESTS -eq 1 ]; then
 	do_test migrate-versioned "Disable migration for versioned resources"
 fi
 
 #echo ""
 #do_test complex1 "Complex	"
 
 do_test bug-lf-2422 "Dependency on partially active group - stop ocfs:*"
 
 echo ""
 do_test clone-anon-probe-1 "Probe the correct (anonymous) clone instance for each node"
 do_test clone-anon-probe-2 "Avoid needless re-probing of anonymous clones"
 do_test clone-anon-failcount "Merge failcounts for anonymous clones"
 do_test inc0 "Incarnation start"
 do_test inc1 "Incarnation start order"
 do_test inc2 "Incarnation silent restart, stop, move"
 do_test inc3 "Inter-incarnation ordering, silent restart, stop, move"
 do_test inc4 "Inter-incarnation ordering, silent restart, stop, move (ordered)"
 do_test inc5 "Inter-incarnation ordering, silent restart, stop, move (restart 1)"
 do_test inc6 "Inter-incarnation ordering, silent restart, stop, move (restart 2)"
 do_test inc7 "Clone colocation"
 do_test inc8 "Clone anti-colocation"
 do_test inc9 "Non-unique clone"
 do_test inc10 "Non-unique clone (stop)"
 do_test inc11 "Primitive colocation with clones"
 do_test inc12 "Clone shutdown"
 do_test cloned-group "Make sure only the correct number of cloned groups are started"
 do_test cloned-group-stop "Ensure stopping qpidd also stops glance and cinder"
 do_test clone-no-shuffle "Don't prioritize allocation of instances that must be moved"
 do_test clone-max-zero "Orphan processing with clone-max=0"
 do_test clone-anon-dup "Bug LF#2087 - Correctly parse the state of anonymous clones that are active more than once per node"
 do_test bug-lf-2160 "Don't shuffle clones due to colocation"
 do_test bug-lf-2213 "clone-node-max enforcement for cloned groups"
 do_test bug-lf-2153 "Clone ordering constraints"
 do_test bug-lf-2361 "Ensure clones observe mandatory ordering constraints if the LHS is unrunnable"
 do_test bug-lf-2317 "Avoid needless restart of primitive depending on a clone"
 do_test clone-colocate-instance-1 "Colocation with a specific clone instance (negative example)"
 do_test clone-colocate-instance-2 "Colocation with a specific clone instance"
 do_test clone-order-instance "Ordering with specific clone instances"
 do_test bug-lf-2453 "Enforce mandatory clone ordering without colocation"
 do_test bug-lf-2508 "Correctly reconstruct the status of anonymous cloned groups"
 do_test bug-lf-2544 "Balanced clone placement"
 do_test bug-lf-2445 "Redistribute clones with node-max > 1 and stickiness = 0"
 do_test bug-lf-2574 "Avoid clone shuffle"
 do_test bug-lf-2581 "Avoid group restart due to unrelated clone (re)start"
 do_test bug-cl-5168 "Don't shuffle clones"
 do_test bug-cl-5170 "Prevent clone from starting with on-fail=block"
 do_test clone-fail-block-colocation "Move colocated group when failed clone has on-fail=block"
 do_test clone-interleave-1 "Clone-3 cannot start on pcmk-1 due to interleaved ordering (no colocation)"
 do_test clone-interleave-2 "Clone-3 must stop on pcmk-1 due to interleaved ordering (no colocation)"
 do_test clone-interleave-3 "Clone-3 must be recovered on pcmk-1 due to interleaved ordering (no colocation)"
 do_test rebalance-unique-clones "Rebalance unique clone instances with no stickiness"
 
 echo ""
 do_test cloned_start_one  "order first clone then clone... first clone_min=2"
 do_test cloned_start_two  "order first clone then clone... first clone_min=2"
 do_test cloned_stop_one   "order first clone then clone... first clone_min=2"
 do_test cloned_stop_two   "order first clone then clone... first clone_min=2"
 do_test clone_min_interleave_start_one "order first clone then clone... first clone_min=2 and then has interleave=true"
 do_test clone_min_interleave_start_two "order first clone then clone... first clone_min=2 and then has interleave=true"
 do_test clone_min_interleave_stop_one  "order first clone then clone... first clone_min=2 and then has interleave=true"
 do_test clone_min_interleave_stop_two  "order first clone then clone... first clone_min=2 and then has interleave=true"
 do_test clone_min_start_one "order first clone then primitive... first clone_min=2"
 do_test clone_min_start_two "order first clone then primitive... first clone_min=2"
 do_test clone_min_stop_all  "order first clone then primitive... first clone_min=2"
 do_test clone_min_stop_one  "order first clone then primitive... first clone_min=2"
 do_test clone_min_stop_two  "order first clone then primitive... first clone_min=2"
 
 echo ""
 do_test unfence-startup "Clean unfencing"
 do_test unfence-definition "Unfencing when the agent changes"
 do_test unfence-parameters "Unfencing when the agent parameters changes"
 do_test unfence-device "Unfencing when a cluster has only fence devices"
 
 echo ""
 do_test master-0 "Stopped -> Slave"
 do_test master-1 "Stopped -> Promote"
 do_test master-2 "Stopped -> Promote : notify"
 do_test master-3 "Stopped -> Promote : master location"
 do_test master-4 "Started -> Promote : master location"
 do_test master-5 "Promoted -> Promoted"
 do_test master-6 "Promoted -> Promoted (2)"
 do_test master-7 "Promoted -> Fenced"
 do_test master-8 "Promoted -> Fenced -> Moved"
 do_test master-9 "Stopped + Promotable + No quorum"
 do_test master-10 "Stopped -> Promotable : notify with monitor"
 do_test master-11 "Stopped -> Promote : colocation"
 do_test novell-239082 "Demote/Promote ordering"
 do_test novell-239087 "Stable master placement"
 do_test master-12 "Promotion based solely on rsc_location constraints"
 do_test master-13 "Include preferences of colocated resources when placing master"
 do_test master-demote "Ordering when actions depends on demoting a slave resource"
 do_test master-ordering "Prevent resources from starting that need a master"
 do_test bug-1765 "Master-Master Colocation (dont stop the slaves)"
 do_test master-group "Promotion of cloned groups"
 do_test bug-lf-1852 "Don't shuffle master/slave instances unnecessarily"
 do_test master-failed-demote "Don't retry failed demote actions"
 do_test master-failed-demote-2 "Don't retry failed demote actions (notify=false)"
 do_test master-depend "Ensure resources that depend on the master don't get allocated until the master does"
 do_test master-reattach "Re-attach to a running master"
 do_test master-allow-start "Don't include master score if it would prevent allocation"
 do_test master-colocation "Allow master instances placemaker to be influenced by colocation constraints"
 do_test master-pseudo "Make sure promote/demote pseudo actions are created correctly"
 do_test master-role "Prevent target-role from promoting more than master-max instances"
 do_test bug-lf-2358 "Master-Master anti-colocation"
 do_test master-promotion-constraint "Mandatory master colocation constraints"
 do_test unmanaged-master "Ensure role is preserved for unmanaged resources"
 do_test master-unmanaged-monitor "Start the correct monitor operation for unmanaged masters"
 do_test master-demote-2 "Demote does not clear past failure"
 do_test master-move "Move master based on failure of colocated group"
 do_test master-probed-score "Observe the promotion score of probed resources"
 do_test colocation_constraint_stops_master "cl#5054 - Ensure master is demoted when stopped by colocation constraint"
 do_test colocation_constraint_stops_slave  "cl#5054 - Ensure slave is not demoted when stopped by colocation constraint"
 do_test order_constraint_stops_master      "cl#5054 - Ensure master is demoted when stopped by order constraint"
 do_test order_constraint_stops_slave       "cl#5054 - Ensure slave is not demoted when stopped by order constraint"
 do_test master_monitor_restart "cl#5072 - Ensure master monitor operation will start after promotion."
 do_test bug-rh-880249 "Handle replacement of an m/s resource with a primitive"
 do_test bug-5143-ms-shuffle "Prevent master shuffling due to promotion score"
 do_test master-demote-block "Block promotion if demote fails with on-fail=block"
 do_test master-dependent-ban "Don't stop instances from being active because a dependent is banned from that host"
 do_test master-stop "Stop instances due to location constraint with role=Started"
 do_test master-partially-demoted-group "Allow partially demoted group to finish demoting"
 do_test bug-cl-5213 "Ensure role colocation with -INFINITY is enforced"
 do_test bug-cl-5219 "Allow unrelated resources with a common colocation target to remain promoted"
 do_test master-asymmetrical-order "Fix the behaviors of multi-state resources with asymmetrical ordering"
 do_test master-notify "Master promotion with notifies"
 do_test master-score-startup "Use permanent master scores without LRM history"
 
 echo ""
 do_test history-1 "Correctly parse stateful-1 resource state"
 
 echo ""
 do_test managed-0 "Managed (reference)"
 do_test managed-1 "Not managed - down "
 do_test managed-2 "Not managed - up   "
 do_test bug-5028 "Shutdown should block if anything depends on an unmanaged resource"
 do_test bug-5028-detach "Ensure detach still works"
 do_test bug-5028-bottom "Ensure shutdown still blocks if the blocked resource is at the bottom of the stack"
 do_test unmanaged-stop-1 "cl#5155 - Block the stop of resources if any depending resource is unmanaged "
 do_test unmanaged-stop-2 "cl#5155 - Block the stop of resources if the first resource in a mandatory stop order is unmanaged "
 do_test unmanaged-stop-3 "cl#5155 - Block the stop of resources if any depending resource in a group is unmanaged "
 do_test unmanaged-stop-4 "cl#5155 - Block the stop of resources if any depending resource in the middle of a group is unmanaged "
 do_test unmanaged-block-restart "Block restart of resources if any dependent resource in a group is unmanaged"
 
 echo ""
 do_test interleave-0 "Interleave (reference)"
 do_test interleave-1 "coloc - not interleaved"
 do_test interleave-2 "coloc - interleaved   "
 do_test interleave-3 "coloc - interleaved (2)"
 do_test interleave-pseudo-stop "Interleaved clone during stonith"
 do_test interleave-stop "Interleaved clone during stop"
 do_test interleave-restart "Interleaved clone during dependency restart"
 
 echo ""
 do_test notify-0 "Notify reference"
 do_test notify-1 "Notify simple"
 do_test notify-2 "Notify simple, confirm"
 do_test notify-3 "Notify move, confirm"
 do_test novell-239079 "Notification priority"
 #do_test notify-2 "Notify - 764"
 
 echo ""
 do_test 594 "OSDL #594 - Unrunnable actions scheduled in transition"
 do_test 662 "OSDL #662 - Two resources start on one node when incarnation_node_max = 1"
 do_test 696 "OSDL #696 - CRM starts stonith RA without monitor"
 do_test 726 "OSDL #726 - Attempting to schedule rsc_posic041_monitor_5000 _after_ a stop"
 do_test 735 "OSDL #735 - Correctly detect that rsc_hadev1 is stopped on hadev3"
 do_test 764 "OSDL #764 - Missing monitor op for DoFencing:child_DoFencing:1"
 do_test 797 "OSDL #797 - Assert triggered: task_id_i > max_call_id"
 do_test 829 "OSDL #829"
 do_test 994 "OSDL #994 - Stopping the last resource in a resource group causes the entire group to be restarted"
 do_test 994-2 "OSDL #994 - with a dependent resource"
 do_test 1360 "OSDL #1360 - Clone stickiness"
 do_test 1484 "OSDL #1484 - on_fail=stop"
 do_test 1494 "OSDL #1494 - Clone stability"
 do_test unrunnable-1 "Unrunnable"
 do_test unrunnable-2 "Unrunnable 2"
 do_test stonith-0 "Stonith loop - 1"
 do_test stonith-1 "Stonith loop - 2"
 do_test stonith-2 "Stonith loop - 3"
 do_test stonith-3 "Stonith startup"
 do_test stonith-4 "Stonith node state"
 do_test bug-1572-1 "Recovery of groups depending on master/slave"
 do_test bug-1572-2 "Recovery of groups depending on master/slave when the master is never re-promoted"
 do_test bug-1685 "Depends-on-master ordering"
 do_test bug-1822 "Don't promote partially active groups"
 do_test bug-pm-11 "New resource added to a m/s group"
 do_test bug-pm-12 "Recover only the failed portion of a cloned group"
 do_test bug-n-387749 "Don't shuffle clone instances"
 do_test bug-n-385265 "Don't ignore the failure stickiness of group children - resource_idvscommon should stay stopped"
 do_test bug-n-385265-2 "Ensure groups are migrated instead of remaining partially active on the current node"
 do_test bug-lf-1920 "Correctly handle probes that find active resources"
 do_test bnc-515172 "Location constraint with multiple expressions"
 do_test colocate-primitive-with-clone "Optional colocation with a clone"
 do_test use-after-free-merge "Use-after-free in native_merge_weights"
 do_test bug-lf-2551 "STONITH ordering for stop"
 do_test bug-lf-2606 "Stonith implies demote"
 do_test bug-lf-2474 "Ensure resource op timeout takes precedence over op_defaults"
 do_test bug-suse-707150 "Prevent vm-01 from starting due to colocation/ordering"
 do_test bug-5014-A-start-B-start "Verify when A starts B starts using symmetrical=false"
 do_test bug-5014-A-stop-B-started "Verify when A stops B does not stop if it has already started using symmetric=false"
 do_test bug-5014-A-stopped-B-stopped "Verify when A is stopped and B has not started, B does not start before A using symmetric=false"
 do_test bug-5014-CthenAthenB-C-stopped "Verify when C then A is symmetrical=true, A then B is symmetric=false, and C is stopped that nothing starts."
 do_test bug-5014-CLONE-A-start-B-start "Verify when A starts B starts using clone resources with symmetric=false"
 do_test bug-5014-CLONE-A-stop-B-started "Verify when A stops B does not stop if it has already started using clone resources with symmetric=false."
 do_test bug-5014-GROUP-A-start-B-start "Verify when A starts B starts when using group resources with symmetric=false."
 do_test bug-5014-GROUP-A-stopped-B-started "Verify when A stops B does not stop if it has already started using group resources with symmetric=false."
 do_test bug-5014-GROUP-A-stopped-B-stopped "Verify when A is stopped and B has not started, B does not start before A using group resources with symmetric=false."
 do_test bug-5014-ordered-set-symmetrical-false "Verify ordered sets work with symmetrical=false"
 do_test bug-5014-ordered-set-symmetrical-true "Verify ordered sets work with symmetrical=true"
 do_test bug-5007-masterslave_colocation "Verify use of colocation scores other than INFINITY and -INFINITY work on multi-state resources."
 do_test bug-5038 "Prevent restart of anonymous clones when clone-max decreases"
 do_test bug-5025-1 "Automatically clean up failcount after resource config change with reload"
 do_test bug-5025-2 "Make sure clear failcount action isn't set when config does not change."
 do_test bug-5025-3 "Automatically clean up failcount after resource config change with restart"
 do_test bug-5025-4 "Clear failcount when last failure is a start op and rsc attributes changed."
 do_test failcount "Ensure failcounts are correctly expired"
 do_test failcount-block "Ensure failcounts are not expired when on-fail=block is present"
 do_test per-op-failcount "Ensure per-operation failcount is handled and not passed to fence agent"
 do_test on-fail-ignore "Ensure on-fail=ignore works even beyond migration-threshold"
 do_test monitor-onfail-restart "bug-5058 - Monitor failure with on-fail set to restart"
 do_test monitor-onfail-stop    "bug-5058 - Monitor failure wiht on-fail set to stop"
 do_test bug-5059 "No need to restart p_stateful1:*"
 do_test bug-5069-op-enabled  "Test on-fail=ignore with failure when monitor is enabled."
 do_test bug-5069-op-disabled "Test on-fail-ignore with failure when monitor is disabled."
 do_test obsolete-lrm-resource "cl#5115 - Do not use obsolete lrm_resource sections"
 do_test expire-non-blocked-failure "Ignore failure-timeout only if the failed operation has on-fail=block"
 do_test asymmetrical-order-move "Respect asymmetrical ordering when trying to move resources"
 do_test start-then-stop-with-unfence "Avoid graph loop with start-then-stop constraint plus unfencing"
 
 do_test ignore_stonith_rsc_order1 "cl#5056- Ignore order constraint between stonith and non-stonith rsc."
 do_test ignore_stonith_rsc_order2 "cl#5056- Ignore order constraint with group rsc containing mixed stonith and non-stonith."
 do_test ignore_stonith_rsc_order3 "cl#5056- Ignore order constraint, stonith clone and mixed group"
 do_test ignore_stonith_rsc_order4 "cl#5056- Ignore order constraint, stonith clone and clone with nested mixed group"
 do_test honor_stonith_rsc_order1 "cl#5056- Honor order constraint, stonith clone and pure stonith group(single rsc)."
 do_test honor_stonith_rsc_order2 "cl#5056- Honor order constraint, stonith clone and pure stonith group(multiple rsc)"
 do_test honor_stonith_rsc_order3 "cl#5056- Honor order constraint, stonith clones with nested pure stonith group."
 do_test honor_stonith_rsc_order4 "cl#5056- Honor order constraint, between two native stonith rscs."
 do_test probe-timeout "cl#5099 - Default probe timeout"
 
 do_test concurrent-fencing "Allow performing fencing operations in parallel"
 
 echo ""
 do_test systemhealth1  "System Health ()               #1"
 do_test systemhealth2  "System Health ()               #2"
 do_test systemhealth3  "System Health ()               #3"
 do_test systemhealthn1 "System Health (None)           #1"
 do_test systemhealthn2 "System Health (None)           #2"
 do_test systemhealthn3 "System Health (None)           #3"
 do_test systemhealthm1 "System Health (Migrate On Red) #1"
 do_test systemhealthm2 "System Health (Migrate On Red) #2"
 do_test systemhealthm3 "System Health (Migrate On Red) #3"
 do_test systemhealtho1 "System Health (Only Green)     #1"
 do_test systemhealtho2 "System Health (Only Green)     #2"
 do_test systemhealtho3 "System Health (Only Green)     #3"
 do_test systemhealthp1 "System Health (Progessive)     #1"
 do_test systemhealthp2 "System Health (Progessive)     #2"
 do_test systemhealthp3 "System Health (Progessive)     #3"
 
 echo ""
 do_test utilization "Placement Strategy - utilization"
 do_test minimal     "Placement Strategy - minimal"
 do_test balanced    "Placement Strategy - balanced"
 
 echo ""
 do_test placement-stickiness "Optimized Placement Strategy - stickiness"
 do_test placement-priority   "Optimized Placement Strategy - priority"
 do_test placement-location   "Optimized Placement Strategy - location"
 do_test placement-capacity   "Optimized Placement Strategy - capacity"
 
 echo ""
 do_test utilization-order1 "Utilization Order - Simple"
 do_test utilization-order2 "Utilization Order - Complex"
 do_test utilization-order3 "Utilization Order - Migrate"
 do_test utilization-order4 "Utilization Order - Live Mirgration (bnc#695440)"
 do_test utilization-shuffle "Don't displace prmExPostgreSQLDB2 on act2, Start prmExPostgreSQLDB1 on act3"
 do_test load-stopped-loop "Avoid transition loop due to load_stopped (cl#5044)"
 do_test load-stopped-loop-2 "cl#5235 - Prevent graph loops that can be introduced by load_stopped -> migrate_to ordering"
 
 echo ""
 do_test colocated-utilization-primitive-1 "Colocated Utilization - Primitive"
 do_test colocated-utilization-primitive-2 "Colocated Utilization - Choose the most capable node"
 do_test colocated-utilization-group "Colocated Utilization - Group"
 do_test colocated-utilization-clone "Colocated Utilization - Clone"
 
 do_test utilization-check-allowed-nodes "Only check the capacities of the nodes that can run the resource"
 
 echo ""
 do_test reprobe-target_rc "Ensure correct target_rc for reprobe of inactive resources"
 do_test node-maintenance-1 "cl#5128 - Node maintenance"
 do_test node-maintenance-2 "cl#5128 - Node maintenance (coming out of maintenance mode)"
 do_test shutdown-maintenance-node "Do not fence a maintenance node if it shuts down cleanly"
 
 do_test rsc-maintenance "Per-resource maintenance"
 
 echo ""
 do_test not-installed-agent "The resource agent is missing"
 do_test not-installed-tools "Something the resource agent needs is missing"
 
 echo ""
 do_test stopped-monitor-00	"Stopped Monitor - initial start"
 do_test stopped-monitor-01	"Stopped Monitor - failed started"
 do_test stopped-monitor-02	"Stopped Monitor - started multi-up"
 do_test stopped-monitor-03	"Stopped Monitor - stop started"
 do_test stopped-monitor-04	"Stopped Monitor - failed stop"
 do_test stopped-monitor-05	"Stopped Monitor - start unmanaged"
 do_test stopped-monitor-06	"Stopped Monitor - unmanaged multi-up"
 do_test stopped-monitor-07	"Stopped Monitor - start unmanaged multi-up"
 do_test stopped-monitor-08	"Stopped Monitor - migrate"
 do_test stopped-monitor-09	"Stopped Monitor - unmanage started"
 do_test stopped-monitor-10	"Stopped Monitor - unmanaged started multi-up"
 do_test stopped-monitor-11	"Stopped Monitor - stop unmanaged started"
 do_test stopped-monitor-12	"Stopped Monitor - unmanaged started multi-up (targer-role="Stopped")"
 do_test stopped-monitor-20	"Stopped Monitor - initial stop"
 do_test stopped-monitor-21	"Stopped Monitor - stopped single-up"
 do_test stopped-monitor-22	"Stopped Monitor - stopped multi-up"
 do_test stopped-monitor-23	"Stopped Monitor - start stopped"
 do_test stopped-monitor-24	"Stopped Monitor - unmanage stopped"
 do_test stopped-monitor-25	"Stopped Monitor - unmanaged stopped multi-up"
 do_test stopped-monitor-26	"Stopped Monitor - start unmanaged stopped"
 do_test stopped-monitor-27	"Stopped Monitor - unmanaged stopped multi-up (target-role="Started")"
 do_test stopped-monitor-30	"Stopped Monitor - new node started"
 do_test stopped-monitor-31	"Stopped Monitor - new node stopped"
 
 echo""
 do_test ticket-primitive-1 "Ticket - Primitive (loss-policy=stop, initial)"
 do_test ticket-primitive-2 "Ticket - Primitive (loss-policy=stop, granted)"
 do_test ticket-primitive-3 "Ticket - Primitive (loss-policy-stop, revoked)"
 do_test ticket-primitive-4 "Ticket - Primitive (loss-policy=demote, initial)"
 do_test ticket-primitive-5 "Ticket - Primitive (loss-policy=demote, granted)"
 do_test ticket-primitive-6 "Ticket - Primitive (loss-policy=demote, revoked)"
 do_test ticket-primitive-7 "Ticket - Primitive (loss-policy=fence, initial)"
 do_test ticket-primitive-8 "Ticket - Primitive (loss-policy=fence, granted)"
 do_test ticket-primitive-9 "Ticket - Primitive (loss-policy=fence, revoked)"
 do_test ticket-primitive-10 "Ticket - Primitive (loss-policy=freeze, initial)"
 do_test ticket-primitive-11 "Ticket - Primitive (loss-policy=freeze, granted)"
 do_test ticket-primitive-12 "Ticket - Primitive (loss-policy=freeze, revoked)"
 
 do_test ticket-primitive-13 "Ticket - Primitive (loss-policy=stop, standby, granted)"
 do_test ticket-primitive-14 "Ticket - Primitive (loss-policy=stop, granted, standby)"
 do_test ticket-primitive-15 "Ticket - Primitive (loss-policy=stop, standby, revoked)"
 do_test ticket-primitive-16 "Ticket - Primitive (loss-policy=demote, standby, granted)"
 do_test ticket-primitive-17 "Ticket - Primitive (loss-policy=demote, granted, standby)"
 do_test ticket-primitive-18 "Ticket - Primitive (loss-policy=demote, standby, revoked)"
 do_test ticket-primitive-19 "Ticket - Primitive (loss-policy=fence, standby, granted)"
 do_test ticket-primitive-20 "Ticket - Primitive (loss-policy=fence, granted, standby)"
 do_test ticket-primitive-21 "Ticket - Primitive (loss-policy=fence, standby, revoked)"
 do_test ticket-primitive-22 "Ticket - Primitive (loss-policy=freeze, standby, granted)"
 do_test ticket-primitive-23 "Ticket - Primitive (loss-policy=freeze, granted, standby)"
 do_test ticket-primitive-24 "Ticket - Primitive (loss-policy=freeze, standby, revoked)"
 
 echo""
 do_test ticket-group-1 "Ticket - Group (loss-policy=stop, initial)"
 do_test ticket-group-2 "Ticket - Group (loss-policy=stop, granted)"
 do_test ticket-group-3 "Ticket - Group (loss-policy-stop, revoked)"
 do_test ticket-group-4 "Ticket - Group (loss-policy=demote, initial)"
 do_test ticket-group-5 "Ticket - Group (loss-policy=demote, granted)"
 do_test ticket-group-6 "Ticket - Group (loss-policy=demote, revoked)"
 do_test ticket-group-7 "Ticket - Group (loss-policy=fence, initial)"
 do_test ticket-group-8 "Ticket - Group (loss-policy=fence, granted)"
 do_test ticket-group-9 "Ticket - Group (loss-policy=fence, revoked)"
 do_test ticket-group-10 "Ticket - Group (loss-policy=freeze, initial)"
 do_test ticket-group-11 "Ticket - Group (loss-policy=freeze, granted)"
 do_test ticket-group-12 "Ticket - Group (loss-policy=freeze, revoked)"
 
 do_test ticket-group-13 "Ticket - Group (loss-policy=stop, standby, granted)"
 do_test ticket-group-14 "Ticket - Group (loss-policy=stop, granted, standby)"
 do_test ticket-group-15 "Ticket - Group (loss-policy=stop, standby, revoked)"
 do_test ticket-group-16 "Ticket - Group (loss-policy=demote, standby, granted)"
 do_test ticket-group-17 "Ticket - Group (loss-policy=demote, granted, standby)"
 do_test ticket-group-18 "Ticket - Group (loss-policy=demote, standby, revoked)"
 do_test ticket-group-19 "Ticket - Group (loss-policy=fence, standby, granted)"
 do_test ticket-group-20 "Ticket - Group (loss-policy=fence, granted, standby)"
 do_test ticket-group-21 "Ticket - Group (loss-policy=fence, standby, revoked)"
 do_test ticket-group-22 "Ticket - Group (loss-policy=freeze, standby, granted)"
 do_test ticket-group-23 "Ticket - Group (loss-policy=freeze, granted, standby)"
 do_test ticket-group-24 "Ticket - Group (loss-policy=freeze, standby, revoked)"
 
 echo""
 do_test ticket-clone-1 "Ticket - Clone (loss-policy=stop, initial)"
 do_test ticket-clone-2 "Ticket - Clone (loss-policy=stop, granted)"
 do_test ticket-clone-3 "Ticket - Clone (loss-policy-stop, revoked)"
 do_test ticket-clone-4 "Ticket - Clone (loss-policy=demote, initial)"
 do_test ticket-clone-5 "Ticket - Clone (loss-policy=demote, granted)"
 do_test ticket-clone-6 "Ticket - Clone (loss-policy=demote, revoked)"
 do_test ticket-clone-7 "Ticket - Clone (loss-policy=fence, initial)"
 do_test ticket-clone-8 "Ticket - Clone (loss-policy=fence, granted)"
 do_test ticket-clone-9 "Ticket - Clone (loss-policy=fence, revoked)"
 do_test ticket-clone-10 "Ticket - Clone (loss-policy=freeze, initial)"
 do_test ticket-clone-11 "Ticket - Clone (loss-policy=freeze, granted)"
 do_test ticket-clone-12 "Ticket - Clone (loss-policy=freeze, revoked)"
 
 do_test ticket-clone-13 "Ticket - Clone (loss-policy=stop, standby, granted)"
 do_test ticket-clone-14 "Ticket - Clone (loss-policy=stop, granted, standby)"
 do_test ticket-clone-15 "Ticket - Clone (loss-policy=stop, standby, revoked)"
 do_test ticket-clone-16 "Ticket - Clone (loss-policy=demote, standby, granted)"
 do_test ticket-clone-17 "Ticket - Clone (loss-policy=demote, granted, standby)"
 do_test ticket-clone-18 "Ticket - Clone (loss-policy=demote, standby, revoked)"
 do_test ticket-clone-19 "Ticket - Clone (loss-policy=fence, standby, granted)"
 do_test ticket-clone-20 "Ticket - Clone (loss-policy=fence, granted, standby)"
 do_test ticket-clone-21 "Ticket - Clone (loss-policy=fence, standby, revoked)"
 do_test ticket-clone-22 "Ticket - Clone (loss-policy=freeze, standby, granted)"
 do_test ticket-clone-23 "Ticket - Clone (loss-policy=freeze, granted, standby)"
 do_test ticket-clone-24 "Ticket - Clone (loss-policy=freeze, standby, revoked)"
 
 echo""
 do_test ticket-master-1 "Ticket - Master (loss-policy=stop, initial)"
 do_test ticket-master-2 "Ticket - Master (loss-policy=stop, granted)"
 do_test ticket-master-3 "Ticket - Master (loss-policy-stop, revoked)"
 do_test ticket-master-4 "Ticket - Master (loss-policy=demote, initial)"
 do_test ticket-master-5 "Ticket - Master (loss-policy=demote, granted)"
 do_test ticket-master-6 "Ticket - Master (loss-policy=demote, revoked)"
 do_test ticket-master-7 "Ticket - Master (loss-policy=fence, initial)"
 do_test ticket-master-8 "Ticket - Master (loss-policy=fence, granted)"
 do_test ticket-master-9 "Ticket - Master (loss-policy=fence, revoked)"
 do_test ticket-master-10 "Ticket - Master (loss-policy=freeze, initial)"
 do_test ticket-master-11 "Ticket - Master (loss-policy=freeze, granted)"
 do_test ticket-master-12 "Ticket - Master (loss-policy=freeze, revoked)"
 
 do_test ticket-master-13 "Ticket - Master (loss-policy=stop, standby, granted)"
 do_test ticket-master-14 "Ticket - Master (loss-policy=stop, granted, standby)"
 do_test ticket-master-15 "Ticket - Master (loss-policy=stop, standby, revoked)"
 do_test ticket-master-16 "Ticket - Master (loss-policy=demote, standby, granted)"
 do_test ticket-master-17 "Ticket - Master (loss-policy=demote, granted, standby)"
 do_test ticket-master-18 "Ticket - Master (loss-policy=demote, standby, revoked)"
 do_test ticket-master-19 "Ticket - Master (loss-policy=fence, standby, granted)"
 do_test ticket-master-20 "Ticket - Master (loss-policy=fence, granted, standby)"
 do_test ticket-master-21 "Ticket - Master (loss-policy=fence, standby, revoked)"
 do_test ticket-master-22 "Ticket - Master (loss-policy=freeze, standby, granted)"
 do_test ticket-master-23 "Ticket - Master (loss-policy=freeze, granted, standby)"
 do_test ticket-master-24 "Ticket - Master (loss-policy=freeze, standby, revoked)"
 
 echo ""
 do_test ticket-rsc-sets-1 "Ticket - Resource sets (1 ticket, initial)"
 do_test ticket-rsc-sets-2 "Ticket - Resource sets (1 ticket, granted)"
 do_test ticket-rsc-sets-3 "Ticket - Resource sets (1 ticket, revoked)"
 do_test ticket-rsc-sets-4 "Ticket - Resource sets (2 tickets, initial)"
 do_test ticket-rsc-sets-5 "Ticket - Resource sets (2 tickets, granted)"
 do_test ticket-rsc-sets-6 "Ticket - Resource sets (2 tickets, granted)"
 do_test ticket-rsc-sets-7 "Ticket - Resource sets (2 tickets, revoked)"
 
 do_test ticket-rsc-sets-8 "Ticket - Resource sets (1 ticket, standby, granted)"
 do_test ticket-rsc-sets-9 "Ticket - Resource sets (1 ticket, granted, standby)"
 do_test ticket-rsc-sets-10 "Ticket - Resource sets (1 ticket, standby, revoked)"
 do_test ticket-rsc-sets-11 "Ticket - Resource sets (2 tickets, standby, granted)"
 do_test ticket-rsc-sets-12 "Ticket - Resource sets (2 tickets, standby, granted)"
 do_test ticket-rsc-sets-13 "Ticket - Resource sets (2 tickets, granted, standby)"
 do_test ticket-rsc-sets-14 "Ticket - Resource sets (2 tickets, standby, revoked)"
 
 do_test cluster-specific-params "Cluster-specific instance attributes based on rules"
 do_test site-specific-params "Site-specific instance attributes based on rules"
 
 echo ""
 do_test template-1 "Template - 1"
 do_test template-2 "Template - 2"
 do_test template-3 "Template - 3 (merge operations)"
 
 do_test template-coloc-1 "Template - Colocation 1"
 do_test template-coloc-2 "Template - Colocation 2"
 do_test template-coloc-3 "Template - Colocation 3"
 do_test template-order-1 "Template - Order 1"
 do_test template-order-2 "Template - Order 2"
 do_test template-order-3 "Template - Order 3"
 do_test template-ticket  "Template - Ticket"
 
 do_test template-rsc-sets-1  "Template - Resource Sets 1"
 do_test template-rsc-sets-2  "Template - Resource Sets 2"
 do_test template-rsc-sets-3  "Template - Resource Sets 3"
 do_test template-rsc-sets-4  "Template - Resource Sets 4"
 
 do_test template-clone-primitive "Cloned primitive from template"
 do_test template-clone-group     "Cloned group from template"
 
 do_test location-sets-templates "Resource sets and templates - Location"
 
 do_test tags-coloc-order-1 "Tags - Colocation and Order (Simple)"
 do_test tags-coloc-order-2 "Tags - Colocation and Order (Resource Sets with Templates)"
 do_test tags-location      "Tags - Location"
 do_test tags-ticket        "Tags - Ticket"
 
 echo ""
 do_test container-1 "Container - initial"
 do_test container-2 "Container - monitor failed"
 do_test container-3 "Container - stop failed"
 do_test container-4 "Container - reached migration-threshold"
 do_test container-group-1 "Container in group - initial"
 do_test container-group-2 "Container in group - monitor failed"
 do_test container-group-3 "Container in group - stop failed"
 do_test container-group-4 "Container in group - reached migration-threshold"
 do_test container-is-remote-node "Place resource within container when container is remote-node"
 do_test bug-rh-1097457 "Kill user defined container/contents ordering"
 do_test bug-cl-5247 "Graph loop when recovering m/s resource in a container"
 
 do_test bundle-order-startup "Bundle startup ordering"
 do_test bundle-order-partial-start "Bundle startup ordering when some dependancies are already running"
 do_test bundle-order-partial-start-2 "Bundle startup ordering when some dependancies and the container are already running"
 do_test bundle-order-stop    "Bundle stop ordering"
 do_test bundle-order-partial-stop "Bundle startup ordering when some dependancies are already stopped"
 
 do_test bundle-order-startup-clone "Prevent startup because bundle isn't promoted"
 do_test bundle-order-startup-clone-2 "Bundle startup with clones"
 do_test bundle-order-stop-clone "Stop bundle because clone is stopping"
 do_test bundle-nested-colocation "Colocation of nested connection resources"
 
 do_test bundle-order-fencing "Order pseudo bundle fencing after parent node fencing if both are happening"
 
 do_test bundle-probe-order-1 "order 1"
 do_test bundle-probe-order-2 "order 2"
 do_test bundle-probe-order-3 "order 3"
 do_test bundle-probe-remotes "Ensure remotes get probed too"
 
 echo ""
 do_test whitebox-fail1 "Fail whitebox container rsc."
 do_test whitebox-fail2 "Fail whitebox container rsc lrmd connection."
 do_test whitebox-fail3 "Failed containers should not run nested on remote nodes."
 do_test whitebox-start "Start whitebox container with resources assigned to it"
 do_test whitebox-stop "Stop whitebox container with resources assigned to it"
 do_test whitebox-move "Move whitebox container with resources assigned to it"
 do_test whitebox-asymmetric "Verify connection rsc opts-in based on container resource"
 do_test whitebox-ms-ordering "Verify promote/demote can not occur before connection is established"
 do_test whitebox-ms-ordering-move "Stop/Start cycle within a moving container"
 do_test whitebox-orphaned    "Properly shutdown orphaned whitebox container"
 do_test whitebox-orphan-ms   "Properly tear down orphan ms resources on remote-nodes"
 do_test whitebox-unexpectedly-running "Recover container nodes the cluster did not start."
 do_test whitebox-migrate1 "Migrate both container and connection resource"
 do_test whitebox-imply-stop-on-fence "imply stop action on container node rsc when host node is fenced"
 do_test whitebox-nested-group "Verify guest remote-node works nested in a group"
 do_test guest-node-host-dies "Verify guest node is recovered if host goes away"
 
 echo ""
 do_test remote-startup-probes  "Baremetal remote-node startup probes"
 do_test remote-startup         "Startup a newly discovered remote-nodes with no status."
 do_test remote-fence-unclean   "Fence unclean baremetal remote-node"
 do_test remote-fence-unclean2  "Fence baremetal remote-node after cluster node fails and connection can not be recovered"
 do_test remote-fence-unclean-3 "Probe failed remote nodes (triggers fencing)"
 do_test remote-move            "Move remote-node connection resource"
 do_test remote-disable         "Disable a baremetal remote-node"
 do_test remote-probe-disable   "Probe then stop a baremetal remote-node"
 do_test remote-orphaned        "Properly shutdown orphaned connection resource"
 do_test remote-orphaned2       "verify we can handle orphaned remote connections with active resources on the remote"
 do_test remote-recover         "Recover connection resource after cluster-node fails."
 do_test remote-stale-node-entry "Make sure we properly handle leftover remote-node entries in the node section"
 do_test remote-partial-migrate  "Make sure partial migrations are handled before ops on the remote node."
 do_test remote-partial-migrate2 "Make sure partial migration target is prefered for remote connection."
 do_test remote-recover-fail     "Make sure start failure causes fencing if rsc are active on remote."
 do_test remote-start-fail       "Make sure a start failure does not result in fencing if no active resources are on remote."
 do_test remote-unclean2         "Make monitor failure always results in fencing, even if no rsc are active on remote."
 do_test remote-fence-before-reconnect "Fence before clearing recurring monitor failure"
 do_test remote-recovery		"Recover remote connections before attempting demotion"
 do_test remote-recover-connection "Optimistically recovery of only the connection"
 do_test remote-recover-all        "Fencing when the connection has no home"
 do_test remote-recover-no-resources   "Fencing when the connection has no home and no active resources"
 do_test remote-recover-unknown        "Fencing when the connection has no home and the remote has no operation history"
 do_test remote-reconnect-delay        "Waiting for remote reconnect interval to expire"
 do_test remote-connection-unrecoverable  "Remote connection host must be fenced, with connection unrecoverable"
 
 echo ""
 do_test resource-discovery      "Exercises resource-discovery location constraint option."
 do_test rsc-discovery-per-node  "Disable resource discovery per node"
 
 if [ $DO_VERSIONED_TESTS -eq 1 ]; then
 	echo ""
 	do_test versioned-resources     "Start resources with #ra-version rules"
 	do_test restart-versioned       "Restart resources on #ra-version change"
 	do_test reload-versioned        "Reload resources on #ra-version change"
 
 	echo ""
 	do_test versioned-operations-1  "Use #ra-version to configure operations of native resources"
 	do_test versioned-operations-2  "Use #ra-version to configure operations of stonith resources"
 	do_test versioned-operations-3  "Use #ra-version to configure operations of master/slave resources"
 	do_test versioned-operations-4  "Use #ra-version to configure operations of groups of the resources"
 fi
 
 echo ""
 test_results
diff --git a/cts/cts-regression b/cts/cts-regression
index ccb995b556..fbc6319f84 100755
--- a/cts/cts-regression
+++ b/cts/cts-regression
@@ -1,204 +1,204 @@
 #!/bin/bash
 #
 # cts-regression
 #
 # Convenience wrapper for running any of the Pacemaker regression tests
 #
 
 USAGE_TEXT="Usage: cts-regression [<options>] [<test> ...]
 Options:
  --help          Display this text, then exit
  -V, --verbose   Increase test verbosity
  -v, --valgrind  Run test commands under valgrind
 
 Tests (default tests are 'pengine cli lrmd'):
  pengine           Policy engine
  cli               Command-line tools
  lrmd              Local resource manager
  pacemaker_remote  Local resource manager in remote mode
  fencing           Fencing daemon
  all               Synonym for 'pengine cli lrmd fencing'"
 
 test_home="$(dirname $(readlink -e $0))"
 valgrind=""
 verbose=""
 tests=""
 
 # These constants must track crm_exit_t values
 CRM_EX_OK=0
 CRM_EX_ERROR=1
 CRM_EX_NOT_INSTALLED=5
 CRM_EX_USAGE=64
 
 function info() {
     printf "$*\n"
 }
 
 function error() {
     printf "      * ERROR:   $*\n"
 }
 
 function run_as_root() {
     CMD="$1"
     shift
     ARGS="$@"
 
     # Test might not be executable if run from source directory
     chmod a+x $CMD
 
     CMD="$CMD $ARGS $verbose"
 
     if [ $EUID -eq 0 ]; then
         $CMD
 
     elif [ -z $TRAVIS ]; then
         # sudo doesn't work in buildbot, su doesn't work in travis
         echo "Enter the root password..."
         su root -c "$CMD"
 
     else
         echo "Enter the root password if prompted..."
         sudo -- $CMD
     fi
 }
 
 add_test() {
     local TEST="$1"
 
     case "$TEST" in
         pengine|lrmd|pacemaker_remote|fencing|cli)
             if [[ ! $tests =~ $TEST ]]; then
                 tests="$tests $TEST"
             fi
             ;;
         *)
             error "unknown test: $TEST"
             echo
             echo "$USAGE_TEXT"
             exit $CRM_EX_USAGE
             ;;
     esac
 }
 
 run_test() {
     local t="$1"
 
     info "Executing the $t regression tests"
     info "============================================================"
     case $t in
         pengine)
-            if [ -x $test_home/pengine/regression.sh ]; then
-                $test_home/pengine/regression.sh $verbose $valgrind
+            if [ -x $test_home/cts-pengine ]; then
+                $test_home/cts-pengine $verbose $valgrind
                 rc=$?
             else
                 error "pengine regression test not found"
                 rc=$CRM_EX_NOT_INSTALLED
             fi
             ;;
         lrmd)
             if [ -x $test_home/cts-lrmd ]; then
                 run_as_root $test_home/cts-lrmd
                 rc=$?
             else
                 error "lrmd regression test not found"
                 rc=$CRM_EX_NOT_INSTALLED
             fi
             ;;
         pacemaker_remote)
             if [ -x $test_home/cts-lrmd ]; then
                 run_as_root $test_home/cts-lrmd -R
                 rc=$?
             else
                 error "pacemaker_remote regression test not found"
                 rc=$CRM_EX_NOT_INSTALLED
             fi
             ;;
         fencing)
             if [ -x $test_home/cts-stonithd ]; then
                 run_as_root $test_home/cts-stonithd
                 rc=$?
             else
                 error "fencing regression test not found"
                 rc=$CRM_EX_NOT_INSTALLED
             fi
             ;;
         cli)
             if [ -x $test_home/cts-cli ]; then
                 $test_home/cts-cli $verbose $valgrind
                 rc=$?
             else
                 error "cli regression test not found"
                 rc=$CRM_EX_NOT_INSTALLED
             fi
             ;;
     esac
 
     info "============================================================"
     info ""
     info ""
     return $rc
 }
 
 run_tests() {
     TESTS="$1"
     local TEST
     local TEST_RC
     local FAILED
 
     FAILED=""
     for TEST in $TESTS; do
         run_test $TEST
 	TEST_RC=$?
         if [ $TEST_RC -ne 0 ]; then
             info "$TEST regression tests failed ($TEST_RC)"
             FAILED="$FAILED $TEST"
         fi
     done
     if [ -n "$FAILED" ]; then
         error "failed regression tests: $FAILED"
         return $CRM_EX_ERROR
     fi
     return $CRM_EX_OK
 }
 
 while [ $# -gt 0 ] ; do
     case "$1" in
         --help)
             echo "$USAGE_TEXT"
             exit $CRM_EX_OK
             ;;
         -V|--verbose)
             verbose="-V"
             shift
             ;;
         -v|--valgrind)
             valgrind="-v"
             shift
             ;;
         pengine|lrmd|pacemaker_remote|fencing|cli)
             add_test $1
             shift
             ;;
         all)
             add_test pengine
             add_test cli
             add_test lrmd
             add_test fencing
             shift
             ;;
         *)
             error "unknown option: $1"
             echo
             echo "$USAGE_TEXT"
             exit $CRM_EX_USAGE
             ;;
     esac
 done
 
 if [ -z "$tests" ]; then
     add_test pengine
     add_test cli
     add_test lrmd
 fi
 
 run_tests $tests
diff --git a/pacemaker.spec.in b/pacemaker.spec.in
index aa748c5042..45d0e081f0 100644
--- a/pacemaker.spec.in
+++ b/pacemaker.spec.in
@@ -1,763 +1,763 @@
 # Globals and defines to control package behavior (configure these as desired)
 
 ## User and group to use for nonprivileged services
 %global uname hacluster
 %global gname haclient
 
 ## Where to install Pacemaker documentation
 %global pcmk_docdir %{_docdir}/%{name}
 
 ## GitHub entity that distributes source (for ease of using a fork)
 %global github_owner ClusterLabs
 
 ## Upstream pacemaker version, and its package version (specversion
 ## can be incremented to build packages reliably considered "newer"
 ## than previously built packages with the same pcmkversion)
 %global pcmkversion 1.1.18
 %global specversion 1
 
 ## Upstream commit (or git tag, such as "Pacemaker-" plus the
 ## {pcmkversion} macro for an official release) to use for this package
 %global commit HEAD
 ## Since git v2.11, the extent of abbreviation is autoscaled by default
 ## (used to be constant of 7), so we need to convey it for non-tags, too.
 %global commit_abbrev 7
 
 ## Path to Python interpreter (leave commented to auto-detect,
 ## or uncomment and edit to use a specific version)
 #%%global python_path /usr/bin/python%{python3_pkgversion} 
 
 # Define globals for convenient use later
 
 ## Workaround to use parentheses in other globals
 %global lparen (
 %global rparen )
 
 ## Short version of git commit
 %define shortcommit %(c=%{commit}; case ${c} in
                       Pacemaker-*%{rparen} echo ${c:10};;
                       *%{rparen} echo ${c:0:%{commit_abbrev}};; esac)
 
 ## Whether this is a tagged release
 %define tag_release %([ %{commit} != Pacemaker-%{shortcommit} ]; echo $?)
 
 ## Whether this is a release candidate (in case of a tagged release)
 %define pre_release %([ "%{tag_release}" -eq 0 ] || {
                       case "%{shortcommit}" in *-rc[[:digit:]]*%{rparen} false;;
                       esac; }; echo $?)
 
 ## Turn off auto-compilation of python files outside site-packages directory,
 ## so that the -libs-devel package is multilib-compliant (no *.py[co] files)
 %global __os_install_post %(echo '%{__os_install_post}' | {
                             sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g'; })
 
 ## Heuristic used to infer bleeding-edge deployments that are
 ## less likely to have working versions of the documentation tools
 %define bleeding %(test ! -e /etc/yum.repos.d/fedora-rawhide.repo; echo $?)
 
 ## Corosync version
 %define cs_version %(pkg-config corosync --modversion 2>/dev/null | awk -F . '{print $1}')
 
 ## Where to install python site libraries (currently, this uses the unversioned
 ## python_sitearch macro to get the default system python, but at some point,
 ## we should explicitly choose python2_sitearch or python3_sitearch -- or both)
 %define py_site %{?python_sitearch}%{!?python_sitearch:%(
   python -c 'from distutils.sysconfig import get_python_lib as gpl; print(gpl(1))' 2>/dev/null)}
 
 ## Whether this platform defaults to using systemd as an init system
 ## (needs to be evaluated prior to BuildRequires being enumerated and
 ## installed as it's intended to conditionally select some of these, and
 ## for that there are only few indicators with varying reliability:
 ## - presence of systemd-defined macros (when building in a full-fledged
 ##   environment, which is not the case with ordinary mock-based builds)
 ## - systemd-aware rpm as manifested with the presence of particular
 ##   macro (rpm itself will trivially always be present when building)
 ## - existence of /usr/lib/os-release file, which is something heavily
 ##   propagated by systemd project
 ## - when not good enough, there's always a possibility to check
 ##   particular distro-specific macros (incl. version comparison)
 %define systemd_native (%{?_unitdir:1}%{?!_unitdir:0}%{nil \
   } || %{?__transaction_systemd_inhibit:1}%{?!__transaction_systemd_inhibit:0}%{nil \
   } || %(test -f /usr/lib/os-release; test $? -ne 0; echo $?))
 
 # Definitions for backward compatibility with older RPM versions
 
 ## Ensure the license macro behaves consistently (older RPM will otherwise
 ## overwrite it once it encounters "License:"). Courtesy Jason Tibbitts:
 ## https://pkgs.fedoraproject.org/cgit/rpms/epel-rpm-macros.git/tree/macros.zzz-epel?h=el6&id=e1adcb77
 %if !%{defined _licensedir}
 %define description %{lua:
     rpm.define("license %doc")
     print("%description")
 }
 %endif
 
 
 # Define conditionals so that "rpmbuild --with <feature>" and
 # "rpmbuild --without <feature>" can enable and disable specific features
 
 ## Add option to enable support for stonith/external fencing agents
 %bcond_with stonithd
 
 ## Add option to create binaries suitable for use with profiling tools
 %bcond_with profiling
 
 ## Add option to create binaries with coverage analysis
 %bcond_with coverage
 
 ## Add option to skip generating documentation
 ## (the build tools aren't available everywhere)
 %bcond_without doc
 
 ## Add option to prefix package version with "0."
 ## (so later "official" packages will be considered updates)
 %bcond_with pre_release
 
 ## Add option to ship Upstart job files
 %bcond_with upstart_job
 
 ## Add option to turn off hardening of libraries and daemon executables
 %bcond_without hardening
 
 
 # Keep sane profiling data if requested
 %if %{with profiling}
 
 ## Disable -debuginfo package and stripping binaries/libraries
 %define debug_package %{nil}
 
 %endif
 
 
 # Define the release version
 # (do not look at externally enforced pre-release flag for tagged releases
 # as only -rc tags, captured with the second condition, implies that then)
 %if (!%{tag_release} && %{with pre_release}) || 0%{pre_release}
 %if 0%{pre_release}
 %define pcmk_release 0.%{specversion}.%(s=%{shortcommit}; echo ${s: -3})
 %else
 %define pcmk_release 0.%{specversion}.%{shortcommit}.git
 %endif
 %else
 %if 0%{tag_release}
 %define pcmk_release %{specversion}
 %else
 %define pcmk_release %{specversion}.%{shortcommit}.git
 %endif
 %endif
 
 Name:          pacemaker
 Summary:       Scalable High-Availability cluster resource manager
 Version:       %{pcmkversion}
 Release:       %{pcmk_release}%{?dist}
 %if %{defined _unitdir}
 License:       GPLv2+ and LGPLv2+
 %else
 # initscript is Revised BSD
 License:       GPLv2+ and LGPLv2+ and BSD
 %endif
 Url:           http://www.clusterlabs.org
 Group:         System Environment/Daemons
 
 # Hint: use "spectool -s 0 pacemaker.spec" (rpmdevtools) to check the final URL:
 # https://github.com/ClusterLabs/pacemaker/archive/e91769e5a39f5cb2f7b097d3c612368f0530535e/pacemaker-e91769e.tar.gz
 Source0:       https://github.com/%{github_owner}/%{name}/archive/%{commit}/%{name}-%{shortcommit}.tar.gz
 BuildRoot:     %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
 AutoReqProv:   on
 Requires:      resource-agents
 Requires:      %{name}-libs = %{version}-%{release}
 Requires:      %{name}-cluster-libs = %{version}-%{release}
 Requires:      %{name}-cli = %{version}-%{release}
 
 %if %{defined systemd_requires}
 %systemd_requires
 %endif
 
 # Pacemaker targets compatibility with python 2.7 and 3.2+
 Requires:      python >= 2.7
 BuildRequires: python-devel >= 2.7
 
 # Pacemaker requires a minimum libqb functionality
 Requires:      libqb >= 0.13.0
 BuildRequires: libqb-devel >= 0.13.0
 
 # Basics required for the build (even if usually satisfied through other BRs)
 BuildRequires: coreutils findutils grep sed
 
 # Required for core functionality
 BuildRequires: automake autoconf libtool pkgconfig libtool-ltdl-devel
 BuildRequires: pkgconfig(glib-2.0) >= 2.16
 BuildRequires: libxml2-devel libxslt-devel libuuid-devel
 BuildRequires: bzip2-devel pam-devel
 
 # Required for agent_config.h which specifies the correct scratch directory
 BuildRequires: resource-agents
 
 # Enables optional functionality
 BuildRequires: ncurses-devel docbook-style-xsl
 BuildRequires: bison byacc flex help2man gnutls-devel pkgconfig(dbus-1)
 
 %if %{systemd_native}
 BuildRequires: pkgconfig(systemd)
 %endif
 
 Requires:      corosync
 BuildRequires: corosynclib-devel
 
 %if %{with stonithd}
 BuildRequires: cluster-glue-libs-devel
 %endif
 
 ## (note no avoiding effect when building through non-customized mock)
 %if !%{bleeding}
 %if %{with doc}
 BuildRequires: publican inkscape asciidoc
 %endif
 %endif
 
 %description
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 It supports more than 16 node clusters with significant capabilities
 for managing resources and dependencies.
 
 It will run scripts at initialization, when machines go up or down,
 when related resources fail and can be configured to periodically check
 resource health.
 
 Available rpmbuild rebuild options:
   --with(out) : coverage doc stonithd hardening pre_release profiling
                 upstart_job
 
 %package cli
 License:       GPLv2+ and LGPLv2+
 Summary:       Command line tools for controlling Pacemaker clusters
 Group:         System Environment/Daemons
 Requires:      %{name}-libs = %{version}-%{release}
 Requires:      perl-TimeDate
 
 %description cli
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 The %{name}-cli package contains command line tools that can be used
 to query and control the cluster from machines that may, or may not,
 be part of the cluster.
 
 %package -n %{name}-libs
 License:       GPLv2+ and LGPLv2+
 Summary:       Core Pacemaker libraries
 Group:         System Environment/Daemons
 
 %description -n %{name}-libs
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 The %{name}-libs package contains shared libraries needed for cluster
 nodes and those just running the CLI tools.
 
 %package -n %{name}-cluster-libs
 License:       GPLv2+ and LGPLv2+
 Summary:       Cluster Libraries used by Pacemaker
 Group:         System Environment/Daemons
 Requires:      %{name}-libs = %{version}-%{release}
 
 %description -n %{name}-cluster-libs
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 The %{name}-cluster-libs package contains cluster-aware shared
 libraries needed for nodes that will form part of the cluster nodes.
 
 %package remote
 %if %{defined _unitdir}
 License:       GPLv2+ and LGPLv2+
 %else
 # initscript is Revised BSD
 License:       GPLv2+ and LGPLv2+ and BSD
 %endif
 Summary:       Pacemaker remote daemon for non-cluster nodes
 Group:         System Environment/Daemons
 Requires:      %{name}-libs = %{version}-%{release}
 Requires:      %{name}-cli = %{version}-%{release}
 Requires:      resource-agents
 %if %{defined systemd_requires}
 %systemd_requires
 %endif
 
 %description remote
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 The %{name}-remote package contains the Pacemaker Remote daemon
 which is capable of extending pacemaker functionality to remote
 nodes not running the full corosync/cluster stack.
 
 %package -n %{name}-libs-devel
 License:       GPLv2+ and LGPLv2+
 Summary:       Pacemaker development package
 Group:         Development/Libraries
 Requires:      %{name}-cts = %{version}-%{release}
 Requires:      %{name}-libs%{?_isa} = %{version}-%{release}
 Requires:      %{name}-cluster-libs%{?_isa} = %{version}-%{release}
 Requires:      libtool-ltdl-devel%{?_isa} libuuid-devel%{?_isa}
 Requires:      libxml2-devel%{?_isa} libxslt-devel%{?_isa}
 Requires:      bzip2-devel%{?_isa} glib2-devel%{?_isa}
 Requires:      libqb-devel%{?_isa} corosynclib-devel%{?_isa}
 
 %description -n %{name}-libs-devel
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 The %{name}-libs-devel package contains headers and shared libraries
 for developing tools for Pacemaker.
 
 # NOTE: can be noarch if lrmd_test is moved to another subpackage
 %package       cts
 License:       GPLv2+ and LGPLv2+
 Summary:       Test framework for cluster-related technologies like Pacemaker
 Group:         System Environment/Daemons
 Requires:      python >= 2.7
 Requires:      %{name}-libs = %{version}-%{release}
 
 # systemd python bindings are separate package in some distros
 %if %{defined systemd_requires}
 
 %if 0%{?fedora} > 22
 Requires:      python2-systemd
 %else
 %if 0%{?fedora} > 20 || 0%{?rhel} > 6
 Requires:      systemd-python
 %endif
 %endif
 
 %endif
 
 %description   cts
 Test framework for cluster-related technologies like Pacemaker
 
 %package       doc
 License:       CC-BY-SA
 Summary:       Documentation for Pacemaker
 Group:         Documentation
 
 %description   doc
 Documentation for Pacemaker.
 
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 %prep
 %setup -q -n %{name}-%{commit}
 
 # Force the local time
 #
 # 'git' sets the file date to the date of the last commit.
 # This can result in files having been created in the future
 # when building on machines in timezones 'behind' the one the
 # commit occurred in - which seriously confuses 'make'
 find . -exec touch \{\} \;
 
 %build
 
 # Early versions of autotools (e.g. RHEL <= 5) do not support --docdir
 export docdir=%{pcmk_docdir}
 
 export systemdunitdir=%{?_unitdir}%{?!_unitdir:no}
 
 %if %{with hardening}
 # prefer distro-provided hardening flags in case they are defined
 # through _hardening_{c,ld}flags macros, configure script will
 # use its own defaults otherwise; if such hardenings are completely
 # undesired, rpmbuild using "--without hardening"
 # (or "--define '_without_hardening 1'")
 export CFLAGS_HARDENED_EXE="%{?_hardening_cflags}"
 export CFLAGS_HARDENED_LIB="%{?_hardening_cflags}"
 export LDFLAGS_HARDENED_EXE="%{?_hardening_ldflags}"
 export LDFLAGS_HARDENED_LIB="%{?_hardening_ldflags}"
 %endif
 
 ./autogen.sh
 
 %{configure}                                       \
         %{?with_profiling:   --with-profiling}     \
         %{?with_coverage:    --with-coverage}      \
         %{!?with_doc:        --with-brand=}        \
         %{!?with_hardening:  --disable-hardening}  \
         %{?python_path:      PYTHON=%{python_path}} \
         --with-initdir=%{_initrddir}               \
         --localstatedir=%{_var}                    \
         --with-version=%{version}-%{release}
 
 %if 0%{?suse_version} >= 1200
 # Fedora handles rpath removal automagically
 sed -i 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|g' libtool
 sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool
 %endif
 
 make %{_smp_mflags} V=1 all
 
 %check
-{ pengine/regression.sh --run one-or-more-unrunnable-instances \
+{ cts/cts-pengine --run one-or-more-unrunnable-instances \
   && cts/cts-cli \
   && touch .CHECKED
 } 2>&1 | sed 's/[fF]ail/faiil/g'  # prevent false positives in rpmlint
 [ -f .CHECKED ] && rm -f -- .CHECKED || false
 
 %install
 rm -rf %{buildroot}
 make DESTDIR=%{buildroot} docdir=%{pcmk_docdir} V=1 install
 
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/sysconfig
 install -m 644 mcp/pacemaker.sysconfig ${RPM_BUILD_ROOT}%{_sysconfdir}/sysconfig/pacemaker
 install -m 644 tools/crm_mon.sysconfig ${RPM_BUILD_ROOT}%{_sysconfdir}/sysconfig/crm_mon
 
 %if %{with upstart_job}
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/init
 install -m 644 mcp/pacemaker.upstart ${RPM_BUILD_ROOT}%{_sysconfdir}/init/pacemaker.conf
 install -m 644 mcp/pacemaker.combined.upstart ${RPM_BUILD_ROOT}%{_sysconfdir}/init/pacemaker.combined.conf
 install -m 644 tools/crm_mon.upstart ${RPM_BUILD_ROOT}%{_sysconfdir}/init/crm_mon.conf
 %endif
 
 %if %{defined _unitdir}
 mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/lib/rpm-state/%{name}
 %endif
 
 # Scripts that should be executable
 chmod a+x %{buildroot}/%{_datadir}/pacemaker/tests/cts/CTSlab.py
 
 # These are not actually scripts
 find %{buildroot} -name '*.xml' -type f -print0 | xargs -0 chmod a-x
 
 # Don't package static libs
 find %{buildroot} -name '*.a' -type f -print0 | xargs -0 rm -f
 find %{buildroot} -name '*.la' -type f -print0 | xargs -0 rm -f
 
 # Do not package these either
 rm -f %{buildroot}/%{_libdir}/service_crm.so
 
 # Don't ship init scripts for systemd based platforms
 %if %{defined _unitdir}
 rm -f %{buildroot}/%{_initrddir}/pacemaker
 rm -f %{buildroot}/%{_initrddir}/pacemaker_remote
 %endif
 
 %if %{with coverage}
 GCOV_BASE=%{buildroot}/%{_var}/lib/pacemaker/gcov
 mkdir -p $GCOV_BASE
 find . -name '*.gcno' -type f | while read F ; do
         D=`dirname $F`
         mkdir -p ${GCOV_BASE}/$D
         cp $F ${GCOV_BASE}/$D
 done
 %endif
 
 %clean
 rm -rf %{buildroot}
 
 %post
 %if %{defined _unitdir}
 %systemd_post pacemaker.service
 %else
 /sbin/chkconfig --add pacemaker || :
 %endif
 
 %preun
 %if %{defined _unitdir}
 %systemd_preun pacemaker.service
 %else
 /sbin/service pacemaker stop >/dev/null 2>&1 || :
 if [ $1 -eq 0 ]; then
     # Package removal, not upgrade
     /sbin/chkconfig --del pacemaker || :
 fi
 %endif
 
 %postun
 %if %{defined _unitdir}
 %systemd_postun_with_restart pacemaker.service
 %endif
 
 %pre remote
 %if %{defined _unitdir}
 # Stop the service before anything is touched, and remember to restart
 # it as one of the last actions (compared to using systemd_postun_with_restart,
 # this avoids suicide when sbd is in use)
 systemctl --quiet is-active pacemaker_remote
 if [ $? -eq 0 ] ; then
     mkdir -p %{_localstatedir}/lib/rpm-state/%{name}
     touch %{_localstatedir}/lib/rpm-state/%{name}/restart_pacemaker_remote
     systemctl stop pacemaker_remote >/dev/null 2>&1
 else
     rm -f %{_localstatedir}/lib/rpm-state/%{name}/restart_pacemaker_remote
 fi
 %endif
 
 %post remote
 %if %{defined _unitdir}
 %systemd_post pacemaker_remote.service
 %else
 /sbin/chkconfig --add pacemaker_remote || :
 %endif
 
 %preun remote
 %if %{defined _unitdir}
 %systemd_preun pacemaker_remote.service
 %else
 /sbin/service pacemaker_remote stop >/dev/null 2>&1 || :
 if [ $1 -eq 0 ]; then
     # Package removal, not upgrade
     /sbin/chkconfig --del pacemaker_remote || :
 fi
 %endif
 
 %postun remote
 %if %{defined _unitdir}
 # This next line is a no-op, because we stopped the service earlier, but
 # we leave it here because it allows us to revert to the standard behavior
 # in the future if desired
 %systemd_postun_with_restart pacemaker_remote.service
 # Explicitly take care of removing the flag-file(s) upon final removal
 if [ $1 -eq 0 ] ; then
     rm -f %{_localstatedir}/lib/rpm-state/%{name}/restart_pacemaker_remote
 fi
 %endif
 
 %posttrans remote
 %if %{defined _unitdir}
 if [ -e %{_localstatedir}/lib/rpm-state/%{name}/restart_pacemaker_remote ] ; then
     systemctl start pacemaker_remote >/dev/null 2>&1
     rm -f %{_localstatedir}/lib/rpm-state/%{name}/restart_pacemaker_remote
 fi
 %endif
 
 %post cli
 %if %{defined _unitdir}
 %systemd_post crm_mon.service
 %endif
 
 %preun cli
 %if %{defined _unitdir}
 %systemd_preun crm_mon.service
 %endif
 
 %postun cli
 %if %{defined _unitdir}
 %systemd_postun_with_restart crm_mon.service
 %endif
 
 %pre -n %{name}-libs
 
 getent group %{gname} >/dev/null || groupadd -r %{gname} -g 189
 getent passwd %{uname} >/dev/null || useradd -r -g %{gname} -u 189 -s /sbin/nologin -c "cluster user" %{uname}
 exit 0
 
 %post -n %{name}-libs -p /sbin/ldconfig
 
 %postun -n %{name}-libs -p /sbin/ldconfig
 
 %post -n %{name}-cluster-libs -p /sbin/ldconfig
 
 %postun -n %{name}-cluster-libs -p /sbin/ldconfig
 
 %files
 ###########################################################
 %defattr(-,root,root)
 
 %config(noreplace) %{_sysconfdir}/sysconfig/pacemaker
 %{_sbindir}/pacemakerd
 
 %if %{defined _unitdir}
 %{_unitdir}/pacemaker.service
 %else
 %{_initrddir}/pacemaker
 %endif
 
 %exclude %{_libexecdir}/pacemaker/lrmd_test
 %exclude %{_sbindir}/pacemaker_remoted
 %{_libexecdir}/pacemaker/*
 
 %{_sbindir}/crm_attribute
 %{_sbindir}/crm_master
 %{_sbindir}/crm_node
 %{_sbindir}/fence_legacy
 %{_sbindir}/stonith_admin
 
 %doc %{_mandir}/man7/crmd.*
 %doc %{_mandir}/man7/pengine.*
 %doc %{_mandir}/man7/stonithd.*
 %doc %{_mandir}/man7/ocf_pacemaker_controld.*
 %doc %{_mandir}/man7/ocf_pacemaker_o2cb.*
 %doc %{_mandir}/man7/ocf_pacemaker_remote.*
 %doc %{_mandir}/man8/crm_attribute.*
 %doc %{_mandir}/man8/crm_node.*
 %doc %{_mandir}/man8/crm_master.*
 %doc %{_mandir}/man8/fence_legacy.*
 %doc %{_mandir}/man8/pacemakerd.*
 %doc %{_mandir}/man8/stonith_admin.*
 
 %doc %{_datadir}/pacemaker/alerts
 
 %license licenses/GPLv2
 %doc COPYING
 %doc ChangeLog
 
 %dir %attr (750, %{uname}, %{gname}) %{_var}/lib/pacemaker/cib
 %dir %attr (750, %{uname}, %{gname}) %{_var}/lib/pacemaker/pengine
 /usr/lib/ocf/resource.d/pacemaker/controld
 /usr/lib/ocf/resource.d/pacemaker/o2cb
 /usr/lib/ocf/resource.d/pacemaker/remote
 
 %if "%{?cs_version}" != "UNKNOWN"
 %if 0%{?cs_version} < 2
 %{_libexecdir}/lcrso/pacemaker.lcrso
 %endif
 %endif
 
 %if %{with upstart_job}
 %config(noreplace) %{_sysconfdir}/init/pacemaker.conf
 %config(noreplace) %{_sysconfdir}/init/pacemaker.combined.conf
 %endif
 
 %files cli
 %defattr(-,root,root)
 
 %config(noreplace) %{_sysconfdir}/logrotate.d/pacemaker
 %config(noreplace) %{_sysconfdir}/sysconfig/crm_mon
 
 %if %{defined _unitdir}
 %{_unitdir}/crm_mon.service
 %endif
 
 %if %{with upstart_job}
 %config(noreplace) %{_sysconfdir}/init/crm_mon.conf
 %endif
 
 %{_sbindir}/attrd_updater
 %{_sbindir}/cibadmin
 %{_sbindir}/crm_diff
 %{_sbindir}/crm_error
 %{_sbindir}/crm_failcount
 %{_sbindir}/crm_mon
 %{_sbindir}/crm_resource
 %{_sbindir}/crm_standby
 %{_sbindir}/crm_verify
 %{_sbindir}/crmadmin
 %{_sbindir}/iso8601
 %{_sbindir}/crm_shadow
 %{_sbindir}/crm_simulate
 %{_sbindir}/crm_report
 %{_sbindir}/crm_ticket
 %exclude %{_datadir}/pacemaker/alerts
 %exclude %{_datadir}/pacemaker/tests
 %{_datadir}/pacemaker
 %{_datadir}/snmp/mibs/PCMK-MIB.txt
 
 %exclude /usr/lib/ocf/resource.d/pacemaker/controld
 %exclude /usr/lib/ocf/resource.d/pacemaker/o2cb
 %exclude /usr/lib/ocf/resource.d/pacemaker/remote
 
 %dir /usr/lib/ocf
 %dir /usr/lib/ocf/resource.d
 /usr/lib/ocf/resource.d/pacemaker
 
 %doc %{_mandir}/man7/*
 %exclude %{_mandir}/man7/crmd.*
 %exclude %{_mandir}/man7/pengine.*
 %exclude %{_mandir}/man7/stonithd.*
 %exclude %{_mandir}/man7/ocf_pacemaker_controld.*
 %exclude %{_mandir}/man7/ocf_pacemaker_o2cb.*
 %exclude %{_mandir}/man7/ocf_pacemaker_remote.*
 %doc %{_mandir}/man8/*
 %exclude %{_mandir}/man8/crm_attribute.*
 %exclude %{_mandir}/man8/crm_node.*
 %exclude %{_mandir}/man8/crm_master.*
 %exclude %{_mandir}/man8/fence_legacy.*
 %exclude %{_mandir}/man8/pacemakerd.*
 %exclude %{_mandir}/man8/pacemaker_remoted.*
 %exclude %{_mandir}/man8/stonith_admin.*
 
 %license licenses/GPLv2
 %doc COPYING
 %doc ChangeLog
 
 %dir %attr (750, %{uname}, %{gname}) %{_var}/lib/pacemaker
 %dir %attr (750, %{uname}, %{gname}) %{_var}/lib/pacemaker/blackbox
 %dir %attr (750, %{uname}, %{gname}) %{_var}/lib/pacemaker/cores
 
 %files -n %{name}-libs
 %defattr(-,root,root)
 
 %{_libdir}/libcib.so.*
 %{_libdir}/liblrmd.so.*
 %{_libdir}/libcrmservice.so.*
 %{_libdir}/libcrmcommon.so.*
 %{_libdir}/libpe_status.so.*
 %{_libdir}/libpe_rules.so.*
 %{_libdir}/libpengine.so.*
 %{_libdir}/libstonithd.so.*
 %{_libdir}/libtransitioner.so.*
 %license licenses/LGPLv2.1
 %doc COPYING
 %doc ChangeLog
 %dir %attr (770, %{uname}, %{gname}) %{_var}/log/pacemaker
 %dir %attr (770, %{uname}, %{gname}) %{_var}/log/pacemaker/bundles
 
 %files -n %{name}-cluster-libs
 %defattr(-,root,root)
 %{_libdir}/libcrmcluster.so.*
 %license licenses/LGPLv2.1
 %doc COPYING
 %doc ChangeLog
 
 %files remote
 %defattr(-,root,root)
 
 %config(noreplace) %{_sysconfdir}/sysconfig/pacemaker
 %if %{defined _unitdir}
 # state directory is shared between the subpackets
 # let rpm take care of removing it once it isn't
 # referenced anymore and empty
 %ghost %dir %{_localstatedir}/lib/rpm-state/%{name}
 %{_unitdir}/pacemaker_remote.service
 %else
 %{_initrddir}/pacemaker_remote
 %endif
 
 %{_sbindir}/pacemaker_remoted
 %{_mandir}/man8/pacemaker_remoted.*
 %license licenses/GPLv2
 %doc COPYING
 %doc ChangeLog
 
 %files doc
 %defattr(-,root,root)
 %doc %{pcmk_docdir}
 %license licenses/CC-BY-SA-4.0
 
 %files cts
 %defattr(-,root,root)
 %{py_site}/cts
 %{_datadir}/pacemaker/tests/cts
 %{_libexecdir}/pacemaker/lrmd_test
 %license licenses/GPLv2
 %doc COPYING
 %doc ChangeLog
 
 %files -n %{name}-libs-devel
 %defattr(-,root,root)
 %exclude %{_datadir}/pacemaker/tests/cts
 %{_datadir}/pacemaker/tests
 %{_includedir}/pacemaker
 %{_libdir}/*.so
 %if %{with coverage}
 %{_var}/lib/pacemaker/gcov
 %endif
 %{_libdir}/pkgconfig/*.pc
 %license licenses/LGPLv2.1
 %doc COPYING
 %doc ChangeLog
 
 %changelog
 
diff --git a/pengine/Makefile.am b/pengine/Makefile.am
index 3f34bf2199..dc9086006d 100644
--- a/pengine/Makefile.am
+++ b/pengine/Makefile.am
@@ -1,76 +1,68 @@
 #
 # Copyright (C) 2004 Andrew Beekhof
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 include $(top_srcdir)/Makefile.common
 
 AM_CPPFLAGS	+= -I$(top_builddir) -I$(top_srcdir)
 
 halibdir	= $(CRM_DAEMON_DIR)
 
-testdir			= $(datadir)/$(PACKAGE)/tests/pengine
-test_SCRIPTS		= regression.sh
-test_DATA		= regression.core.sh
-
-XML_LOG_COMPILER = ./regression.sh
-AM_XML_LOG_FLAGS = -V --run 
-
 COMMONLIBS	= $(top_builddir)/lib/common/libcrmcommon.la	\
 		$(top_builddir)/lib/pengine/libpe_status.la	\
 		libpengine.la
 
-
 ## libraries
 lib_LTLIBRARIES	= libpengine.la
 
 ## binary progs
 halib_PROGRAMS	= pengine
 
 if BUILD_XML_HELP
 man7_MANS =	pengine.7
 endif
 
 ## SOURCES
 
 noinst_HEADERS	= allocate.h notif.h utils.h pengine.h
 
 libpengine_la_LDFLAGS	= -version-info 14:0:4
 
 libpengine_la_CFLAGS	= $(CFLAGS_HARDENED_LIB)
 libpengine_la_LDFLAGS	+= $(LDFLAGS_HARDENED_LIB)
 
 libpengine_la_LIBADD	= $(top_builddir)/lib/pengine/libpe_status.la \
 			$(top_builddir)/lib/cib/libcib.la
 # -L$(top_builddir)/lib/pils -lpils -export-dynamic -module -avoid-version
 libpengine_la_SOURCES	= pengine.c allocate.c notif.c utils.c constraints.c \
 			native.c group.c clone.c pe_promotable.c graph.c \
 			utilization.c container.c
 
 pengine_CFLAGS	= $(CFLAGS_HARDENED_EXE)
 pengine_LDFLAGS	= $(LDFLAGS_HARDENED_EXE)
 pengine_LDADD	= $(top_builddir)/lib/cib/libcib.la $(COMMONLIBS)
 # libcib for get_object_root()
 pengine_SOURCES	= main.c
 
 install-exec-local:
 	$(mkinstalldirs) $(DESTDIR)/$(PE_STATE_DIR)
 	-chown $(CRM_DAEMON_USER) $(DESTDIR)/$(PE_STATE_DIR)
 	-chgrp $(CRM_DAEMON_GROUP) $(DESTDIR)/$(PE_STATE_DIR)
 	-chmod 750 $(DESTDIR)/$(PE_STATE_DIR)
 
 uninstall-local:
 
 CLEANFILES = $(man7_MANS)
diff --git a/pengine/regression.core.sh.in b/pengine/regression.core.sh.in
deleted file mode 100644
index 731844f66a..0000000000
--- a/pengine/regression.core.sh.in
+++ /dev/null
@@ -1,324 +0,0 @@
- # Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
- #
- # This program is free software; you can redistribute it and/or
- # modify it under the terms of the GNU General Public
- # License as published by the Free Software Foundation; either
- # version 2.1 of the License, or (at your option) any later version.
- #
- # This software is distributed in the hope that it will be useful,
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- # General Public License for more details.
- #
- # You should have received a copy of the GNU General Public
- # License along with this library; if not, write to the Free Software
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- #
-
-verbose=0
-num_failed=0
-num_tests=0
-force_local=0
-VALGRIND_CMD=""
-diff_opts="--ignore-all-space --ignore-blank-lines -u -N"
-
-if [ "x$0" = "x-bash" ]; then 
-    test_home=@abs_top_builddir@/pengine
-else
-    test_home=`dirname $0`
-    test_name=`basename $0`
-fi
-
-function info() {
-    printf "$*\n"
-}
-
-function error() {
-    printf "      * ERROR:   $*\n"
-}
-
-function failed() {
-    printf "      * FAILED:  $*\n"
-}
-
-function show_test() {
-    name=$1; shift
-    printf "  Test %-25s $*\n" "$name:"
-}
-
-info "Test home is:\t$test_home"
-io_dir=$test_home/test10
-test_binary=@abs_top_builddir@/tools/crm_simulate
-PCMK_schema_directory=@abs_top_builddir@/xml
-
-
-failed=$test_home/.regression.failed.diff
-single_test=
-
-while true ; do
-    case "$1" in
-	-V|--verbose) verbose=1; shift;;
-	-v|--valgrind)
-	    export G_SLICE=always-malloc
-	    VALGRIND_CMD="valgrind -q --gen-suppressions=all --log-file=%q{valgrind_output} --show-reachable=no --leak-check=full --trace-children=no --time-stamp=yes --num-callers=20 --suppressions=@datadir@/@PACKAGE@/tests/valgrind-pcmk.suppressions"
-	    test_binary=
-	    shift;;
-	--valgrind-dhat)
-	    VALGRIND_CMD="valgrind --log-file=%q{valgrind_output} --show-top-n=100 --num-callers=4 --time-stamp=yes --trace-children=no --tool=exp-dhat"
-	    test_binary=
-	    shift;;
-	--valgrind-skip-output)
-	    VALGRIND_SKIP_OUTPUT=1
-	    shift;;
-	--run) single_test="$2"; shift; shift; break;;
-	-b|--binary) test_binary=$2; PCMK_schema_directory=""; shift; shift;;
-	-i|--io-dir) io_dir=$2; shift; shift;;
-	-?|--help) echo "$0 [--binary name] [--force-local]"; shift; exit 0;;
-	--) shift ; break ;;
-	"") break;;
-	*) echo "unknown option: $1"; exit 1;;
-    esac
-done
-
-if [ "x$test_binary" = "x" ]; then
-    test_binary=@sbindir@/crm_simulate
-    PCMK_schema_directory=@CRM_DTD_DIRECTORY@
-elif [ ! -x $test_binary ]; then
-    test_binary=@sbindir@/crm_simulate
-    PCMK_schema_directory=@CRM_DTD_DIRECTORY@
-fi
-
-export PCMK_schema_directory
-
-if [ "x$test_binary" = "x" ]; then
-    info "crm_simulate not installed. Aborting."
-    exit 1
-fi
-
-info "Test binary is:\t$test_binary"
-info "Schema home is:\t$PCMK_schema_directory"
-if [ "x$VALGRIND_CMD" != "x" ]; then
-    info "Activating memory testing with valgrind";
-fi
-
-info " "
-
-test_cmd="$VALGRIND_CMD $test_binary"
-#echo $test_cmd
-
-if [ `whoami` != root ]; then
-    declare -x CIB_shadow_dir=/tmp
-fi
-
-function do_test {
-
-    did_fail=0
-    expected_rc=0
-    num_tests=`expr $num_tests + 1`
-
-    base=$1; shift
-    name=$1; shift
-
-    input=$io_dir/${base}.xml
-    output=$io_dir/${base}.out
-    expected=$io_dir/${base}.exp
-
-    dot_png=$io_dir/${base}.png
-    dot_expected=$io_dir/${base}.dot
-    dot_output=$io_dir/${base}.pe.dot
-
-    scores=$io_dir/${base}.scores
-    score_output=$io_dir/${base}.scores.pe
-
-    stderr_expected=$io_dir/${base}.stderr
-    stderr_output=$io_dir/${base}.stderr.pe
-
-    summary=$io_dir/${base}.summary
-    summary_output=$io_dir/${base}.summary.pe
-
-    valgrind_output=$io_dir/${base}.valgrind
-    export valgrind_output
-
-    if [ "x$1" = "x--rc" ]; then
-	expected_rc=$2
-	shift; shift;
-    fi
-
-    show_test "$base" "$name"
-
-    if [ ! -f $input ]; then
-	error "No input";
-	did_fail=1
-	num_failed=`expr $num_failed + $did_fail`
-	return;
-    fi
-
-    if [ "$create_mode" != "true" -a ! -f $expected ]; then
-	error "no stored output";
-#	return;
-    fi
-
-#    ../admin/crm_verify -X $input
-    if [ ! -z $single_test ]; then
-	echo CIB_shadow_dir=$io_dir $test_cmd -x $input -D $dot_output -G $output -S $*
-	CIB_shadow_dir=$io_dir $test_cmd -x $input -D $dot_output -G $output -S $* 2>&1 | tee $summary_output
-    else
-	CIB_shadow_dir=$io_dir $test_cmd -x $input -S &> $summary_output
-    fi
-
-    CIB_shadow_dir=$io_dir $test_cmd -x $input -D $dot_output -G $output -SQ -s $* 2> $stderr_output > $score_output
-    rc=$?
-
-    if [ $rc != $expected_rc ]; then
-	failed "Test returned: $rc";
-	did_fail=1
-	echo "CIB_shadow_dir=$io_dir $test_cmd -x $input -D $dot_output -G $output -SQ -s $*"
-    fi
-
-    if [ -z "$VALGRIND_SKIP_OUTPUT" ]; then
-	if [ -s ${valgrind_output} ]; then
-	    error "Valgrind reported errors";
-	    did_fail=1
-	    cat ${valgrind_output}
-	fi
-	rm -f ${valgrind_output}
-    fi
-
-    if [ -s core ]; then
-	error "Core-file detected: core.${base}";
-	did_fail=1
-	rm -f $test_home/core.$base
-	mv core $test_home/core.$base
-    fi
-
-    if [ -e $stderr_expected ]; then
-
-	diff $diff_opts $stderr_expected $stderr_output >/dev/null
-	rc2=$?
-	if [ $rc2 != 0 ]; then
-	    failed "stderr changed";
-	    diff $diff_opts $stderr_expected $stderr_output 2>/dev/null >> $failed
-	    echo "" >> $failed
-	    did_fail=1
-	fi
-
-    elif [ -s $stderr_output ]; then
-	error "Output was written to stderr"
-	did_fail=1
-	cat $stderr_output
-    fi
-    rm -f $stderr_output
-
-    if [ ! -s $output ]; then
-	error "No graph produced";
-	did_fail=1
-	num_failed=`expr $num_failed + $did_fail`
-	rm -f $output
-	return;
-#    else
-#	mv $output $output.sed
-#	cat $output.sed | sed 's/id=.[0-9]*.\ //g' >> $output
-    fi
-
-    if [ ! -s $dot_output ]; then
-	error "No dot-file summary produced";
-	did_fail=1
-	num_failed=`expr $num_failed + $did_fail`
-	rm -f $output
-	return;
-    else
-	echo "digraph \"g\" {" > $dot_output.sort
-	LC_ALL=POSIX sort -u $dot_output | grep -v -e ^}$ -e digraph >> $dot_output.sort
-	echo "}" >> $dot_output.sort
-	mv -f $dot_output.sort $dot_output
-    fi
-
-    if [ ! -s $score_output ]; then
-	error "No allocation scores produced";
-	did_fail=1
-	num_failed=`expr $num_failed + $did_fail`
-	rm $output
-	return;
-    else
-	LC_ALL=POSIX sort $score_output > $score_output.sorted
-	mv -f $score_output.sorted $score_output
-    fi
-
-    if [ "$create_mode" = "true" ]; then
-	cp "$output" "$expected"
-	cp "$dot_output" "$dot_expected"
-	cp "$score_output" "$scores"
-	cp "$summary_output" "$summary"
-	info "	Created expected outputs"
-    fi
-
-    diff $diff_opts $summary $summary_output >/dev/null
-    rc2=$?
-    if [ $rc2 != 0 ]; then
-	failed "summary changed";
-	diff $diff_opts $summary $summary_output 2>/dev/null >> $failed
-	echo "" >> $failed
-	did_fail=1
-    fi
-
-    diff $diff_opts $dot_expected $dot_output >/dev/null
-    rc=$?
-    if [ $rc != 0 ]; then
-	failed "dot-file summary changed";
-	diff $diff_opts $dot_expected $dot_output 2>/dev/null >> $failed
-	echo "" >> $failed
-	did_fail=1
-    else
-	rm -f $dot_output
-    fi
-
-    sed -i -e 's/crm_feature_set="[^"]*"//' -e 's/batch-limit="[0-9]*"//' $expected $output
-    diff $diff_opts $expected $output >/dev/null
-    rc2=$?
-    if [ $rc2 != 0 ]; then
-	failed "xml-file changed";
-	diff $diff_opts $expected $output 2>/dev/null >> $failed
-	echo "" >> $failed
-	did_fail=1
-    fi
-
-    diff $diff_opts $scores $score_output >/dev/null
-    rc=$?
-    if [ $rc != 0 ]; then
-	failed "scores-file changed";
-	diff $diff_opts $scores $score_output 2>/dev/null >> $failed
-	echo "" >> $failed
-	did_fail=1
-    fi
-    rm -f $output $score_output $summary_output
-    num_failed=`expr $num_failed + $did_fail`
-}
-
-function test_results {
-    if [ $num_failed != 0 ]; then
-	if [ -s $failed ]; then
-	    if [ "$verbose" = "1" ]; then
-		error "Results of $num_failed failed tests (out of $num_tests)...."
-		cat $failed
-	    else
-		error "Results of $num_failed failed tests (out of $num_tests) are in $failed...."
-		error "Use $0 -V to display them automatically."
-	    fi
-	else
-	    error "$num_failed (of $num_tests) tests failed (no diff results)"
-	    rm $failed
-	fi
-    fi
-    exit $num_failed
-}
-
-if [ -z $single_test ]; then
-# zero out the error log
-    > $failed
-else
-    failed=.single
-    > $failed
-    do_test $single_test "Single shot" $*
-    cat $failed
-    exit $?
-fi