No OneTemporary
Actions

Size

561 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

This document is not UTF8. It was detected as Shift JIS and converted to UTF8 for display.

	diff --git a/.gitignore b/.gitignore
	index 36bebf0560..536013518b 100644
	--- a/.gitignore
	+++ b/.gitignore
	@@ -1,207 +1,212 @@
	# Common
	\#*
	.\#*
	GPATH
	GRTAGS
	GTAGS
	TAGS
	Makefile
	Makefile.in
	.deps
	.dirstamp
	.libs
	*.pc
	*.pyc
	*.bz2
	*.tar.gz
	*.rpm
	*.la
	*.lo
	*.o
	*~
	*.gcda
	*.gcno


	# Autobuild
	aclocal.m4
	autoconf
	autoheader
	autom4te.cache/
	automake
	build.counter
	compile
	config.guess
	config.log
	config.status
	config.sub
	configure
	depcomp
	install-sh
	include/stamp-*
	libtool
	libtool.m4
	ltdl.m4
	libltdl
	ltmain.sh
	missing
	py-compile
	/m4/argz.m4
	/m4/ltargz.m4
	/m4/ltoptions.m4
	/m4/ltsugar.m4
	/m4/ltversion.m4
	/m4/lt~obsolete.m4
	test-driver
	ylwrap

	# Configure targets
	Doxyfile
	/cts/CTS.py
	/cts/CTSlab.py
	/cts/CTSvars.py
	/cts/LSBDummy
	/cts/OCFIPraTest.py
	/cts/benchmark/clubench
	/cts/cluster_test
	/cts/cts
	/cts/cts-cli
	/cts/cts-coverage
	/cts/cts-exec
	/cts/cts-fencing
	/cts/cts-log-watcher
	/cts/cts-regression
	/cts/cts-scheduler
	/cts/cts-support
	/cts/fence_dummy
	/cts/lxc_autogen.sh
	/cts/pacemaker-cts-dummyd
	/cts/pacemaker-cts-dummyd@.service
	/daemons/execd/pacemaker_remote
	/daemons/execd/pacemaker_remote.service
	/daemons/fenced/fence_legacy
	/daemons/pacemakerd/pacemaker
	/daemons/pacemakerd/pacemaker.combined.upstart
	/daemons/pacemakerd/pacemaker.service
	/daemons/pacemakerd/pacemaker.upstart
	-extra/logrotate/pacemaker
	+/extra/logrotate/pacemaker
	+/extra/resources/ClusterMon
	+/extra/resources/HealthSMART
	+/extra/resources/SysInfo
	+/extra/resources/ifspeed
	+/extra/resources/o2cb
	include/config.h
	include/config.h.in
	include/crm_config.h
	publican.cfg
	/tools/cibsecret
	/tools/crm_error
	/tools/crm_failcount
	/tools/crm_master
	/tools/crm_mon.service
	/tools/crm_mon.upstart
	/tools/crm_report
	/tools/crm_standby
	/tools/report.collector
	/tools/report.common

	# Build targets
	*.7
	*.7.xml
	*.7.html
	*.8
	*.8.xml
	*.8.html
	doc//en-US/images/.png
	doc//tmp/*
	doc/*/publish
	/daemons/attrd/pacemaker-attrd
	/daemons/based/pacemaker-based
	/daemons/based/cibmon
	/daemons/controld/pacemaker-controld
	/daemons/execd/cts-exec-helper
	/daemons/execd/pacemaker-execd
	/daemons/execd/pacemaker-remoted
	/daemons/fenced/cts-fence-helper
	/daemons/fenced/pacemaker-fenced
	/daemons/fenced/pacemaker-fenced.xml
	/daemons/pacemakerd/pacemakerd
	/daemons/schedulerd/pacemaker-schedulerd
	/daemons/schedulerd/pacemaker-schedulerd.xml
	/daemons/schedulerd/ptest
	doc/api/*
	doc/Clusters_from_Scratch.txt
	doc/Pacemaker_Explained.txt
	doc/acls.html
	doc/crm_fencing.html
	doc/publican-catalog*
	scratch
	/tools/attrd_updater
	/tools/cibadmin
	/tools/crmadmin
	/tools/crm_attribute
	/tools/crm_diff
	/tools/crm_mon
	/tools/crm_node
	/tools/crm_resource
	/tools/crm_shadow
	/tools/crm_simulate
	/tools/crm_ticket
	/tools/crm_verify
	/tools/iso8601
	/tools/stonith_admin
	xml/crm.dtd
	xml/pacemaker*.rng
	xml/versions.rng
	doc/shared/en-US/*.xml
	doc/Clusters_from_Scratch.build
	doc/Clusters_from_Scratch/en-US/Ap-*.xml
	doc/Clusters_from_Scratch/en-US/Ch-*.xml
	doc/Pacemaker_Administration.build
	doc/Pacemaker_Administration/en-US/Ch-*.xml
	doc/Pacemaker_Development.build
	doc/Pacemaker_Development/en-US/Ch-*.xml
	doc/Pacemaker_Explained.build
	doc/Pacemaker_Explained/en-US/Ch-*.xml
	doc/Pacemaker_Explained/en-US/Ap-*.xml
	doc/Pacemaker_Remote.build
	doc/Pacemaker_Remote/en-US/Ch-*.xml
	lib/gnu/libgnu.a
	lib/gnu/stdalign.h
	*.coverity

	# Test detritus
	/cts/.regression.failed.diff
	/cts/scheduler/*.ref
	/cts/scheduler/*.up
	/cts/scheduler/*.up.err
	/cts/scheduler/bug-rh-1097457.log
	/cts/scheduler/bug-rh-1097457.trs
	/cts/scheduler/shadow.*
	/cts/test-suite.log
	/xml/test-2/*.up
	/xml/test-2/*.up.err

	# Formerly built files (helps when jumping back and forth in checkout)
	/attrd
	/cib
	/coverage.sh
	/crmd
	/cts/HBDummy
	/fencing
	/lrmd
	/mcp
	/pengine

	#Other
	mock
	HTML
	pacemaker*.spec
	coverity-*

	compat_reports
	.ABI-build
	abi_dumps
	logs

	*.patch
	*.diff
	*.sed
	*.orig
	*.rej
	*.swp
	diff --git a/configure.ac b/configure.ac
	index cea05461d5..f21c0cfaa6 100644
	--- a/configure.ac
	+++ b/configure.ac
	@@ -1,1805 +1,1810 @@
	dnl
	dnl autoconf for Pacemaker
	dnl
	dnl Copyright 2009-2018 Andrew Beekhof <andrew@beekhof.net>
	dnl
	dnl This source code is licensed under the GNU General Public License version 2
	dnl or later (GPLv2+) WITHOUT ANY WARRANTY.

	dnl ===============================================
	dnl Bootstrap
	dnl ===============================================
	AC_PREREQ(2.64)

	AC_CONFIG_MACRO_DIR([m4])
	AC_DEFUN([AC_DATAROOTDIR_CHECKED])

	dnl Suggested structure:
	dnl information on the package
	dnl checks for programs
	dnl checks for libraries
	dnl checks for header files
	dnl checks for types
	dnl checks for structures
	dnl checks for compiler characteristics
	dnl checks for library functions
	dnl checks for system services

	m4_include([version.m4])
	AC_INIT([pacemaker], VERSION_NUMBER, [users@clusterlabs.org], [pacemaker],
	PCMK_URL)

	PCMK_FEATURES=""

	AC_CONFIG_AUX_DIR(.)
	AC_CANONICAL_HOST

	dnl Where #defines go (e.g. `AC_CHECK_HEADERS' below)
	dnl
	dnl Internal header: include/config.h
	dnl - Contains ALL defines
	dnl - include/config.h.in is generated automatically by autoheader
	dnl - NOT to be included in any header files except crm_internal.h
	dnl (which is also not to be included in any other header files)
	dnl
	dnl External header: include/crm_config.h
	dnl - Contains a subset of defines checked here
	dnl - Manually edit include/crm_config.h.in to have configure include
	dnl new defines
	dnl - Should not include HAVE_* defines
	dnl - Safe to include anywhere
	AM_CONFIG_HEADER(include/config.h include/crm_config.h)

	AC_ARG_WITH(version,
	[ --with-version=version Override package version (if you are a packager needing to pretend) ],
	[ PACKAGE_VERSION="$withval" ])

	AC_ARG_WITH(pkg-name,
	[ --with-pkg-name=name Override package name (if you are a packager needing to pretend) ],
	[ PACKAGE_NAME="$withval" ])

	dnl 1.11: minimum automake version required
	dnl foreign: don't require GNU-standard top-level files
	dnl silent-rules: allow "--enable-silent-rules" (no-op in 1.13+)
	dnl subdir-objects: keep .o's with their .c's (no-op in 2.0+)
	AM_INIT_AUTOMAKE([1.11 foreign silent-rules subdir-objects])

	dnl Example 2.4. Silent Custom Rule to Generate a File
	dnl %-bar.pc: %.pc
	dnl $(AM_V_GEN)$(LN_S) $(notdir $^) $@

	AC_DEFINE_UNQUOTED(PACEMAKER_VERSION, "$PACKAGE_VERSION",
	[Current pacemaker version])

	dnl Versioned attributes implementation is not yet production-ready
	AC_DEFINE_UNQUOTED(ENABLE_VERSIONED_ATTRS, 0, [Enable versioned attributes])

	PACKAGE_SERIES=`echo $PACKAGE_VERSION \| awk -F. '{ print $1"."$2 }'`
	AC_SUBST(PACKAGE_SERIES)
	AC_SUBST(PACKAGE_VERSION)

	CC_IN_CONFIGURE=yes
	export CC_IN_CONFIGURE

	LDD=ldd

	dnl ========================================================================
	dnl Compiler characteristics
	dnl ========================================================================

	AC_PROG_CC dnl Can force other with environment variable "CC".
	AC_PROG_CC_STDC
	gl_EARLY
	gl_INIT

	LT_INIT([dlopen])
	LTDL_INIT([convenience])

	AC_TYPE_SIZE_T
	AC_CHECK_SIZEOF(char)
	AC_CHECK_SIZEOF(short)
	AC_CHECK_SIZEOF(int)
	AC_CHECK_SIZEOF(long)
	AC_CHECK_SIZEOF(long long)

	dnl ===============================================
	dnl Helpers
	dnl ===============================================
	cc_supports_flag() {
	local CFLAGS="-Werror $@"
	AC_MSG_CHECKING(whether $CC supports "$@")
	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ ]])],
	[RC=0; AC_MSG_RESULT(yes)],
	[RC=1; AC_MSG_RESULT(no)])
	return $RC
	}

	# Some tests need to use their own CFLAGS

	cc_temp_flags() {
	ac_save_CFLAGS="$CFLAGS"
	CFLAGS="$*"
	}

	cc_restore_flags() {
	CFLAGS=$ac_save_CFLAGS
	}

	dnl ===============================================
	dnl Configure Options
	dnl ===============================================

	dnl Some systems, like Solaris require a custom package name
	AC_ARG_WITH(pkgname,
	[ --with-pkgname=name name for pkg (typically for Solaris) ],
	[ PKGNAME="$withval" ],
	[ PKGNAME="LXHAhb" ],
	)
	AC_SUBST(PKGNAME)

	AC_ARG_ENABLE([ansi],
	[ --enable-ansi Force GCC to compile to ANSI standard for older compilers. @<:@no@:>@])

	AC_ARG_ENABLE([fatal-warnings],
	[ --enable-fatal-warnings Enable pedantic and fatal warnings for gcc @<:@yes@:>@])

	AC_ARG_ENABLE([quiet],
	[ --enable-quiet Suppress make output unless there is an error @<:@no@:>@])

	AC_ARG_ENABLE([no-stack],
	[ --enable-no-stack Build only the scheduler and its requirements @<:@no@:>@])

	AC_ARG_ENABLE([upstart],
	[ --enable-upstart Enable support for managing resources via Upstart @<:@try@:>@ ],
	[],
	[enable_upstart=try],
	)

	AC_ARG_ENABLE([systemd],
	[ --enable-systemd Enable support for managing resources via systemd @<:@try@:>@],
	[],
	[enable_systemd=try],
	)

	AC_ARG_ENABLE(hardening,
	[ --enable-hardening Harden the resulting executables/libraries @<:@try@:>@],
	[ HARDENING="${enableval}" ],
	[ HARDENING=try ],
	)

	# By default, we add symlinks at the pre-2.0.0 daemon name locations, so that:
	# (1) tools that directly invoke those names for metadata etc. will still work
	# (2) this installation can be used in a bundle container image used with
	# cluster hosts running Pacemaker 1.1.17+
	# If you know your target systems will not have any need for it, you can
	# disable this option. Once the above use cases are no longer in wide use, we
	# can disable this option by default, and once we no longer want to support
	# them at all, we can drop the option altogether.
	AC_ARG_ENABLE(legacy-links,
	[ --enable-legacy-links Add symlinks for old daemon names @<:@yes@:>@],
	[ LEGACY_LINKS="${enableval}" ],
	[ LEGACY_LINKS=yes ],
	)
	AM_CONDITIONAL(BUILD_LEGACY_LINKS, test "x${LEGACY_LINKS}" = "xyes")

	AC_ARG_WITH(corosync,
	[ --with-corosync Support the Corosync messaging and membership layer ],
	[ SUPPORT_CS=$withval ],
	[ SUPPORT_CS=try ],
	)

	AC_ARG_WITH(nagios,
	[ --with-nagios Support nagios remote monitoring ],
	[ SUPPORT_NAGIOS=$withval ],
	[ SUPPORT_NAGIOS=try ],
	)

	AC_ARG_WITH(nagios-plugin-dir,
	[ --with-nagios-plugin-dir=DIR Directory for nagios plugins @<:@LIBEXECDIR/nagios/plugins@:>@],
	[ NAGIOS_PLUGIN_DIR="$withval" ]
	)

	AC_ARG_WITH(nagios-metadata-dir,
	[ --with-nagios-metadata-dir=DIR Directory for nagios plugins metadata @<:@DATADIR/nagios/plugins-metadata@:>@],
	[ NAGIOS_METADATA_DIR="$withval" ]
	)

	AC_ARG_WITH(acl,
	[ --with-acl Support CIB ACL ],
	[ SUPPORT_ACL=$withval ],
	[ SUPPORT_ACL=yes ],
	)

	AC_ARG_WITH(cibsecrets,
	[ --with-cibsecrets Support separate file for CIB secrets ],
	[ SUPPORT_CIBSECRETS=$withval ],
	[ SUPPORT_CIBSECRETS=no ],
	)

	PCMK_GNUTLS_PRIORITIES="NORMAL"
	AC_ARG_WITH(gnutls-priorities,
	[ --with-gnutls-priorities GnuTLS cipher priorities @<:@NORMAL@:>@ ],
	[ test x"$withval" = x"no" \|\| PCMK_GNUTLS_PRIORITIES="$withval" ])

	INITDIR=""
	AC_ARG_WITH(initdir,
	[ --with-initdir=DIR Directory for init (rc) scripts],
	[ INITDIR="$withval" ])

	SUPPORT_PROFILING=0
	AC_ARG_WITH(profiling,
	[ --with-profiling Disable optimizations for effective profiling ],
	[ SUPPORT_PROFILING=$withval ])

	AC_ARG_WITH(coverage,
	[ --with-coverage Disable optimizations for effective profiling ],
	[ SUPPORT_COVERAGE=$withval ])

	PUBLICAN_BRAND="common"
	AC_ARG_WITH(brand,
	[ --with-brand=brand Brand to use for generated documentation (set empty for no docs) @<:@common@:>@],
	[ test x"$withval" = x"no" \|\| PUBLICAN_BRAND="$withval" ])
	AC_SUBST(PUBLICAN_BRAND)

	CONFIGDIR=""
	AC_ARG_WITH(configdir,
	[ --with-configdir=DIR Directory for Pacemaker configuration file @<:@SYSCONFDIR/sysconfig@:>@],
	[ CONFIGDIR="$withval" ]
	)

	CRM_LOG_DIR=""
	AC_ARG_WITH(logdir,
	[ --with-logdir=DIR Directory for Pacemaker log file @<:@LOCALSTATEDIR/log/pacemaker@:>@ ],
	[ CRM_LOG_DIR="$withval" ]
	)

	CRM_BUNDLE_DIR=""
	AC_ARG_WITH(bundledir,
	[ --with-bundledir=DIR Directory for Pacemaker bundle logs @<:@LOCALSTATEDIR/log/pacemaker/bundles@:>@ ],
	[ CRM_BUNDLE_DIR="$withval" ]
	)

	dnl ===============================================
	dnl General Processing
	dnl ===============================================

	AC_PROG_LN_S
	AC_PROG_MKDIR_P

	if cc_supports_flag -Werror; then
	WERROR="-Werror"
	else
	WERROR=""
	fi

	# Normalize enable_fatal_warnings (defaulting to yes, when compiler supports it)
	if test "x${enable_fatal_warnings}" != "xno" ; then
	if test "$GCC" = "yes" && test "x${WERROR}" != "x" ; then
	enable_fatal_warnings=yes
	else
	AC_MSG_NOTICE(Compiler does not support fatal warnings)
	enable_fatal_warnings=no
	fi
	fi

	INIT_EXT=""
	echo Our Host OS: $host_os/$host

	AC_MSG_NOTICE(Sanitizing prefix: ${prefix})
	case $prefix in
	NONE)
	prefix=/usr
	dnl Fix default variables - "prefix" variable if not specified
	if test "$localstatedir" = "\${prefix}/var"; then
	localstatedir="/var"
	fi
	if test "$sysconfdir" = "\${prefix}/etc"; then
	sysconfdir="/etc"
	fi
	;;
	esac

	AC_MSG_NOTICE(Sanitizing exec_prefix: ${exec_prefix})
	case $exec_prefix in
	prefix\|NONE)
	exec_prefix=$prefix
	;;
	esac

	AC_MSG_NOTICE(Sanitizing INITDIR: ${INITDIR})
	case $INITDIR in
	prefix) INITDIR=$prefix;;
	"")
	AC_MSG_CHECKING(which init (rc) directory to use)
	for initdir in /etc/init.d /etc/rc.d/init.d /sbin/init.d \
	/usr/local/etc/rc.d /etc/rc.d
	do
	if
	test -d $initdir
	then
	INITDIR=$initdir
	break
	fi
	done
	AC_MSG_RESULT($INITDIR)
	;;
	esac
	AC_SUBST(INITDIR)

	AC_MSG_NOTICE(Sanitizing libdir: ${libdir})
	case $libdir in
	prefix\|NONE)
	AC_MSG_CHECKING(which lib directory to use)
	for aDir in lib64 lib
	do
	trydir="${exec_prefix}/${aDir}"
	if
	test -d ${trydir}
	then
	libdir=${trydir}
	break
	fi
	done
	AC_MSG_RESULT($libdir);
	;;
	esac

	dnl Expand autoconf variables so that we don't end up with '${prefix}'
	dnl in #defines and python scripts
	dnl NOTE: Autoconf deliberately leaves them unexpanded to allow
	dnl make exec_prefix=/foo install
	dnl No longer being able to do this seems like no great loss to me...

	eval prefix="`eval echo ${prefix}`"
	eval exec_prefix="`eval echo ${exec_prefix}`"
	eval bindir="`eval echo ${bindir}`"
	eval sbindir="`eval echo ${sbindir}`"
	eval libexecdir="`eval echo ${libexecdir}`"
	eval datadir="`eval echo ${datadir}`"
	eval sysconfdir="`eval echo ${sysconfdir}`"
	eval sharedstatedir="`eval echo ${sharedstatedir}`"
	eval localstatedir="`eval echo ${localstatedir}`"
	eval libdir="`eval echo ${libdir}`"
	eval includedir="`eval echo ${includedir}`"
	eval oldincludedir="`eval echo ${oldincludedir}`"
	eval infodir="`eval echo ${infodir}`"
	eval mandir="`eval echo ${mandir}`"

	dnl Home-grown variables
	eval INITDIR="${INITDIR}"
	eval docdir="`eval echo ${docdir}`"
	if test x"${docdir}" = x""; then
	docdir=${datadir}/doc/${PACKAGE}-${VERSION}
	fi
	AC_SUBST(docdir)
	if test x"${CONFIGDIR}" = x""; then
	CONFIGDIR="${sysconfdir}/sysconfig"
	fi
	AC_SUBST(CONFIGDIR)

	if test x"${CRM_LOG_DIR}" = x""; then
	CRM_LOG_DIR="${localstatedir}/log/pacemaker"
	fi
	AC_DEFINE_UNQUOTED(CRM_LOG_DIR,"$CRM_LOG_DIR", Location for Pacemaker log file)
	AC_SUBST(CRM_LOG_DIR)

	if test x"${CRM_BUNDLE_DIR}" = x""; then
	CRM_BUNDLE_DIR="${localstatedir}/log/pacemaker/bundles"
	fi
	AC_DEFINE_UNQUOTED(CRM_BUNDLE_DIR,"$CRM_BUNDLE_DIR", Location for Pacemaker bundle logs)
	AC_SUBST(CRM_BUNDLE_DIR)


	if test x"${PCMK_GNUTLS_PRIORITIES}" = x""; then
	AC_MSG_ERROR([Empty string not applicable with --with-gnutls-priorities])
	fi
	AC_DEFINE_UNQUOTED([PCMK_GNUTLS_PRIORITIES], ["$PCMK_GNUTLS_PRIORITIES"],
	[GnuTLS cipher priorities])

	for j in prefix exec_prefix bindir sbindir libexecdir datadir sysconfdir \
	sharedstatedir localstatedir libdir includedir oldincludedir infodir \
	mandir INITDIR docdir CONFIGDIR
	do
	dirname=`eval echo '${'${j}'}'`
	if
	test ! -d "$dirname"
	then
	AC_MSG_WARN([$j directory ($dirname) does not exist!])
	fi
	done

	dnl This OS-based decision-making is poor autotools practice;
	dnl feature-based mechanisms are strongly preferred.
	dnl
	dnl So keep this section to a bare minimum; regard as a "necessary evil".

	case "$host_os" in
	bsd)
	AC_DEFINE_UNQUOTED(ON_BSD, 1, Compiling for BSD platform)
	LIBS="-L/usr/local/lib"
	CPPFLAGS="$CPPFLAGS -I/usr/local/include"
	INIT_EXT=".sh"
	;;
	solaris)
	AC_DEFINE_UNQUOTED(ON_SOLARIS, 1, Compiling for Solaris platform)
	;;
	linux)
	AC_DEFINE_UNQUOTED(ON_LINUX, 1, Compiling for Linux platform)
	;;
	darwin*)
	AC_DEFINE_UNQUOTED(ON_DARWIN, 1, Compiling for Darwin platform)
	LIBS="$LIBS -L${prefix}/lib"
	CFLAGS="$CFLAGS -I${prefix}/include"
	;;
	esac

	AC_SUBST(INIT_EXT)
	AC_MSG_NOTICE(Host CPU: $host_cpu)

	case "$host_cpu" in
	ppc64\|powerpc64)
	case $CFLAGS in
	powerpc64)
	;;
	*)
	if test "$GCC" = yes; then
	CFLAGS="$CFLAGS -m64"
	fi
	;;
	esac
	;;
	esac

	AC_MSG_CHECKING(which format is needed to print uint64_t)

	cc_temp_flags "-Wall $WERROR"

	AC_COMPILE_IFELSE(
	[AC_LANG_PROGRAM(
	[
	#include <stdio.h>
	#include <stdint.h>
	#include <stdlib.h>
	],
	[
	int max = 512;
	uint64_t bignum = 42;
	char *buffer = malloc(max);
	const char *random = "random";
	snprintf(buffer, max-1, "<quorum id=%lu quorate=%s/>", bignum, random);
	fprintf(stderr, "Result: %s\n", buffer);
	]
	)],
	[U64T="%lu"],
	[U64T="%llu"]
	)
	cc_restore_flags

	AC_MSG_RESULT($U64T)
	AC_DEFINE_UNQUOTED(U64T, "$U64T", Correct printf format for logging uint64_t)

	dnl ===============================================
	dnl Program Paths
	dnl ===============================================

	PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin"
	export PATH

	dnl Replacing AC_PROG_LIBTOOL with AC_CHECK_PROG because LIBTOOL
	dnl was NOT being expanded all the time thus causing things to fail.
	AC_CHECK_PROGS(LIBTOOL, glibtool libtool libtool15 libtool13)

	dnl Pacemaker's executable python scripts will invoke the python specified by
	dnl configure's PYTHON variable. If not specified, AM_PATH_PYTHON will check a
	dnl built-in list with (unversioned) "python" having precedence. To configure
	dnl Pacemaker to use a specific python interpreter version, define PYTHON
	dnl when calling configure, for example: ./configure PYTHON=/usr/bin/python3.6

	dnl Ensure PYTHON is an absolute path
	AC_PATH_PROG([PYTHON], [$PYTHON])

	case "x$PYTHON" in
	xpython3)
	dnl When used with Python 3, Pacemaker requires a minimum of 3.2
	AM_PATH_PYTHON([3.2])
	;;
	*)
	dnl Otherwise, Pacemaker requires a minimum of 2.7
	AM_PATH_PYTHON([2.7])
	;;
	esac

	AC_CHECK_PROGS(MAKE, gmake make)
	AC_PATH_PROGS(HTML2TXT, lynx w3m)
	AC_PATH_PROGS(HELP2MAN, help2man)
	AC_PATH_PROGS(POD2MAN, pod2man, pod2man)
	AC_PATH_PROGS(ASCIIDOC, asciidoc)
	AC_PATH_PROGS(PUBLICAN, publican)
	AC_PATH_PROGS(INKSCAPE, inkscape)
	AC_PATH_PROGS(XSLTPROC, xsltproc)
	AC_PATH_PROGS(XMLCATALOG, xmlcatalog)
	AC_PATH_PROGS(FOP, fop)
	AC_PATH_PROGS(SSH, ssh, /usr/bin/ssh)
	AC_PATH_PROGS(SCP, scp, /usr/bin/scp)
	AC_PATH_PROGS(TAR, tar)
	AC_PATH_PROGS(MD5, md5)
	dnl BASH is already an environment variable, so use something else
	AC_PATH_PROG([BASH_PATH], [bash])
	AC_PATH_PROGS(TEST, test)
	PKG_PROG_PKG_CONFIG
	AC_PATH_PROGS(VALGRIND_BIN, valgrind, /usr/bin/valgrind)
	AC_DEFINE_UNQUOTED(VALGRIND_BIN, "$VALGRIND_BIN", Valgrind command)

	if test x"${LIBTOOL}" = x""; then
	AC_MSG_ERROR(You need (g)libtool installed in order to build ${PACKAGE})
	fi
	if test x"${MAKE}" = x""; then
	AC_MSG_ERROR(You need (g)make installed in order to build ${PACKAGE})
	fi

	dnl Bash is needed for building man pages and running regression tests
	if test x"${BASH_PATH}" = x""; then
	AC_MSG_ERROR(bash must be installed in order to build ${PACKAGE})
	fi

	AM_CONDITIONAL(BUILD_HELP, test x"${HELP2MAN}" != x"")
	if test x"${HELP2MAN}" != x""; then
	PCMK_FEATURES="$PCMK_FEATURES generated-manpages"
	fi

	MANPAGE_XSLT=""
	if test x"${XSLTPROC}" != x""; then
	AC_MSG_CHECKING(docbook to manpage transform)
	# first try to figure out correct template using xmlcatalog query,
	# resort to extensive (semi-deterministic) file search if that fails
	DOCBOOK_XSL_URI='http://docbook.sourceforge.net/release/xsl/current'
	DOCBOOK_XSL_PATH='manpages/docbook.xsl'
	MANPAGE_XSLT=$(${XMLCATALOG} "" ${DOCBOOK_XSL_URI}/${DOCBOOK_XSL_PATH} \
	\| sed -n 's\|^file://\|\|p;q')
	if test x"${MANPAGE_XSLT}" = x""; then
	DIRS=$(find "${datadir}" -name $(basename $(dirname ${DOCBOOK_XSL_PATH})) \
	-type d \| LC_ALL=C sort)
	XSLT=$(basename ${DOCBOOK_XSL_PATH})
	for d in ${DIRS}; do
	if test -f "${d}/${XSLT}"; then
	MANPAGE_XSLT="${d}/${XSLT}"
	break
	fi
	done
	fi
	fi
	AC_MSG_RESULT($MANPAGE_XSLT)
	AC_SUBST(MANPAGE_XSLT)

	AM_CONDITIONAL(BUILD_XML_HELP, test x"${MANPAGE_XSLT}" != x"")
	if test x"${MANPAGE_XSLT}" != x""; then
	PCMK_FEATURES="$PCMK_FEATURES agent-manpages"
	fi

	AM_CONDITIONAL(BUILD_ASCIIDOC, test x"${ASCIIDOC}" != x"")
	if test x"${ASCIIDOC}" != x""; then
	PCMK_FEATURES="$PCMK_FEATURES ascii-docs"
	fi

	publican_intree_brand=no
	if test x"${PUBLICAN_BRAND}" != x"" \
	&& test x"${PUBLICAN}" != x"" \
	&& test x"${INKSCAPE}" != x""; then

	dnl special handling for clusterlabs brand (possibly in-tree version used)
	test "${PUBLICAN_BRAND}" != "clusterlabs" \
	\|\| test -d /usr/share/publican/Common_Content/clusterlabs
	if test $? -ne 0; then
	dnl Unknown option: brand_dir vs. Option brand_dir requires an argument
	if ${PUBLICAN} build --brand_dir 2>&1 \| grep -Eq 'brand_dir$'; then
	AC_MSG_WARN([Cannot use in-tree clusterlabs brand, resorting to common])
	PUBLICAN_BRAND=common
	else
	publican_intree_brand=yes
	fi
	fi
	AC_MSG_NOTICE([Enabling Publican-generated documentation using ${PUBLICAN_BRAND} brand])
	PCMK_FEATURES="$PCMK_FEATURES publican-docs"
	fi
	AM_CONDITIONAL([BUILD_DOCBOOK],
	[test x"${PUBLICAN_BRAND}" != x"" \
	&& test x"${PUBLICAN}" != x"" \
	&& test x"${INKSCAPE}" != x""])
	AM_CONDITIONAL([PUBLICAN_INTREE_BRAND],
	[test x"${publican_intree_brand}" = x"yes"])

	dnl Pacemaker's shell scripts (and thus man page builders) rely on GNU getopt
	AC_MSG_CHECKING([for GNU-compatible getopt])
	IFS_orig=$IFS
	IFS=:
	for PATH_DIR in $PATH; do
	IFS=$IFS_orig
	GETOPT_PATH="${PATH_DIR}/getopt"
	if test -f "$GETOPT_PATH" && test -x "$GETOPT_PATH" ; then
	$GETOPT_PATH -T >/dev/null 2>/dev/null
	if test $? -eq 4; then
	break
	fi
	fi
	GETOPT_PATH=""
	done
	IFS=$IFS_orig
	if test -n "$GETOPT_PATH"; then
	AC_MSG_RESULT([$GETOPT_PATH])
	else
	AC_MSG_RESULT([no])
	AC_MSG_ERROR(Pacemaker build requires a GNU-compatible getopt)
	fi
	AC_SUBST([GETOPT_PATH])

	dnl ========================================================================
	dnl checks for library functions to replace them
	dnl
	dnl NoSuchFunctionName:
	dnl is a dummy function which no system supplies. It is here to make
	dnl the system compile semi-correctly on OpenBSD which doesn't know
	dnl how to create an empty archive
	dnl
	dnl scandir: Only on BSD.
	dnl System-V systems may have it, but hidden and/or deprecated.
	dnl A replacement function is supplied for it.
	dnl
	dnl setenv: is some bsdish function that should also be avoided (use
	dnl putenv instead)
	dnl On the other hand, putenv doesn't provide the right API for the
	dnl code and has memory leaks designed in (sigh...) Fortunately this
	dnl A replacement function is supplied for it.
	dnl
	dnl strerror: returns a string that corresponds to an errno.
	dnl A replacement function is supplied for it.
	dnl
	dnl strnlen: is a gnu function similar to strlen, but safer.
	dnl We wrote a tolearably-fast replacement function for it.
	dnl
	dnl strndup: is a gnu function similar to strdup, but safer.
	dnl We wrote a tolearably-fast replacement function for it.

	AC_REPLACE_FUNCS(alphasort NoSuchFunctionName scandir setenv strerror strchrnul unsetenv strnlen strndup)

	dnl ===============================================
	dnl Libraries
	dnl ===============================================
	AC_CHECK_LIB(socket, socket) dnl -lsocket
	AC_CHECK_LIB(c, dlopen) dnl if dlopen is in libc...
	AC_CHECK_LIB(dl, dlopen) dnl -ldl (for Linux)
	AC_CHECK_LIB(rt, sched_getscheduler) dnl -lrt (for Tru64)
	AC_CHECK_LIB(gnugetopt, getopt_long) dnl -lgnugetopt ( if available )
	AC_CHECK_LIB(pam, pam_start) dnl -lpam (if available)

	AC_CHECK_FUNCS([sched_setscheduler])

	AC_CHECK_LIB(uuid, uuid_parse) dnl load the library if necessary
	AC_CHECK_FUNCS(uuid_unparse) dnl OSX ships uuid_* as standard functions

	AC_CHECK_HEADERS(uuid/uuid.h)

	if test "x$ac_cv_func_uuid_unparse" != xyes; then
	AC_MSG_ERROR(You do not have the libuuid development package installed)
	fi

	if test x"${PKG_CONFIG}" = x""; then
	AC_MSG_ERROR(You need pkgconfig installed in order to build ${PACKAGE})
	fi

	# Require glib 2.16.0 (2008-03) or later for g_hash_table_iter_init() etc.
	PKG_CHECK_MODULES([GLIB], [glib-2.0 >= 2.16.0],
	[CPPFLAGS="${CPPFLAGS} ${GLIB_CFLAGS}"
	LIBS="${LIBS} ${GLIB_LIBS}"])

	#
	# Where is dlopen?
	#
	if test "$ac_cv_lib_c_dlopen" = yes; then
	LIBADD_DL=""
	elif test "$ac_cv_lib_dl_dlopen" = yes; then
	LIBADD_DL=-ldl
	else
	LIBADD_DL=${lt_cv_dlopen_libs}
	fi

	dnl FreeBSD needs -lcompat for ftime() used by lrmd.c
	AC_CHECK_LIB([compat], [ftime], [COMPAT_LIBS='-lcompat'])
	AC_SUBST(COMPAT_LIBS)

	dnl ========================================================================
	dnl Headers
	dnl ========================================================================

	dnl Some distributions insert #warnings into deprecated headers such as
	dnl timeb.h. If we will enable fatal warnings for the build, then enable
	dnl them for the header checks as well, otherwise the build could fail
	dnl even though the header check succeeds. (We should probably be doing
	dnl this in more places.)
	if test "x${enable_fatal_warnings}" = xyes ; then
	cc_temp_flags "$CFLAGS $WERROR"
	fi
	AC_CHECK_HEADERS(arpa/inet.h)
	AC_CHECK_HEADERS(ctype.h)
	AC_CHECK_HEADERS(dirent.h)
	AC_CHECK_HEADERS(errno.h)
	AC_CHECK_HEADERS(getopt.h)
	AC_CHECK_HEADERS(glib.h)
	AC_CHECK_HEADERS(grp.h)
	AC_CHECK_HEADERS(limits.h)
	AC_CHECK_HEADERS(linux/swab.h)
	AC_CHECK_HEADERS(malloc.h)
	AC_CHECK_HEADERS(netdb.h)
	AC_CHECK_HEADERS(netinet/in.h)
	AC_CHECK_HEADERS(netinet/ip.h)
	AC_CHECK_HEADERS(pwd.h)
	AC_CHECK_HEADERS(sgtty.h)
	AC_CHECK_HEADERS(signal.h)
	AC_CHECK_HEADERS(stdarg.h)
	AC_CHECK_HEADERS(stddef.h)
	AC_CHECK_HEADERS(stdio.h)
	AC_CHECK_HEADERS(stdlib.h)
	AC_CHECK_HEADERS(string.h)
	AC_CHECK_HEADERS(strings.h)
	AC_CHECK_HEADERS(sys/dir.h)
	AC_CHECK_HEADERS(sys/ioctl.h)
	AC_CHECK_HEADERS(sys/param.h)
	AC_CHECK_HEADERS(sys/reboot.h)
	AC_CHECK_HEADERS(sys/resource.h)
	AC_CHECK_HEADERS(sys/socket.h)
	AC_CHECK_HEADERS(sys/signalfd.h)
	AC_CHECK_HEADERS(sys/sockio.h)
	AC_CHECK_HEADERS(sys/stat.h)
	AC_CHECK_HEADERS(sys/time.h)
	AC_CHECK_HEADERS(sys/timeb.h)
	AC_CHECK_HEADERS(sys/types.h)
	AC_CHECK_HEADERS(sys/utsname.h)
	AC_CHECK_HEADERS(sys/wait.h)
	AC_CHECK_HEADERS(time.h)
	AC_CHECK_HEADERS(unistd.h)
	if test "x${enable_fatal_warnings}" = xyes ; then
	cc_restore_flags
	fi

	dnl These headers need prerequisites before the tests will pass
	dnl AC_CHECK_HEADERS(net/if.h)

	PKG_CHECK_MODULES(LIBXML2, [libxml-2.0],
	[CPPFLAGS="${CPPFLAGS} ${LIBXML2_CFLAGS}"
	LIBS="${LIBS} ${LIBXML2_LIBS}"])
	AC_CHECK_HEADERS(libxml/xpath.h)
	if test "$ac_cv_header_libxml_xpath_h" != "yes"; then
	AC_MSG_ERROR(libxml development headers not found)
	fi

	AC_CHECK_LIB(xslt, xsltApplyStylesheet, [],
	AC_MSG_ERROR(Unsupported libxslt library version))
	AC_CHECK_HEADERS(libxslt/xslt.h)
	if test "$ac_cv_header_libxslt_xslt_h" != "yes"; then
	AC_MSG_ERROR(libxslt development headers not found)
	fi

	AC_CACHE_CHECK(whether __progname and __progname_full are available,
	pf_cv_var_progname,
	AC_TRY_LINK([extern char __progname, __progname_full;],
	[__progname = "foo"; __progname_full = "foo bar";],
	pf_cv_var_progname="yes", pf_cv_var_progname="no"))

	if test "$pf_cv_var_progname" = "yes"; then
	AC_DEFINE(HAVE___PROGNAME,1,[ ])
	fi

	dnl ========================================================================
	dnl Structures
	dnl ========================================================================

	AC_CHECK_MEMBERS([struct tm.tm_gmtoff],,,[[#include <time.h>]])
	AC_CHECK_MEMBERS([lrm_op_t.rsc_deleted],,,[[#include <lrm/lrm_api.h>]])
	AC_CHECK_MEMBER([struct dirent.d_type],
	AC_DEFINE(HAVE_STRUCT_DIRENT_D_TYPE,1,[Define this if struct dirent has d_type]),,
	[#include <dirent.h>])

	dnl ========================================================================
	dnl Functions
	dnl ========================================================================

	AC_CHECK_FUNCS(getopt, AC_DEFINE(HAVE_DECL_GETOPT, 1, [Have getopt function]))
	AC_CHECK_FUNCS(nanosleep, AC_DEFINE(HAVE_DECL_NANOSLEEP, 1, [Have nanosleep function]))

	dnl ========================================================================
	dnl bzip2
	dnl ========================================================================
	AC_CHECK_HEADERS(bzlib.h)
	AC_CHECK_LIB(bz2, BZ2_bzBuffToBuffCompress)

	if test x$ac_cv_lib_bz2_BZ2_bzBuffToBuffCompress != xyes ; then
	AC_MSG_ERROR(BZ2 libraries not found)
	fi

	if test x$ac_cv_header_bzlib_h != xyes; then
	AC_MSG_ERROR(BZ2 Development headers not found)
	fi

	dnl ========================================================================
	dnl sighandler_t is missing from Illumos, Solaris11 systems
	dnl ========================================================================

	AC_MSG_CHECKING([for sighandler_t])
	AC_TRY_COMPILE([#include <signal.h>],[sighandler_t *f;],
	has_sighandler_t=yes,has_sighandler_t=no)
	AC_MSG_RESULT($has_sighandler_t)
	if test "$has_sighandler_t" = "yes" ; then
	AC_DEFINE( HAVE_SIGHANDLER_T, 1, [Define if sighandler_t available] )
	fi

	dnl ========================================================================
	dnl ncurses
	dnl ========================================================================
	dnl
	dnl A few OSes (e.g. Linux) deliver a default "ncurses" alongside "curses".
	dnl Many non-Linux deliver "curses"; sites may add "ncurses".
	dnl
	dnl However, the source-code recommendation for both is to #include "curses.h"
	dnl (i.e. "ncurses" still wants the include to be simple, no-'n', "curses.h").
	dnl
	dnl ncurse takes precedence.
	dnl
	AC_CHECK_HEADERS(curses.h)
	AC_CHECK_HEADERS(curses/curses.h)
	AC_CHECK_HEADERS(ncurses.h)
	AC_CHECK_HEADERS(ncurses/ncurses.h)

	dnl Although n-library is preferred, only look for it if the n-header was found.
	CURSESLIBS=''
	if test "$ac_cv_header_ncurses_h" = "yes"; then
	AC_CHECK_LIB(ncurses, printw,
	[AC_DEFINE(HAVE_LIBNCURSES,1, have ncurses library)])
	CURSESLIBS=`$PKG_CONFIG --libs ncurses` \|\| CURSESLIBS='-lncurses'
	fi

	if test "$ac_cv_header_ncurses_ncurses_h" = "yes"; then
	AC_CHECK_LIB(ncurses, printw,
	[AC_DEFINE(HAVE_LIBNCURSES,1, have ncurses library)])
	CURSESLIBS=`$PKG_CONFIG --libs ncurses` \|\| CURSESLIBS='-lncurses'
	fi

	dnl Only look for non-n-library if there was no n-library.
	if test X"$CURSESLIBS" = X"" -a "$ac_cv_header_curses_h" = "yes"; then
	AC_CHECK_LIB(curses, printw,
	[CURSESLIBS='-lcurses'; AC_DEFINE(HAVE_LIBCURSES,1, have curses library)])
	fi

	dnl Only look for non-n-library if there was no n-library.
	if test X"$CURSESLIBS" = X"" -a "$ac_cv_header_curses_curses_h" = "yes"; then
	AC_CHECK_LIB(curses, printw,
	[CURSESLIBS='-lcurses'; AC_DEFINE(HAVE_LIBCURSES,1, have curses library)])
	fi

	if test "x$CURSESLIBS" != "x"; then
	PCMK_FEATURES="$PCMK_FEATURES ncurses"
	fi

	dnl Check for printw() prototype compatibility
	if test X"$CURSESLIBS" != X"" && cc_supports_flag -Wcast-qual; then
	ac_save_LIBS=$LIBS
	LIBS="$CURSESLIBS"
	cc_temp_flags "-Wcast-qual $WERROR"
	# avoid broken test because of hardened build environment in Fedora 23+
	# - https://fedoraproject.org/wiki/Changes/Harden_All_Packages
	# - https://bugzilla.redhat.com/1297985
	if cc_supports_flag -fPIC; then
	CFLAGS="$CFLAGS -fPIC"
	fi

	AC_MSG_CHECKING(whether printw() requires argument of "const char *")
	AC_LINK_IFELSE(
	[AC_LANG_PROGRAM([
	#if defined(HAVE_NCURSES_H)
	# include <ncurses.h>
	#elif defined(HAVE_NCURSES_NCURSES_H)
	# include <ncurses/ncurses.h>
	#elif defined(HAVE_CURSES_H)
	# include <curses.h>
	#endif
	],
	[printw((const char *)"Test");]
	)],
	[ac_cv_compatible_printw=yes],
	[ac_cv_compatible_printw=no]
	)

	LIBS=$ac_save_LIBS
	cc_restore_flags

	AC_MSG_RESULT([$ac_cv_compatible_printw])

	if test "$ac_cv_compatible_printw" = no; then
	AC_MSG_WARN([The printw() function of your ncurses or curses library is old, we will disable usage of the library. If you want to use this library anyway, please update to newer version of the library, ncurses 5.4 or later is recommended. You can get the library from http://www.gnu.org/software/ncurses/.])
	AC_MSG_NOTICE([Disabling curses])
	AC_DEFINE(HAVE_INCOMPATIBLE_PRINTW, 1, [Do we have incompatible printw() in curses library?])
	fi
	fi

	AC_SUBST(CURSESLIBS)

	dnl ========================================================================
	dnl Profiling and GProf
	dnl ========================================================================

	AC_MSG_NOTICE(Old CFLAGS: $CFLAGS)
	case $SUPPORT_COVERAGE in
	1\|yes\|true)
	SUPPORT_PROFILING=1
	PCMK_FEATURES="$PCMK_FEATURES coverage"
	CFLAGS="$CFLAGS -fprofile-arcs -ftest-coverage"
	dnl During linking, make sure to specify -lgcov or -coverage
	;;
	esac

	case $SUPPORT_PROFILING in
	1\|yes\|true)
	SUPPORT_PROFILING=1

	dnl Disable various compiler optimizations
	CFLAGS="$CFLAGS -fno-omit-frame-pointer -fno-inline -fno-builtin "
	dnl CFLAGS="$CFLAGS -fno-inline-functions -fno-default-inline -fno-inline-functions-called-once -fno-optimize-sibling-calls"

	dnl Turn off optimization so tools can get accurate line numbers
	CFLAGS=`echo $CFLAGS \| sed -e 's/-O.\ //g' -e 's/-Wp,-D_FORTIFY_SOURCE=.\ //g' -e 's/-D_FORTIFY_SOURCE=.\ //g'`
	CFLAGS="$CFLAGS -O0 -g3 -gdwarf-2"

	dnl Update features
	PCMK_FEATURES="$PCMK_FEATURES profile"
	;;
	*)
	SUPPORT_PROFILING=0
	;;
	esac
	AC_MSG_NOTICE(New CFLAGS: $CFLAGS)
	AC_DEFINE_UNQUOTED(SUPPORT_PROFILING, $SUPPORT_PROFILING, Support for profiling)

	dnl ========================================================================
	dnl Cluster infrastructure - LibQB
	dnl ========================================================================

	if test x${enable_no_stack} = xyes; then
	SUPPORT_CS=no
	fi

	PKG_CHECK_MODULES(libqb, libqb >= 0.13)
	CPPFLAGS="$libqb_CFLAGS $CPPFLAGS"
	LIBS="$libqb_LIBS $LIBS"

	dnl libqb 0.14.0+ (2012-06)
	AC_CHECK_LIB(qb, qb_ipcs_connection_auth_set)

	PCMK_FEATURES="$PCMK_FEATURES libqb-logging libqb-ipc"

	dnl libqb 0.17.0+ (2014-02)
	AC_CHECK_FUNCS(qb_ipcs_connection_get_buffer_size,
	AC_DEFINE(HAVE_IPCS_GET_BUFFER_SIZE, 1,
	[Have qb_ipcc_get_buffer_size function]))

	dnl libqb not yet released (as of 2018-05)
	CHECK_ENUM_VALUE([qb/qblog.h],[qb_log_conf],[QB_LOG_CONF_MAX_LINE_LEN])
	CHECK_ENUM_VALUE([qb/qblog.h],[qb_log_conf],[QB_LOG_CONF_ELLIPSIS])

	dnl Support Linux-HA fence agents if available
	if test "$cross_compiling" != "yes"; then
	CPPFLAGS="$CPPFLAGS -I${prefix}/include/heartbeat"
	fi
	AC_CHECK_HEADERS(stonith/stonith.h)
	if test "$ac_cv_header_stonith_stonith_h" = "yes"; then
	dnl On Debian, AC_CHECK_LIBS fail if a library has any unresolved symbols
	dnl So check for all the dependencies (so they're added to LIBS) before checking for -lplumb
	AC_CHECK_LIB(pils, PILLoadPlugin)
	AC_CHECK_LIB(plumb, G_main_add_IPC_Channel)
	PCMK_FEATURES="$PCMK_FEATURES lha-fencing"
	fi

	dnl ===============================================
	dnl Variables needed for substitution
	dnl ===============================================
	CRM_SCHEMA_DIRECTORY="${datadir}/pacemaker"
	AC_DEFINE_UNQUOTED(CRM_SCHEMA_DIRECTORY,"$CRM_SCHEMA_DIRECTORY", Location for the Pacemaker Relax-NG Schema)
	AC_SUBST(CRM_SCHEMA_DIRECTORY)

	CRM_CORE_DIR="${localstatedir}/lib/pacemaker/cores"
	AC_DEFINE_UNQUOTED(CRM_CORE_DIR,"$CRM_CORE_DIR", Location to store core files produced by Pacemaker daemons)
	AC_SUBST(CRM_CORE_DIR)

	CRM_DAEMON_USER="hacluster"
	AC_DEFINE_UNQUOTED(CRM_DAEMON_USER,"$CRM_DAEMON_USER", User to run Pacemaker daemons as)
	AC_SUBST(CRM_DAEMON_USER)

	CRM_DAEMON_GROUP="haclient"
	AC_DEFINE_UNQUOTED(CRM_DAEMON_GROUP,"$CRM_DAEMON_GROUP", Group to run Pacemaker daemons as)
	AC_SUBST(CRM_DAEMON_GROUP)

	CRM_STATE_DIR=${localstatedir}/run/crm
	AC_DEFINE_UNQUOTED(CRM_STATE_DIR,"$CRM_STATE_DIR", Where to keep state files and sockets)
	AC_SUBST(CRM_STATE_DIR)

	CRM_PACEMAKER_DIR=${localstatedir}/lib/pacemaker
	AC_DEFINE_UNQUOTED(CRM_PACEMAKER_DIR,"$CRM_PACEMAKER_DIR", Location to store directory produced by Pacemaker daemons)
	AC_SUBST(CRM_PACEMAKER_DIR)

	CRM_BLACKBOX_DIR=${localstatedir}/lib/pacemaker/blackbox
	AC_DEFINE_UNQUOTED(CRM_BLACKBOX_DIR,"$CRM_BLACKBOX_DIR", Where to keep blackbox dumps)
	AC_SUBST(CRM_BLACKBOX_DIR)

	PE_STATE_DIR="${localstatedir}/lib/pacemaker/pengine"
	AC_DEFINE_UNQUOTED(PE_STATE_DIR,"$PE_STATE_DIR", Where to keep scheduler outputs)
	AC_SUBST(PE_STATE_DIR)

	CRM_CONFIG_DIR="${localstatedir}/lib/pacemaker/cib"
	AC_DEFINE_UNQUOTED(CRM_CONFIG_DIR,"$CRM_CONFIG_DIR", Where to keep configuration files)
	AC_SUBST(CRM_CONFIG_DIR)

	CRM_CONFIG_CTS="${localstatedir}/lib/pacemaker/cts"
	AC_DEFINE_UNQUOTED(CRM_CONFIG_CTS,"$CRM_CONFIG_CTS", Where to keep cts stateful data)
	AC_SUBST(CRM_CONFIG_CTS)

	CRM_DAEMON_DIR="${libexecdir}/pacemaker"
	AC_DEFINE_UNQUOTED(CRM_DAEMON_DIR,"$CRM_DAEMON_DIR", Location for Pacemaker daemons)
	AC_SUBST(CRM_DAEMON_DIR)

	HA_STATE_DIR="${localstatedir}/run"
	AC_DEFINE_UNQUOTED(HA_STATE_DIR,"$HA_STATE_DIR", Where sbd keeps its PID file)
	AC_SUBST(HA_STATE_DIR)

	CRM_RSCTMP_DIR="${localstatedir}/run/resource-agents"
	AC_DEFINE_UNQUOTED(CRM_RSCTMP_DIR,"$CRM_RSCTMP_DIR", Where resource agents should keep state files)
	AC_SUBST(CRM_RSCTMP_DIR)

	PACEMAKER_CONFIG_DIR="${sysconfdir}/pacemaker"
	AC_DEFINE_UNQUOTED(PACEMAKER_CONFIG_DIR,"$PACEMAKER_CONFIG_DIR", Where to keep configuration files like authkey)
	AC_SUBST(PACEMAKER_CONFIG_DIR)

	OCF_ROOT_DIR="/usr/lib/ocf"
	if test "X$OCF_ROOT_DIR" = X; then
	AC_MSG_ERROR(Could not locate OCF directory)
	fi
	AC_SUBST(OCF_ROOT_DIR)

	OCF_RA_DIR="$OCF_ROOT_DIR/resource.d"
	AC_DEFINE_UNQUOTED(OCF_RA_DIR,"$OCF_RA_DIR", Location for OCF RAs)
	AC_SUBST(OCF_RA_DIR)

	RH_STONITH_DIR="$sbindir"
	AC_DEFINE_UNQUOTED(RH_STONITH_DIR,"$RH_STONITH_DIR", Location for Red Hat Stonith agents)
	AC_DEFINE_UNQUOTED(SBIN_DIR,"$sbindir", Location for system binaries)

	RH_STONITH_PREFIX="fence_"
	AC_DEFINE_UNQUOTED(RH_STONITH_PREFIX,"$RH_STONITH_PREFIX", Prefix for Red Hat Stonith agents)

	AC_PATH_PROGS(GIT, git false)
	AC_MSG_CHECKING(build version)

	BUILD_VERSION=$Format:%h$
	if test $BUILD_VERSION != ":%h$"; then
	AC_MSG_RESULT(archive hash: $BUILD_VERSION)
	elif test -x $GIT -a -d .git; then
	BUILD_VERSION=`$GIT log --pretty="format:%h" -n 1`
	AC_MSG_RESULT(git hash: $BUILD_VERSION)
	else
	# The current directory name make a reasonable default
	# Most generated archives will include the hash or tag
	BASE=`basename $PWD`
	BUILD_VERSION=`echo $BASE \| sed s:.*[[Pp]]acemaker-::`
	AC_MSG_RESULT(directory based hash: $BUILD_VERSION)
	fi

	AC_DEFINE_UNQUOTED(BUILD_VERSION, "$BUILD_VERSION", Build version)
	AC_SUBST(BUILD_VERSION)

	HAVE_dbus=1
	PKG_CHECK_MODULES([DBUS], [dbus-1],
	[CPPFLAGS="${CPPFLAGS} ${DBUS_CFLAGS}"],
	[HAVE_dbus=0])
	AC_DEFINE_UNQUOTED(SUPPORT_DBUS, $HAVE_dbus, Support dbus)
	AM_CONDITIONAL(BUILD_DBUS, test $HAVE_dbus = 1)
	AC_CHECK_TYPES([DBusBasicValue],,,[[#include <dbus/dbus.h>]])

	if test "x${enable_systemd}" != xno; then
	if test $HAVE_dbus = 0; then
	if test "x${enable_systemd}" = xyes; then
	AC_MSG_FAILURE([cannot enable systemd without DBus])
	else
	enable_systemd=no
	fi
	fi
	if test "x${enable_systemd}" = xtry; then
	AC_MSG_CHECKING([for systemd version query result via dbus-send])
	ret=$({ dbus-send --system --print-reply \
	--dest=org.freedesktop.systemd1 \
	/org/freedesktop/systemd1 \
	org.freedesktop.DBus.Properties.Get \
	string:org.freedesktop.systemd1.Manager \
	string:Version 2>/dev/null \
	\|\| echo "this borked"; } \| tail -n1)
	# sanitize output a bit (interested just in value, not type),
	# ret is intentionally unenquoted so as to normalize whitespace
	ret=$(echo ${ret} \| cut -d' ' -f2-)
	AC_MSG_RESULT([${ret}])
	if test "x${ret}" != xborked \
	\|\| systemctl --version 2>/dev/null \| grep -q systemd; then
	enable_systemd=yes
	else
	enable_systemd=no
	fi
	fi
	fi

	AC_MSG_CHECKING([whether to enable support for managing resources via systemd])
	AC_MSG_RESULT([${enable_systemd}])
	HAVE_systemd=0
	if test "x${enable_systemd}" = xyes; then
	HAVE_systemd=1
	PCMK_FEATURES="$PCMK_FEATURES systemd"

	AC_MSG_CHECKING([for systemd path for system unit files])
	systemdunitdir="${systemdunitdir-}"
	PKG_CHECK_VAR([systemdunitdir], [systemd],
	[systemdsystemunitdir], [], [systemdunitdir=no])
	AC_MSG_RESULT([${systemdunitdir}])
	if test "x${systemdunitdir}" = xno; then
	AC_MSG_FAILURE([cannot enable systemd when systemdunitdir unresolved])
	fi
	fi
	AC_SUBST(systemdunitdir)

	AC_DEFINE_UNQUOTED(SUPPORT_SYSTEMD, $HAVE_systemd, Support systemd based system services)
	AM_CONDITIONAL(BUILD_SYSTEMD, test $HAVE_systemd = 1)
	AC_SUBST(SUPPORT_SYSTEMD)

	if test "x${enable_upstart}" != xno; then
	if test $HAVE_dbus = 0; then
	if test "x${enable_upstart}" = xyes; then
	AC_MSG_FAILURE([cannot enable Upstart without DBus])
	else
	enable_upstart=no
	fi
	fi
	if test "x${enable_upstart}" = xtry; then
	AC_MSG_CHECKING([for Upstart version query result via dbus-send])
	ret=$({ dbus-send --system --print-reply --dest=com.ubuntu.Upstart \
	/com/ubuntu/Upstart org.freedesktop.DBus.Properties.Get \
	string:com.ubuntu.Upstart0_6 string:version 2>/dev/null \
	\|\| echo "this borked"; } \| tail -n1)
	# sanitize output a bit (interested just in value, not type),
	# ret is intentionally unenquoted so as to normalize whitespace
	ret=$(echo ${ret} \| cut -d' ' -f2-)
	AC_MSG_RESULT([${ret}])
	if test "x${ret}" != xborked \
	\|\| initctl --version 2>/dev/null \| grep -q upstart; then
	enable_upstart=yes
	else
	enable_upstart=no
	fi
	fi
	fi
	AC_MSG_CHECKING([whether to enable support for managing resources via Upstart])
	AC_MSG_RESULT([${enable_upstart}])
	HAVE_upstart=0
	if test "x${enable_upstart}" = xyes; then
	HAVE_upstart=1
	PCMK_FEATURES="$PCMK_FEATURES upstart"
	fi

	AC_DEFINE_UNQUOTED(SUPPORT_UPSTART, $HAVE_upstart, Support upstart based system services)
	AM_CONDITIONAL(BUILD_UPSTART, test $HAVE_upstart = 1)
	AC_SUBST(SUPPORT_UPSTART)

	case $SUPPORT_NAGIOS in
	1\|yes\|true\|try)
	SUPPORT_NAGIOS=1
	;;
	*)
	SUPPORT_NAGIOS=0
	;;
	esac

	if test $SUPPORT_NAGIOS = 1; then
	PCMK_FEATURES="$PCMK_FEATURES nagios"
	fi

	AC_DEFINE_UNQUOTED(SUPPORT_NAGIOS, $SUPPORT_NAGIOS, Support nagios plugins)
	AM_CONDITIONAL(BUILD_NAGIOS, test $SUPPORT_NAGIOS = 1)

	if test x"$NAGIOS_PLUGIN_DIR" = x""; then
	NAGIOS_PLUGIN_DIR="${libexecdir}/nagios/plugins"
	fi

	AC_DEFINE_UNQUOTED(NAGIOS_PLUGIN_DIR, "$NAGIOS_PLUGIN_DIR", Directory for nagios plugins)
	AC_SUBST(NAGIOS_PLUGIN_DIR)

	if test x"$NAGIOS_METADATA_DIR" = x""; then
	NAGIOS_METADATA_DIR="${datadir}/nagios/plugins-metadata"
	fi

	AC_DEFINE_UNQUOTED(NAGIOS_METADATA_DIR, "$NAGIOS_METADATA_DIR", Directory for nagios plugins metadata)
	AC_SUBST(NAGIOS_METADATA_DIR)

	STACKS=""
	CLUSTERLIBS=""

	dnl ========================================================================
	dnl Cluster stack - Corosync
	dnl ========================================================================

	dnl Normalize the values
	case $SUPPORT_CS in
	1\|yes\|true)
	SUPPORT_CS=yes
	missingisfatal=1
	;;
	try)
	missingisfatal=0
	;;
	*)
	SUPPORT_CS=no
	;;
	esac

	AC_MSG_CHECKING(for native corosync)
	COROSYNC_LIBS=""

	if test $SUPPORT_CS = no; then
	AC_MSG_RESULT(no (disabled))
	SUPPORT_CS=0
	else
	AC_MSG_RESULT($SUPPORT_CS)
	SUPPORT_CS=1
	PKG_CHECK_MODULES(cpg, libcpg) dnl Fatal
	PKG_CHECK_MODULES(cfg, libcfg) dnl Fatal
	PKG_CHECK_MODULES(cmap, libcmap) dnl Fatal
	PKG_CHECK_MODULES(quorum, libquorum) dnl Fatal
	PKG_CHECK_MODULES(libcorosync_common, libcorosync_common) dnl Fatal

	CFLAGS="$CFLAGS $libqb_FLAGS $cpg_FLAGS $cfg_FLAGS $cmap_CFLAGS $quorum_CFLAGS $libcorosync_common_CFLAGS"
	COROSYNC_LIBS="$COROSYNC_LIBS $libqb_LIBS $cpg_LIBS $cfg_LIBS $cmap_LIBS $quorum_LIBS $libcorosync_common_LIBS"
	CLUSTERLIBS="$CLUSTERLIBS $COROSYNC_LIBS"
	STACKS="$STACKS corosync-native"
	fi

	AC_DEFINE_UNQUOTED(SUPPORT_COROSYNC, $SUPPORT_CS, Support the Corosync messaging and membership layer)
	AM_CONDITIONAL(BUILD_CS_SUPPORT, test $SUPPORT_CS = 1)
	AC_SUBST(SUPPORT_COROSYNC)

	dnl
	dnl Cluster stack - Sanity
	dnl

	if test x${enable_no_stack} = xyes; then
	AC_MSG_NOTICE(No cluster stack supported, building only the scheduler)
	PCMK_FEATURES="$PCMK_FEATURES no-cluster-stack"
	else
	AC_MSG_CHECKING(for supported stacks)
	if test x"$STACKS" = x; then
	AC_MSG_FAILURE(You must support at least one cluster stack)
	fi
	AC_MSG_RESULT($STACKS)
	PCMK_FEATURES="$PCMK_FEATURES $STACKS"
	fi

	PCMK_FEATURES="$PCMK_FEATURES atomic-attrd"
	AC_SUBST(CLUSTERLIBS)

	dnl ========================================================================
	dnl ACL
	dnl ========================================================================

	case $SUPPORT_ACL in
	1\|yes\|true)
	missingisfatal=1
	;;
	try)
	missingisfatal=0
	;;
	*)
	SUPPORT_ACL=no
	;;
	esac

	AC_MSG_CHECKING(for acl support)
	if test $SUPPORT_ACL = no; then
	AC_MSG_RESULT(no (disabled))
	SUPPORT_ACL=0
	else
	AC_MSG_RESULT($SUPPORT_ACL)

	SUPPORT_ACL=1
	AC_CHECK_LIB(qb, qb_ipcs_connection_auth_set)
	if test $ac_cv_lib_qb_qb_ipcs_connection_auth_set != yes; then
	SUPPORT_ACL=0
	fi

	if test $SUPPORT_ACL = 0; then
	if test $missingisfatal = 0; then
	AC_MSG_WARN(Unable to support ACL. You need to use libqb > 0.13.0)
	else
	AC_MSG_FAILURE(Unable to support ACL. You need to use libqb > 0.13.0)
	fi
	fi
	fi

	if test $SUPPORT_ACL = 1; then
	PCMK_FEATURES="$PCMK_FEATURES acls"
	fi

	AM_CONDITIONAL(ENABLE_ACL, test "$SUPPORT_ACL" = "1")
	AC_DEFINE_UNQUOTED(ENABLE_ACL, $SUPPORT_ACL, Build in support for CIB ACL)

	dnl ========================================================================
	dnl CIB secrets
	dnl ========================================================================

	case $SUPPORT_CIBSECRETS in
	1\|yes\|true\|try)
	SUPPORT_CIBSECRETS=1
	;;
	*)
	SUPPORT_CIBSECRETS=0
	;;
	esac

	AC_DEFINE_UNQUOTED(SUPPORT_CIBSECRETS, $SUPPORT_CIBSECRETS, Support CIB secrets)
	AM_CONDITIONAL(BUILD_CIBSECRETS, test $SUPPORT_CIBSECRETS = 1)

	if test $SUPPORT_CIBSECRETS = 1; then
	PCMK_FEATURES="$PCMK_FEATURES cibsecrets"

	LRM_CIBSECRETS_DIR="${localstatedir}/lib/pacemaker/lrm/secrets"
	AC_DEFINE_UNQUOTED(LRM_CIBSECRETS_DIR,"$LRM_CIBSECRETS_DIR", Location for CIB secrets)
	AC_SUBST(LRM_CIBSECRETS_DIR)
	fi

	dnl ========================================================================
	dnl GnuTLS
	dnl ========================================================================

	dnl gnutls_priority_set_direct available since 2.1.7 (released 2007-11-29)
	AC_CHECK_LIB(gnutls, gnutls_priority_set_direct)
	if test "$ac_cv_lib_gnutls_gnutls_priority_set_direct" != ""; then
	AC_CHECK_HEADERS(gnutls/gnutls.h)
	fi

	dnl ========================================================================
	dnl PAM
	dnl ========================================================================

	AC_CHECK_HEADERS(security/pam_appl.h pam/pam_appl.h)

	dnl ========================================================================
	dnl System Health
	dnl ========================================================================

	dnl Check if servicelog development package is installed
	SERVICELOG=servicelog-1
	SERVICELOG_EXISTS="no"
	AC_MSG_CHECKING(for $SERVICELOG packages)
	if
	$PKG_CONFIG --exists $SERVICELOG
	then
	PKG_CHECK_MODULES([SERVICELOG], [servicelog-1])
	SERVICELOG_EXISTS="yes"
	fi
	AC_MSG_RESULT($SERVICELOG_EXISTS)
	AM_CONDITIONAL(BUILD_SERVICELOG, test "$SERVICELOG_EXISTS" = "yes")

	dnl Check if OpenIMPI packages and servicelog are installed
	OPENIPMI="OpenIPMI OpenIPMIposix"
	OPENIPMI_SERVICELOG_EXISTS="no"
	AC_MSG_CHECKING(for $SERVICELOG $OPENIPMI packages)
	if
	$PKG_CONFIG --exists $OPENIPMI $SERVICELOG
	then
	PKG_CHECK_MODULES([OPENIPMI_SERVICELOG],[OpenIPMI OpenIPMIposix])
	OPENIPMI_SERVICELOG_EXISTS="yes"
	fi
	AC_MSG_RESULT($OPENIPMI_SERVICELOG_EXISTS)
	AM_CONDITIONAL(BUILD_OPENIPMI_SERVICELOG, test "$OPENIPMI_SERVICELOG_EXISTS" = "yes")

	dnl ========================================================================
	dnl Compiler flags
	dnl ========================================================================

	dnl Make sure that CFLAGS is not exported. If the user did
	dnl not have CFLAGS in their environment then this should have
	dnl no effect. However if CFLAGS was exported from the user's
	dnl environment, then the new CFLAGS will also be exported
	dnl to sub processes.
	if export \| fgrep " CFLAGS=" > /dev/null; then
	SAVED_CFLAGS="$CFLAGS"
	unset CFLAGS
	CFLAGS="$SAVED_CFLAGS"
	unset SAVED_CFLAGS
	fi

	AC_ARG_VAR([CFLAGS_HARDENED_LIB], [extra C compiler flags for hardened libraries])
	AC_ARG_VAR([LDFLAGS_HARDENED_LIB], [extra linker flags for hardened libraries])

	AC_ARG_VAR([CFLAGS_HARDENED_EXE], [extra C compiler flags for hardened executables])
	AC_ARG_VAR([LDFLAGS_HARDENED_EXE], [extra linker flags for hardened executables])

	CC_EXTRAS=""

	if test "$GCC" != yes; then
	CFLAGS="$CFLAGS -g"
	else
	CFLAGS="$CFLAGS -ggdb"

	dnl When we don't have diagnostic push / pull, we can't explicitly disable
	dnl checking for nonliteral formats in the places where they occur on purpose
	dnl thus we disable nonliteral format checking globally as we are aborting
	dnl on warnings.
	dnl what makes the things really ugly is that nonliteral format checking is
	dnl obviously available as an extra switch in very modern gcc but for older
	dnl gcc this is part of -Wformat=2
	dnl so if we have push/pull we can enable -Wformat=2 -Wformat-nonliteral
	dnl if we don't have push/pull but -Wformat-nonliteral we can enable -Wformat=2
	dnl otherwise none of both

	gcc_diagnostic_push_pull=no
	cc_temp_flags "$CFLAGS $WERROR"
	AC_MSG_CHECKING([for gcc diagnostic push / pull])
	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
	#pragma GCC diagnostic push
	#pragma GCC diagnostic pop
	]])],
	[
	AC_MSG_RESULT([yes])
	gcc_diagnostic_push_pull=yes
	], AC_MSG_RESULT([no]))
	cc_restore_flags

	if cc_supports_flag "-Wformat-nonliteral"; then
	gcc_format_nonliteral=yes
	else
	gcc_format_nonliteral=no
	fi

	# We had to eliminate -Wnested-externs because of libtool changes
	# Make sure to order options so that the former stand for prerequisites
	# of the latter (e.g., -Wformat-nonliteral requires -Wformat).
	EXTRA_FLAGS="-fgnu89-inline
	-Wall
	-Waggregate-return
	-Wbad-function-cast
	-Wcast-align
	-Wdeclaration-after-statement
	-Wendif-labels
	-Wfloat-equal
	-Wformat-security
	-Wmissing-prototypes
	-Wmissing-declarations
	-Wnested-externs
	-Wno-long-long
	-Wno-strict-aliasing
	-Wpointer-arith
	-Wstrict-prototypes
	-Wwrite-strings
	-Wunused-but-set-variable
	-Wunsigned-char"

	if test "x$gcc_diagnostic_push_pull" = "xyes"; then
	AC_DEFINE([GCC_FORMAT_NONLITERAL_CHECKING_ENABLED], [],
	[gcc can complain about nonliterals in format])
	EXTRA_FLAGS="$EXTRA_FLAGS
	-Wformat=2
	-Wformat-nonliteral"
	else
	if test "x$gcc_format_nonliteral" = "xyes"; then
	EXTRA_FLAGS="$EXTRA_FLAGS -Wformat=2"
	fi
	fi

	# Additional warnings it might be nice to enable one day
	# -Wshadow
	# -Wunreachable-code
	for j in $EXTRA_FLAGS
	do
	if
	cc_supports_flag $CC_EXTRAS $j
	then
	CC_EXTRAS="$CC_EXTRAS $j"
	fi
	done

	if test "x${enable_ansi}" = xyes && cc_supports_flag -std=iso9899:199409 ; then
	AC_MSG_NOTICE(Enabling ANSI Compatibility)
	CC_EXTRAS="$CC_EXTRAS -ansi -D_GNU_SOURCE -DANSI_ONLY"
	fi

	AC_MSG_NOTICE(Activated additional gcc flags: ${CC_EXTRAS})
	fi

	dnl
	dnl Hardening flags
	dnl
	dnl The prime control of whether to apply (targeted) hardening build flags and
	dnl which ones is --{enable,disable}-hardening option passed to ./configure:
	dnl
	dnl --enable-hardening=try (default):
	dnl depending on whether any of CFLAGS_HARDENED_EXE, LDFLAGS_HARDENED_EXE,
	dnl CFLAGS_HARDENED_LIB or LDFLAGS_HARDENED_LIB environment variables
	dnl (see below) is set and non-null, all these custom flags (even if not
	dnl set) are used as are, otherwise the best effort is made to offer
	dnl reasonably strong hardening in several categories (RELRO, PIE,
	dnl "bind now", stack protector) according to what the selected toolchain
	dnl can offer
	dnl
	dnl --enable-hardening:
	dnl same effect as --enable-hardening=try when the environment variables
	dnl in question are suppressed
	dnl
	dnl --disable-hardening:
	dnl do not apply any targeted hardening measures at all
	dnl
	dnl The user-injected environment variables that regulate the hardening in
	dnl default case are as follows:
	dnl
	dnl * CFLAGS_HARDENED_EXE, LDFLAGS_HARDENED_EXE
	dnl compiler and linker flags (respectively) for daemon programs
	dnl (pacemakerd, pacemaker-attrd, pacemaker-controld, pacemaker-execd,
	dnl cib, stonithd, pacemaker-remoted, pacemaker-schedulerd)
	dnl
	dnl * CFLAGS_HARDENED_LIB, LDFLAGS_HARDENED_LIB
	dnl compiler and linker flags (respectively) for libraries linked
	dnl with the daemon programs
	dnl
	dnl Note that these are purposedly targeted variables (addressing particular
	dnl targets all over the scattered Makefiles) and have no effect outside of
	dnl the predestined scope (e.g., CLI utilities). For a global reach,
	dnl use CFLAGS, LDFLAGS, etc. as usual.
	dnl
	dnl For guidance on the suitable flags consult, for instance:
	dnl https://fedoraproject.org/wiki/Changes/Harden_All_Packages#Detailed_Harden_Flags_Description
	dnl https://owasp.org/index.php/C-Based_Toolchain_Hardening#GCC.2FBinutils
	dnl

	if test "x${HARDENING}" != "xtry"; then
	unset CFLAGS_HARDENED_EXE
	unset CFLAGS_HARDENED_LIB
	unset LDFLAGS_HARDENED_EXE
	unset LDFLAGS_HARDENED_LIB
	fi
	if test "x${HARDENING}" = "xno"; then
	AC_MSG_NOTICE([Hardening: explicitly disabled])
	elif test "x${HARDENING}" = "xyes" \
	\|\| test "$(env \| grep -Ec '^(C\|LD)FLAGS_HARDENED_(EXE\|LIB)=.')" = 0; then
	dnl We'll figure out on our own...
	CFLAGS_HARDENED_EXE=
	CFLAGS_HARDENED_LIB=
	LDFLAGS_HARDENED_EXE=
	LDFLAGS_HARDENED_LIB=
	relro=0
	pie=0
	bindnow=0
	# daemons incl. libs: partial RELRO
	flag="-Wl,-z,relro"
	CC_CHECK_LDFLAGS(["${flag}"],
	[LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}";
	LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}";
	relro=1])
	# daemons: PIE for both CFLAGS and LDFLAGS
	if cc_supports_flag -fPIE; then
	flag="-pie"
	CC_CHECK_LDFLAGS(["${flag}"],
	[CFLAGS_HARDENED_EXE="${CFLAGS_HARDENED_EXE} -fPIE";
	LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}";
	pie=1])
	fi
	# daemons incl. libs: full RELRO if sensible + as-needed linking
	# so as to possibly mitigate startup performance
	# hit caused by excessive linking with unneeded
	# libraries
	if test "${relro}" = 1 && test "${pie}" = 1; then
	flag="-Wl,-z,now"
	CC_CHECK_LDFLAGS(["${flag}"],
	[LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}";
	LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}";
	bindnow=1])
	fi
	if test "${bindnow}" = 1; then
	flag="-Wl,--as-needed"
	CC_CHECK_LDFLAGS(["${flag}"],
	[LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}";
	LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}"])
	fi
	# universal: prefer strong > all > default stack protector if possible
	flag=
	if cc_supports_flag -fstack-protector-strong; then
	flag="-fstack-protector-strong"
	elif cc_supports_flag -fstack-protector-all; then
	flag="-fstack-protector-all"
	elif cc_supports_flag -fstack-protector; then
	flag="-fstack-protector"
	fi
	if test -n "${flag}"; then
	CC_EXTRAS="${CC_EXTRAS} ${flag}"
	stackprot=1
	fi
	if test "${relro}" = 1 \
	\|\| test "${pie}" = 1 \
	\|\| test "${stackprot}" = 1; then
	AC_MSG_NOTICE([Hardening: relro=${relro} pie=${pie} bindnow=${bindnow} stackprot=${flag}])
	else
	AC_MSG_WARN([Hardening: no suitable features in the toolchain detected])
	fi
	else
	AC_MSG_NOTICE([Hardening: using custom flags])
	fi

	CFLAGS="$CFLAGS $CC_EXTRAS"

	NON_FATAL_CFLAGS="$CFLAGS"
	AC_SUBST(NON_FATAL_CFLAGS)

	dnl
	dnl We reset CFLAGS to include our warnings after all function
	dnl checking goes on, so that our warning flags don't keep the
	dnl AC_*FUNCS() calls above from working. In particular, -Werror will
	dnl always cause us troubles if we set it before here.
	dnl
	dnl
	if test "x${enable_fatal_warnings}" = xyes ; then
	AC_MSG_NOTICE(Enabling Fatal Warnings)
	CFLAGS="$CFLAGS $WERROR"
	fi
	AC_SUBST(CFLAGS)

	dnl This is useful for use in Makefiles that need to remove one specific flag
	CFLAGS_COPY="$CFLAGS"
	AC_SUBST(CFLAGS_COPY)

	AC_SUBST(LIBADD_DL) dnl extra flags for dynamic linking libraries
	AC_SUBST(LIBADD_INTL) dnl extra flags for GNU gettext stuff...

	AC_SUBST(LOCALE)

	dnl Options for cleaning up the compiler output
	QUIET_LIBTOOL_OPTS=""
	QUIET_MAKE_OPTS=""
	if test "x${enable_quiet}" = "xyes"; then
	QUIET_LIBTOOL_OPTS="--quiet"
	QUIET_MAKE_OPTS="--quiet"
	fi

	AC_MSG_RESULT(Supress make details: ${enable_quiet})

	dnl Put the above variables to use
	LIBTOOL="${LIBTOOL} --tag=CC \$(QUIET_LIBTOOL_OPTS)"
	MAKE="${MAKE} \$(QUIET_MAKE_OPTS)"

	AC_SUBST(CC)
	AC_SUBST(MAKE)
	AC_SUBST(LIBTOOL)
	AC_SUBST(QUIET_MAKE_OPTS)
	AC_SUBST(QUIET_LIBTOOL_OPTS)
	AC_DEFINE_UNQUOTED(CRM_FEATURES, "$PCMK_FEATURES", Set of enabled features)
	AC_SUBST(PCMK_FEATURES)

	dnl Files we output that need to be executable
	AC_CONFIG_FILES([cts/CTSlab.py], [chmod +x cts/CTSlab.py])
	AC_CONFIG_FILES([cts/LSBDummy], [chmod +x cts/LSBDummy])
	AC_CONFIG_FILES([cts/OCFIPraTest.py], [chmod +x cts/OCFIPraTest.py])
	AC_CONFIG_FILES([cts/cluster_test], [chmod +x cts/cluster_test])
	AC_CONFIG_FILES([cts/cts], [chmod +x cts/cts])
	AC_CONFIG_FILES([cts/cts-cli], [chmod +x cts/cts-cli])
	AC_CONFIG_FILES([cts/cts-coverage], [chmod +x cts/cts-coverage])
	AC_CONFIG_FILES([cts/cts-exec], [chmod +x cts/cts-exec])
	AC_CONFIG_FILES([cts/cts-fencing], [chmod +x cts/cts-fencing])
	AC_CONFIG_FILES([cts/cts-log-watcher], [chmod +x cts/cts-log-watcher])
	AC_CONFIG_FILES([cts/cts-regression], [chmod +x cts/cts-regression])
	AC_CONFIG_FILES([cts/cts-scheduler], [chmod +x cts/cts-scheduler])
	AC_CONFIG_FILES([cts/cts-support], [chmod +x cts/cts-support])
	AC_CONFIG_FILES([cts/lxc_autogen.sh], [chmod +x cts/lxc_autogen.sh])
	AC_CONFIG_FILES([cts/benchmark/clubench], [chmod +x cts/benchmark/clubench])
	AC_CONFIG_FILES([cts/fence_dummy], [chmod +x cts/fence_dummy])
	AC_CONFIG_FILES([cts/pacemaker-cts-dummyd], [chmod +x cts/pacemaker-cts-dummyd])
	AC_CONFIG_FILES([daemons/fenced/fence_legacy], [chmod +x daemons/fenced/fence_legacy])
	+AC_CONFIG_FILES([extra/resources/ClusterMon], [chmod +x extra/resources/ClusterMon])
	+AC_CONFIG_FILES([extra/resources/HealthSMART], [chmod +x extra/resources/HealthSMART])
	+AC_CONFIG_FILES([extra/resources/SysInfo], [chmod +x extra/resources/SysInfo])
	+AC_CONFIG_FILES([extra/resources/ifspeed], [chmod +x extra/resources/ifspeed])
	+AC_CONFIG_FILES([extra/resources/o2cb], [chmod +x extra/resources/o2cb])
	AC_CONFIG_FILES([tools/crm_failcount], [chmod +x tools/crm_failcount])
	AC_CONFIG_FILES([tools/crm_master], [chmod +x tools/crm_master])
	AC_CONFIG_FILES([tools/crm_report], [chmod +x tools/crm_report])
	AC_CONFIG_FILES([tools/crm_standby], [chmod +x tools/crm_standby])
	AC_CONFIG_FILES([tools/cibsecret], [chmod +x tools/cibsecret])

	dnl Other files we output
	AC_CONFIG_FILES(Makefile \
	Doxyfile \
	cts/Makefile \
	cts/CTS.py \
	cts/CTSvars.py \
	cts/benchmark/Makefile \
	cts/pacemaker-cts-dummyd@.service \
	daemons/Makefile \
	daemons/attrd/Makefile \
	daemons/based/Makefile \
	daemons/controld/Makefile \
	daemons/execd/Makefile \
	daemons/execd/pacemaker_remote \
	daemons/execd/pacemaker_remote.service \
	daemons/fenced/Makefile \
	daemons/pacemakerd/Makefile \
	daemons/pacemakerd/pacemaker \
	daemons/pacemakerd/pacemaker.service \
	daemons/pacemakerd/pacemaker.upstart \
	daemons/pacemakerd/pacemaker.combined.upstart \
	daemons/schedulerd/Makefile \
	doc/Makefile \
	doc/Clusters_from_Scratch/publican.cfg \
	doc/Pacemaker_Administration/publican.cfg \
	doc/Pacemaker_Development/publican.cfg \
	doc/Pacemaker_Explained/publican.cfg \
	doc/Pacemaker_Remote/publican.cfg \
	extra/Makefile \
	extra/alerts/Makefile \
	extra/resources/Makefile \
	extra/logrotate/Makefile \
	extra/logrotate/pacemaker \
	include/Makefile \
	include/crm/Makefile \
	include/crm/cib/Makefile \
	include/crm/common/Makefile \
	include/crm/cluster/Makefile \
	include/crm/fencing/Makefile \
	include/crm/pengine/Makefile \
	replace/Makefile \
	lib/Makefile \
	lib/pacemaker.pc \
	lib/pacemaker-cib.pc \
	lib/pacemaker-lrmd.pc \
	lib/pacemaker-service.pc \
	lib/pacemaker-pengine.pc \
	lib/pacemaker-fencing.pc \
	lib/pacemaker-cluster.pc \
	lib/common/Makefile \
	lib/cluster/Makefile \
	lib/cib/Makefile \
	lib/gnu/Makefile \
	lib/pengine/Makefile \
	lib/transition/Makefile \
	lib/fencing/Makefile \
	lib/lrmd/Makefile \
	lib/services/Makefile \
	tools/Makefile \
	tools/report.collector \
	tools/report.common \
	tools/crm_mon.service \
	tools/crm_mon.upstart \
	xml/Makefile \
	)

	dnl Now process the entire list of files added by previous
	dnl calls to AC_CONFIG_FILES()
	AC_OUTPUT()

	dnl *****************
	dnl Configure summary
	dnl *****************

	AC_MSG_RESULT([])
	AC_MSG_RESULT([$PACKAGE configuration:])
	AC_MSG_RESULT([ Version = ${VERSION} (Build: $BUILD_VERSION)])
	AC_MSG_RESULT([ Features =${PCMK_FEATURES}])
	AC_MSG_RESULT([])
	AC_MSG_RESULT([ Prefix = ${prefix}])
	AC_MSG_RESULT([ Executables = ${sbindir}])
	AC_MSG_RESULT([ Man pages = ${mandir}])
	AC_MSG_RESULT([ Libraries = ${libdir}])
	AC_MSG_RESULT([ Header files = ${includedir}])
	AC_MSG_RESULT([ Arch-independent files = ${datadir}])
	AC_MSG_RESULT([ State information = ${localstatedir}])
	AC_MSG_RESULT([ System configuration = ${sysconfdir}])
	AC_MSG_RESULT([])
	AC_MSG_RESULT([ HA group name = ${CRM_DAEMON_GROUP}])
	AC_MSG_RESULT([ HA user name = ${CRM_DAEMON_USER}])
	AC_MSG_RESULT([])
	AC_MSG_RESULT([ CFLAGS = ${CFLAGS}])
	AC_MSG_RESULT([ CFLAGS_HARDENED_EXE = ${CFLAGS_HARDENED_EXE}])
	AC_MSG_RESULT([ CFLAGS_HARDENED_LIB = ${CFLAGS_HARDENED_LIB}])
	AC_MSG_RESULT([ LDFLAGS_HARDENED_EXE = ${LDFLAGS_HARDENED_EXE}])
	AC_MSG_RESULT([ LDFLAGS_HARDENED_LIB = ${LDFLAGS_HARDENED_LIB}])
	AC_MSG_RESULT([ Libraries = ${LIBS}])
	AC_MSG_RESULT([ Stack Libraries = ${CLUSTERLIBS}])
	diff --git a/cts/cli/regression.tools.exp b/cts/cli/regression.tools.exp
	index 2caf1c44ff..f8c7773c71 100644
	--- a/cts/cli/regression.tools.exp
	+++ b/cts/cli/regression.tools.exp
	@@ -1,2982 +1,2978 @@
	Created new pacemaker configuration
	Setting up shadow instance
	A new shadow instance was created. To begin using it paste the following into your shell:
	CIB_shadow=cts-cli ; export CIB_shadow
	=#=#=#= Begin test: Validate CIB =#=#=#=
	<cib epoch="0" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config/>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= Current cib after: Validate CIB =#=#=#=
	<cib epoch="0" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config/>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Validate CIB - OK (0) =#=#=#=
	* Passed: cibadmin - Validate CIB
	=#=#=#= Begin test: Configure something before erasing =#=#=#=
	=#=#=#= Current cib after: Configure something before erasing =#=#=#=
	<cib epoch="1" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Configure something before erasing - OK (0) =#=#=#=
	* Passed: crm_attribute - Configure something before erasing
	=#=#=#= Begin test: Require --force for CIB erasure =#=#=#=
	The supplied command is considered dangerous. To prevent accidental destruction of the cluster, the --force flag is required in order to proceed.
	=#=#=#= Current cib after: Require --force for CIB erasure =#=#=#=
	<cib epoch="1" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Require --force for CIB erasure - Operation not safe (107) =#=#=#=
	* Passed: cibadmin - Require --force for CIB erasure
	=#=#=#= Begin test: Allow CIB erasure with --force =#=#=#=
	=#=#=#= Current cib after: Allow CIB erasure with --force =#=#=#=
	<cib epoch="2" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config/>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Allow CIB erasure with --force - OK (0) =#=#=#=
	* Passed: cibadmin - Allow CIB erasure with --force
	=#=#=#= Begin test: Query CIB =#=#=#=
	=#=#=#= Current cib after: Query CIB =#=#=#=
	<cib epoch="2" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config/>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Query CIB - OK (0) =#=#=#=
	* Passed: cibadmin - Query CIB
	=#=#=#= Begin test: Set cluster option =#=#=#=
	=#=#=#= Current cib after: Set cluster option =#=#=#=
	<cib epoch="3" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Set cluster option - OK (0) =#=#=#=
	* Passed: crm_attribute - Set cluster option
	=#=#=#= Begin test: Query new cluster option =#=#=#=
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	=#=#=#= Current cib after: Query new cluster option =#=#=#=
	<cib epoch="3" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Query new cluster option - OK (0) =#=#=#=
	* Passed: cibadmin - Query new cluster option
	=#=#=#= Begin test: Query cluster options =#=#=#=
	=#=#=#= Current cib after: Query cluster options =#=#=#=
	<cib epoch="3" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Query cluster options - OK (0) =#=#=#=
	* Passed: cibadmin - Query cluster options
	=#=#=#= Begin test: Set no-quorum policy =#=#=#=
	=#=#=#= Current cib after: Set no-quorum policy =#=#=#=
	<cib epoch="4" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Set no-quorum policy - OK (0) =#=#=#=
	* Passed: crm_attribute - Set no-quorum policy
	=#=#=#= Begin test: Delete nvpair =#=#=#=
	=#=#=#= Current cib after: Delete nvpair =#=#=#=
	<cib epoch="5" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Delete nvpair - OK (0) =#=#=#=
	* Passed: cibadmin - Delete nvpair
	=#=#=#= Begin test: Create operation should fail =#=#=#=
	Call failed: File exists
	<failed>
	<failed_update id="cib-bootstrap-options" object_type="cluster_property_set" operation="cib_create" reason="File exists">
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</failed_update>
	</failed>
	=#=#=#= Current cib after: Create operation should fail =#=#=#=
	<cib epoch="5" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Create operation should fail - Requested item already exists (108) =#=#=#=
	* Passed: cibadmin - Create operation should fail
	=#=#=#= Begin test: Modify cluster options section =#=#=#=
	=#=#=#= Current cib after: Modify cluster options section =#=#=#=
	<cib epoch="6" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Modify cluster options section - OK (0) =#=#=#=
	* Passed: cibadmin - Modify cluster options section
	=#=#=#= Begin test: Query updated cluster option =#=#=#=
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	=#=#=#= Current cib after: Query updated cluster option =#=#=#=
	<cib epoch="6" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Query updated cluster option - OK (0) =#=#=#=
	* Passed: cibadmin - Query updated cluster option
	=#=#=#= Begin test: Set duplicate cluster option =#=#=#=
	=#=#=#= Current cib after: Set duplicate cluster option =#=#=#=
	<cib epoch="7" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="40s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Set duplicate cluster option - OK (0) =#=#=#=
	* Passed: crm_attribute - Set duplicate cluster option
	=#=#=#= Begin test: Setting multiply defined cluster option should fail =#=#=#=
	-Error performing operation: Name not unique on network
	Multiple attributes match name=cluster-delay
	Value: 60s (id=cib-bootstrap-options-cluster-delay)
	Value: 40s (id=duplicate-cluster-delay)
	+Please choose from one of the matches above and supply the 'id' with --attr-id
	=#=#=#= Current cib after: Setting multiply defined cluster option should fail =#=#=#=
	<cib epoch="7" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="40s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Setting multiply defined cluster option should fail - Multiple items match request (109) =#=#=#=
	* Passed: crm_attribute - Setting multiply defined cluster option should fail
	=#=#=#= Begin test: Set cluster option with -s =#=#=#=
	=#=#=#= Current cib after: Set cluster option with -s =#=#=#=
	<cib epoch="8" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Set cluster option with -s - OK (0) =#=#=#=
	* Passed: crm_attribute - Set cluster option with -s
	=#=#=#= Begin test: Delete cluster option with -i =#=#=#=
	Deleted crm_config option: id=(null) name=cluster-delay

	=#=#=#= Current cib after: Delete cluster option with -i =#=#=#=
	<cib epoch="9" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Delete cluster option with -i - OK (0) =#=#=#=
	* Passed: crm_attribute - Delete cluster option with -i
	=#=#=#= Begin test: Create node1 and bring it online =#=#=#=

	Current cluster status:


	Performing requested modifications
	+ Bringing node node1 online

	Transition Summary:

	Executing cluster transition:

	Revised cluster status:
	Online: [ node1 ]


	=#=#=#= Current cib after: Create node1 and bring it online =#=#=#=
	<cib epoch="10" num_updates="2" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1"/>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate"/>
	</status>
	</cib>
	=#=#=#= End test: Create node1 and bring it online - OK (0) =#=#=#=
	* Passed: crm_simulate - Create node1 and bring it online
	=#=#=#= Begin test: Create node attribute =#=#=#=
	=#=#=#= Current cib after: Create node attribute =#=#=#=
	<cib epoch="11" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate"/>
	</status>
	</cib>
	=#=#=#= End test: Create node attribute - OK (0) =#=#=#=
	* Passed: crm_attribute - Create node attribute
	=#=#=#= Begin test: Query new node attribute =#=#=#=
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	=#=#=#= Current cib after: Query new node attribute =#=#=#=
	<cib epoch="11" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate"/>
	</status>
	</cib>
	=#=#=#= End test: Query new node attribute - OK (0) =#=#=#=
	* Passed: cibadmin - Query new node attribute
	=#=#=#= Begin test: Set a transient (fail-count) node attribute =#=#=#=
	=#=#=#= Current cib after: Set a transient (fail-count) node attribute =#=#=#=
	<cib epoch="11" num_updates="1" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1">
	<nvpair id="status-node1-fail-count-foo" name="fail-count-foo" value="3"/>
	</instance_attributes>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Set a transient (fail-count) node attribute - OK (0) =#=#=#=
	* Passed: crm_attribute - Set a transient (fail-count) node attribute
	=#=#=#= Begin test: Query a fail count =#=#=#=
	scope=status name=fail-count-foo value=3
	=#=#=#= Current cib after: Query a fail count =#=#=#=
	<cib epoch="11" num_updates="1" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1">
	<nvpair id="status-node1-fail-count-foo" name="fail-count-foo" value="3"/>
	</instance_attributes>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Query a fail count - OK (0) =#=#=#=
	* Passed: crm_failcount - Query a fail count
	=#=#=#= Begin test: Delete a transient (fail-count) node attribute =#=#=#=
	Deleted status attribute: id=status-node1-fail-count-foo name=fail-count-foo

	=#=#=#= Current cib after: Delete a transient (fail-count) node attribute =#=#=#=
	<cib epoch="11" num_updates="2" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Delete a transient (fail-count) node attribute - OK (0) =#=#=#=
	* Passed: crm_attribute - Delete a transient (fail-count) node attribute
	=#=#=#= Begin test: Digest calculation =#=#=#=
	Digest: =#=#=#= Current cib after: Digest calculation =#=#=#=
	<cib epoch="11" num_updates="2" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Digest calculation - OK (0) =#=#=#=
	* Passed: cibadmin - Digest calculation
	=#=#=#= Begin test: Replace operation should fail =#=#=#=
	Call failed: Update was older than existing configuration
	=#=#=#= Current cib after: Replace operation should fail =#=#=#=
	<cib epoch="11" num_updates="2" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Replace operation should fail - Update was older than existing configuration (103) =#=#=#=
	* Passed: cibadmin - Replace operation should fail
	=#=#=#= Begin test: Default standby value =#=#=#=
	-Error performing operation: No such device or address
	scope=status name=standby value=off
	=#=#=#= Current cib after: Default standby value =#=#=#=
	<cib epoch="11" num_updates="2" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Default standby value - OK (0) =#=#=#=
	* Passed: crm_standby - Default standby value
	=#=#=#= Begin test: Set standby status =#=#=#=
	=#=#=#= Current cib after: Set standby status =#=#=#=
	<cib epoch="12" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	<nvpair id="nodes-node1-standby" name="standby" value="true"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Set standby status - OK (0) =#=#=#=
	* Passed: crm_standby - Set standby status
	=#=#=#= Begin test: Query standby value =#=#=#=
	scope=nodes name=standby value=true
	=#=#=#= Current cib after: Query standby value =#=#=#=
	<cib epoch="12" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	<nvpair id="nodes-node1-standby" name="standby" value="true"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Query standby value - OK (0) =#=#=#=
	* Passed: crm_standby - Query standby value
	=#=#=#= Begin test: Delete standby value =#=#=#=
	Deleted nodes attribute: id=nodes-node1-standby name=standby

	=#=#=#= Current cib after: Delete standby value =#=#=#=
	<cib epoch="13" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Delete standby value - OK (0) =#=#=#=
	* Passed: crm_standby - Delete standby value
	=#=#=#= Begin test: Create a resource =#=#=#=
	=#=#=#= Current cib after: Create a resource =#=#=#=
	<cib epoch="14" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Create a resource - OK (0) =#=#=#=
	* Passed: cibadmin - Create a resource
	=#=#=#= Begin test: Create a resource meta attribute =#=#=#=

	Set 'dummy' option: id=dummy-meta_attributes-is-managed set=dummy-meta_attributes name=is-managed=false
	=#=#=#= Current cib after: Create a resource meta attribute =#=#=#=
	<cib epoch="15" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Create a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Create a resource meta attribute
	=#=#=#= Begin test: Query a resource meta attribute =#=#=#=
	false
	=#=#=#= Current cib after: Query a resource meta attribute =#=#=#=
	<cib epoch="15" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Query a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Query a resource meta attribute
	=#=#=#= Begin test: Remove a resource meta attribute =#=#=#=
	Deleted 'dummy' option: id=dummy-meta_attributes-is-managed name=is-managed
	=#=#=#= Current cib after: Remove a resource meta attribute =#=#=#=
	<cib epoch="16" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Remove a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Remove a resource meta attribute
	=#=#=#= Begin test: Create a resource attribute =#=#=#=

	Set 'dummy' option: id=dummy-instance_attributes-delay set=dummy-instance_attributes name=delay=10s
	=#=#=#= Current cib after: Create a resource attribute =#=#=#=
	<cib epoch="17" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Create a resource attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Create a resource attribute
	=#=#=#= Begin test: List the configured resources =#=#=#=
	dummy (ocf::pacemaker:Dummy): Stopped
	=#=#=#= Current cib after: List the configured resources =#=#=#=
	<cib epoch="17" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: List the configured resources - OK (0) =#=#=#=
	* Passed: crm_resource - List the configured resources
	=#=#=#= Begin test: Require a destination when migrating a resource that is stopped =#=#=#=
	Resource 'dummy' not moved: active in 0 locations.
	You can prevent 'dummy' from running on a specific location with: --ban --node <name>
	Error performing operation: Invalid argument
	=#=#=#= Current cib after: Require a destination when migrating a resource that is stopped =#=#=#=
	<cib epoch="17" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Require a destination when migrating a resource that is stopped - Incorrect usage (64) =#=#=#=
	* Passed: crm_resource - Require a destination when migrating a resource that is stopped
	=#=#=#= Begin test: Don't support migration to non-existent locations =#=#=#=
	-Error performing operation: node 'i.dont.exist' is unknown
	-Error performing operation: No such device or address
	+Error performing operation: Node not found
	=#=#=#= Current cib after: Don't support migration to non-existent locations =#=#=#=
	<cib epoch="17" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Don't support migration to non-existent locations - No such object (105) =#=#=#=
	* Passed: crm_resource - Don't support migration to non-existent locations
	=#=#=#= Begin test: Create a fencing resource =#=#=#=
	=#=#=#= Current cib after: Create a fencing resource =#=#=#=
	<cib epoch="18" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Create a fencing resource - OK (0) =#=#=#=
	* Passed: cibadmin - Create a fencing resource
	=#=#=#= Begin test: Bring resources online =#=#=#=

	Current cluster status:
	Online: [ node1 ]

	dummy (ocf::pacemaker:Dummy): Stopped
	Fence (stonith:fence_true): Stopped

	Transition Summary:
	* Start dummy ( node1 )
	* Start Fence ( node1 )

	Executing cluster transition:
	* Resource action: dummy monitor on node1
	* Resource action: Fence monitor on node1
	* Resource action: dummy start on node1
	* Resource action: Fence start on node1

	Revised cluster status:
	Online: [ node1 ]

	dummy (ocf::pacemaker:Dummy): Started node1
	Fence (stonith:fence_true): Started node1

	=#=#=#= Current cib after: Bring resources online =#=#=#=
	<cib epoch="18" num_updates="4" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Bring resources online - OK (0) =#=#=#=
	* Passed: crm_simulate - Bring resources online
	=#=#=#= Begin test: Try to move a resource to its existing location =#=#=#=
	-Error performing operation: dummy is already active on node1
	-Error performing operation: File exists
	+Error performing operation: Situation already as requested
	=#=#=#= Current cib after: Try to move a resource to its existing location =#=#=#=
	<cib epoch="18" num_updates="4" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Try to move a resource to its existing location - Requested item already exists (108) =#=#=#=
	* Passed: crm_resource - Try to move a resource to its existing location
	=#=#=#= Begin test: Move a resource from its existing location =#=#=#=
	WARNING: Creating rsc_location constraint 'cli-ban-dummy-on-node1' with a score of -INFINITY for resource dummy on node1.
	This will prevent dummy from running on node1 until the constraint is removed using the 'crm_resource --clear' command or manually with cibadmin
	This will be the case even if node1 is the last node in the cluster
	This message can be disabled with --quiet
	=#=#=#= Current cib after: Move a resource from its existing location =#=#=#=
	<cib epoch="19" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints>
	<rsc_location id="cli-ban-dummy-on-node1" rsc="dummy" role="Started" node="node1" score="-INFINITY"/>
	</constraints>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Move a resource from its existing location - OK (0) =#=#=#=
	* Passed: crm_resource - Move a resource from its existing location
	=#=#=#= Begin test: Clear out constraints generated by --move =#=#=#=
	=#=#=#= Current cib after: Clear out constraints generated by --move =#=#=#=
	<cib epoch="20" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Clear out constraints generated by --move - OK (0) =#=#=#=
	* Passed: crm_resource - Clear out constraints generated by --move
	=#=#=#= Begin test: Default ticket granted state =#=#=#=
	false
	=#=#=#= Current cib after: Default ticket granted state =#=#=#=
	<cib epoch="20" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Default ticket granted state - OK (0) =#=#=#=
	* Passed: crm_ticket - Default ticket granted state
	=#=#=#= Begin test: Set ticket granted state =#=#=#=
	=#=#=#= Current cib after: Set ticket granted state =#=#=#=
	<cib epoch="20" num_updates="1" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA" granted="false"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Set ticket granted state - OK (0) =#=#=#=
	* Passed: crm_ticket - Set ticket granted state
	=#=#=#= Begin test: Query ticket granted state =#=#=#=
	false
	=#=#=#= Current cib after: Query ticket granted state =#=#=#=
	<cib epoch="20" num_updates="1" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA" granted="false"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Query ticket granted state - OK (0) =#=#=#=
	* Passed: crm_ticket - Query ticket granted state
	=#=#=#= Begin test: Delete ticket granted state =#=#=#=
	=#=#=#= Current cib after: Delete ticket granted state =#=#=#=
	<cib epoch="20" num_updates="2" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Delete ticket granted state - OK (0) =#=#=#=
	* Passed: crm_ticket - Delete ticket granted state
	=#=#=#= Begin test: Make a ticket standby =#=#=#=
	=#=#=#= Current cib after: Make a ticket standby =#=#=#=
	<cib epoch="20" num_updates="3" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA" standby="true"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Make a ticket standby - OK (0) =#=#=#=
	* Passed: crm_ticket - Make a ticket standby
	=#=#=#= Begin test: Query ticket standby state =#=#=#=
	true
	=#=#=#= Current cib after: Query ticket standby state =#=#=#=
	<cib epoch="20" num_updates="3" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA" standby="true"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Query ticket standby state - OK (0) =#=#=#=
	* Passed: crm_ticket - Query ticket standby state
	=#=#=#= Begin test: Activate a ticket =#=#=#=
	=#=#=#= Current cib after: Activate a ticket =#=#=#=
	<cib epoch="20" num_updates="4" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA" standby="false"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Activate a ticket - OK (0) =#=#=#=
	* Passed: crm_ticket - Activate a ticket
	=#=#=#= Begin test: Delete ticket standby state =#=#=#=
	=#=#=#= Current cib after: Delete ticket standby state =#=#=#=
	<cib epoch="20" num_updates="5" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Delete ticket standby state - OK (0) =#=#=#=
	* Passed: crm_ticket - Delete ticket standby state
	=#=#=#= Begin test: Ban a resource on unknown node =#=#=#=
	-Error performing operation: node 'host1' is unknown
	-Error performing operation: No such device or address
	+Error performing operation: Node not found
	=#=#=#= Current cib after: Ban a resource on unknown node =#=#=#=
	<cib epoch="20" num_updates="5" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Ban a resource on unknown node - No such object (105) =#=#=#=
	* Passed: crm_resource - Ban a resource on unknown node
	=#=#=#= Begin test: Create two more nodes and bring them online =#=#=#=

	Current cluster status:
	Online: [ node1 ]

	dummy (ocf::pacemaker:Dummy): Started node1
	Fence (stonith:fence_true): Started node1

	Performing requested modifications
	+ Bringing node node2 online
	+ Bringing node node3 online

	Transition Summary:
	* Move Fence ( node1 -> node2 )

	Executing cluster transition:
	* Resource action: dummy monitor on node3
	* Resource action: dummy monitor on node2
	* Resource action: Fence stop on node1
	* Resource action: Fence monitor on node3
	* Resource action: Fence monitor on node2
	* Pseudo action: all_stopped
	* Resource action: Fence start on node2

	Revised cluster status:
	Online: [ node1 node2 node3 ]

	dummy (ocf::pacemaker:Dummy): Started node1
	Fence (stonith:fence_true): Started node2

	=#=#=#= Current cib after: Create two more nodes and bring them online =#=#=#=
	<cib epoch="22" num_updates="8" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	<node_state id="node2" uname="node2" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node2">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<node_state id="node3" uname="node3" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node3">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Create two more nodes and bring them online - OK (0) =#=#=#=
	* Passed: crm_simulate - Create two more nodes and bring them online
	=#=#=#= Begin test: Ban dummy from node1 =#=#=#=
	WARNING: Creating rsc_location constraint 'cli-ban-dummy-on-node1' with a score of -INFINITY for resource dummy on node1.
	This will prevent dummy from running on node1 until the constraint is removed using the 'crm_resource --clear' command or manually with cibadmin
	This will be the case even if node1 is the last node in the cluster
	This message can be disabled with --quiet
	=#=#=#= Current cib after: Ban dummy from node1 =#=#=#=
	<cib epoch="23" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints>
	<rsc_location id="cli-ban-dummy-on-node1" rsc="dummy" role="Started" node="node1" score="-INFINITY"/>
	</constraints>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	<node_state id="node2" uname="node2" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node2">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<node_state id="node3" uname="node3" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node3">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Ban dummy from node1 - OK (0) =#=#=#=
	* Passed: crm_resource - Ban dummy from node1
	=#=#=#= Begin test: Ban dummy from node2 =#=#=#=
	WARNING: Creating rsc_location constraint 'cli-ban-dummy-on-node2' with a score of -INFINITY for resource dummy on node2.
	This will prevent dummy from running on node2 until the constraint is removed using the 'crm_resource --clear' command or manually with cibadmin
	This will be the case even if node2 is the last node in the cluster
	This message can be disabled with --quiet
	=#=#=#= Current cib after: Ban dummy from node2 =#=#=#=
	<cib epoch="24" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints>
	<rsc_location id="cli-ban-dummy-on-node1" rsc="dummy" role="Started" node="node1" score="-INFINITY"/>
	<rsc_location id="cli-ban-dummy-on-node2" rsc="dummy" role="Started" node="node2" score="-INFINITY"/>
	</constraints>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	<node_state id="node2" uname="node2" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node2">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<node_state id="node3" uname="node3" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node3">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Ban dummy from node2 - OK (0) =#=#=#=
	* Passed: crm_resource - Ban dummy from node2
	=#=#=#= Begin test: Relocate resources due to ban =#=#=#=

	Current cluster status:
	Online: [ node1 node2 node3 ]

	dummy (ocf::pacemaker:Dummy): Started node1
	Fence (stonith:fence_true): Started node2

	Transition Summary:
	* Move dummy ( node1 -> node3 )

	Executing cluster transition:
	* Resource action: dummy stop on node1
	* Pseudo action: all_stopped
	* Resource action: dummy start on node3

	Revised cluster status:
	Online: [ node1 node2 node3 ]

	dummy (ocf::pacemaker:Dummy): Started node3
	Fence (stonith:fence_true): Started node2

	=#=#=#= Current cib after: Relocate resources due to ban =#=#=#=
	<cib epoch="24" num_updates="2" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints>
	<rsc_location id="cli-ban-dummy-on-node1" rsc="dummy" role="Started" node="node1" score="-INFINITY"/>
	<rsc_location id="cli-ban-dummy-on-node2" rsc="dummy" role="Started" node="node2" score="-INFINITY"/>
	</constraints>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	<node_state id="node2" uname="node2" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node2">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<node_state id="node3" uname="node3" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node3">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Relocate resources due to ban - OK (0) =#=#=#=
	* Passed: crm_simulate - Relocate resources due to ban
	=#=#=#= Begin test: Move dummy to node1 =#=#=#=
	=#=#=#= Current cib after: Move dummy to node1 =#=#=#=
	<cib epoch="26" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints>
	<rsc_location id="cli-ban-dummy-on-node2" rsc="dummy" role="Started" node="node2" score="-INFINITY"/>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	<node_state id="node2" uname="node2" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node2">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<node_state id="node3" uname="node3" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node3">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Move dummy to node1 - OK (0) =#=#=#=
	* Passed: crm_resource - Move dummy to node1
	=#=#=#= Begin test: Clear implicit constraints for dummy on node2 =#=#=#=
	=#=#=#= Current cib after: Clear implicit constraints for dummy on node2 =#=#=#=
	<cib epoch="27" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	<node_state id="node2" uname="node2" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node2">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<node_state id="node3" uname="node3" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node3">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Clear implicit constraints for dummy on node2 - OK (0) =#=#=#=
	* Passed: crm_resource - Clear implicit constraints for dummy on node2
	=#=#=#= Begin test: Drop the status section =#=#=#=
	=#=#=#= End test: Drop the status section - OK (0) =#=#=#=
	* Passed: cibadmin - Drop the status section
	=#=#=#= Begin test: Create a clone =#=#=#=
	=#=#=#= End test: Create a clone - OK (0) =#=#=#=
	* Passed: cibadmin - Create a clone
	=#=#=#= Begin test: Create a resource meta attribute =#=#=#=

	Performing update of 'is-managed' on 'test-clone', the parent of 'test-primitive'
	Set 'test-clone' option: id=test-clone-meta_attributes-is-managed set=test-clone-meta_attributes name=is-managed=false
	=#=#=#= Current cib after: Create a resource meta attribute =#=#=#=
	<cib epoch="29" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy"/>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Create a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Create a resource meta attribute
	=#=#=#= Begin test: Create a resource meta attribute in the primitive =#=#=#=
	Set 'test-primitive' option: id=test-primitive-meta_attributes-is-managed set=test-primitive-meta_attributes name=is-managed=false
	=#=#=#= Current cib after: Create a resource meta attribute in the primitive =#=#=#=
	<cib epoch="30" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Create a resource meta attribute in the primitive - OK (0) =#=#=#=
	* Passed: crm_resource - Create a resource meta attribute in the primitive
	=#=#=#= Begin test: Update resource meta attribute with duplicates =#=#=#=
	Multiple attributes match name=is-managed
	Value: false (id=test-primitive-meta_attributes-is-managed)
	Value: false (id=test-clone-meta_attributes-is-managed)

	A value for 'is-managed' already exists in child 'test-primitive', performing update on that instead of 'test-clone'
	Set 'test-primitive' option: id=test-primitive-meta_attributes-is-managed name=is-managed=true
	=#=#=#= Current cib after: Update resource meta attribute with duplicates =#=#=#=
	<cib epoch="31" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Update resource meta attribute with duplicates - OK (0) =#=#=#=
	* Passed: crm_resource - Update resource meta attribute with duplicates
	=#=#=#= Begin test: Update resource meta attribute with duplicates (force clone) =#=#=#=
	Set 'test-clone' option: id=test-clone-meta_attributes-is-managed name=is-managed=true
	=#=#=#= Current cib after: Update resource meta attribute with duplicates (force clone) =#=#=#=
	<cib epoch="32" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Update resource meta attribute with duplicates (force clone) - OK (0) =#=#=#=
	* Passed: crm_resource - Update resource meta attribute with duplicates (force clone)
	=#=#=#= Begin test: Update child resource meta attribute with duplicates =#=#=#=
	Multiple attributes match name=is-managed
	Value: true (id=test-primitive-meta_attributes-is-managed)
	Value: true (id=test-clone-meta_attributes-is-managed)

	Set 'test-primitive' option: id=test-primitive-meta_attributes-is-managed name=is-managed=false
	=#=#=#= Current cib after: Update child resource meta attribute with duplicates =#=#=#=
	<cib epoch="33" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Update child resource meta attribute with duplicates - OK (0) =#=#=#=
	* Passed: crm_resource - Update child resource meta attribute with duplicates
	=#=#=#= Begin test: Delete resource meta attribute with duplicates =#=#=#=
	Multiple attributes match name=is-managed
	Value: false (id=test-primitive-meta_attributes-is-managed)
	Value: true (id=test-clone-meta_attributes-is-managed)

	A value for 'is-managed' already exists in child 'test-primitive', performing delete on that instead of 'test-clone'
	Deleted 'test-primitive' option: id=test-primitive-meta_attributes-is-managed name=is-managed
	=#=#=#= Current cib after: Delete resource meta attribute with duplicates =#=#=#=
	<cib epoch="34" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes"/>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Delete resource meta attribute with duplicates - OK (0) =#=#=#=
	* Passed: crm_resource - Delete resource meta attribute with duplicates
	=#=#=#= Begin test: Delete resource meta attribute in parent =#=#=#=
	Performing delete of 'is-managed' on 'test-clone', the parent of 'test-primitive'
	Deleted 'test-clone' option: id=test-clone-meta_attributes-is-managed name=is-managed
	=#=#=#= Current cib after: Delete resource meta attribute in parent =#=#=#=
	<cib epoch="35" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes"/>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes"/>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Delete resource meta attribute in parent - OK (0) =#=#=#=
	* Passed: crm_resource - Delete resource meta attribute in parent
	=#=#=#= Begin test: Create a resource meta attribute in the primitive =#=#=#=
	Set 'test-primitive' option: id=test-primitive-meta_attributes-is-managed set=test-primitive-meta_attributes name=is-managed=false
	=#=#=#= Current cib after: Create a resource meta attribute in the primitive =#=#=#=
	<cib epoch="36" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes"/>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Create a resource meta attribute in the primitive - OK (0) =#=#=#=
	* Passed: crm_resource - Create a resource meta attribute in the primitive
	=#=#=#= Begin test: Update existing resource meta attribute =#=#=#=
	A value for 'is-managed' already exists in child 'test-primitive', performing update on that instead of 'test-clone'
	Set 'test-primitive' option: id=test-primitive-meta_attributes-is-managed name=is-managed=true
	=#=#=#= Current cib after: Update existing resource meta attribute =#=#=#=
	<cib epoch="37" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes"/>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Update existing resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Update existing resource meta attribute
	=#=#=#= Begin test: Create a resource meta attribute in the parent =#=#=#=
	Set 'test-clone' option: id=test-clone-meta_attributes-is-managed set=test-clone-meta_attributes name=is-managed=true
	=#=#=#= Current cib after: Create a resource meta attribute in the parent =#=#=#=
	<cib epoch="38" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Create a resource meta attribute in the parent - OK (0) =#=#=#=
	* Passed: crm_resource - Create a resource meta attribute in the parent
	=#=#=#= Begin test: Copy resources =#=#=#=
	=#=#=#= End test: Copy resources - OK (0) =#=#=#=
	* Passed: cibadmin - Copy resources
	=#=#=#= Begin test: Delete resource paremt meta attribute (force) =#=#=#=
	Deleted 'test-clone' option: id=test-clone-meta_attributes-is-managed name=is-managed
	=#=#=#= Current cib after: Delete resource paremt meta attribute (force) =#=#=#=
	<cib epoch="39" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes"/>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Delete resource paremt meta attribute (force) - OK (0) =#=#=#=
	* Passed: crm_resource - Delete resource paremt meta attribute (force)
	=#=#=#= Begin test: Restore duplicates =#=#=#=
	=#=#=#= Current cib after: Restore duplicates =#=#=#=
	<cib epoch="40" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Restore duplicates - OK (0) =#=#=#=
	* Passed: cibadmin - Restore duplicates
	=#=#=#= Begin test: Delete resource child meta attribute =#=#=#=
	Multiple attributes match name=is-managed
	Value: true (id=test-primitive-meta_attributes-is-managed)
	Value: true (id=test-clone-meta_attributes-is-managed)

	Deleted 'test-primitive' option: id=test-primitive-meta_attributes-is-managed name=is-managed
	=#=#=#= Current cib after: Delete resource child meta attribute =#=#=#=
	<cib epoch="41" num_updates="0" admin_epoch="1">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes"/>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Delete resource child meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Delete resource child meta attribute
	diff --git a/cts/cts-cli.in b/cts/cts-cli.in
	index 3a97f2e55f..7abe4bbf2c 100755
	--- a/cts/cts-cli.in
	+++ b/cts/cts-cli.in
	@@ -1,910 +1,914 @@
	#!@BASH_PATH@
	#
	# Copyright 2008-2018 Andrew Beekhof <andrew@beekhof.net>
	#
	# This source code is licensed under the GNU General Public License version 2
	# or later (GPLv2+) WITHOUT ANY WARRANTY.
	#

	+#
	+# Note on portable usage of sed: GNU/POSIX/*BSD sed have a limited subset of
	+# compatible functionality. Do not use the -i option, alternation (\\|),
	+# \0, or character sequences such as \n or \s.
	+#
	+
	USAGE_TEXT="Usage: cts-cli [<options>]
	Options:
	--help Display this text, then exit
	-V, --verbose Display any differences from expected output
	-t 'TEST [...]' Run only specified tests (default: 'dates tools acls validity')
	-p DIR Look for executables in DIR (may be specified multiple times)
	-v, --valgrind Run all commands under valgrind
	-s Save actual output as expected output"

	# If readlink supports -e (i.e. GNU), use it
	readlink -e / >/dev/null 2>/dev/null
	if [ $? -eq 0 ]; then
	test_home="$(dirname $(readlink -e $0))"
	else
	test_home="$(dirname $0)"
	fi

	: ${shadow=cts-cli}
	shadow_dir=$(mktemp -d ${TMPDIR:-/tmp}/cts-cli.shadow.XXXXXXXXXX)
	num_errors=0
	num_passed=0
	GREP_OPTIONS=
	verbose=0
	tests="dates tools acls validity"
	do_save=0
	VALGRIND_CMD=
	VALGRIND_OPTS="
	-q
	--gen-suppressions=all
	--show-reachable=no
	--leak-check=full
	--trace-children=no
	--time-stamp=yes
	--num-callers=20
	--suppressions=$test_home/valgrind-pcmk.suppressions
	"

	# These constants must track crm_exit_t values
	CRM_EX_OK=0
	CRM_EX_ERROR=1
	CRM_EX_INSUFFICIENT_PRIV=4
	CRM_EX_USAGE=64
	CRM_EX_CONFIG=78
	CRM_EX_OLD=103
	CRM_EX_NOSUCH=105
	CRM_EX_UNSAFE=107
	CRM_EX_EXISTS=108
	CRM_EX_MULTIPLE=109

	function test_assert() {
	target=$1; shift
	cib=$1; shift
	app=`echo "$cmd" \| sed 's/\ .*//'`
	printf "* Running: $app - $desc\n" 1>&2

	printf "=#=#=#= Begin test: $desc =#=#=#=\n"
	eval $VALGRIND_CMD $cmd 2>&1
	rc=$?

	if [ x$cib != x0 ]; then
	printf "=#=#=#= Current cib after: $desc =#=#=#=\n"
	CIB_user=root cibadmin -Q
	fi

	printf "=#=#=#= End test: $desc - $(crm_error --exit $rc) ($rc) =#=#=#=\n"

	if [ $rc -ne $target ]; then
	num_errors=$(( $num_errors + 1 ))
	printf "* Failed (rc=%.3d): %-14s - %s\n" $rc $app "$desc"
	printf "* Failed (rc=%.3d): %-14s - %s\n" $rc $app "$desc (`which $app`)" 1>&2
	return
	exit $CRM_EX_ERROR
	else
	printf "* Passed: %-14s - %s\n" $app "$desc"
	num_passed=$(( $num_passed + 1 ))
	fi
	}

	function test_tools() {
	local TMPXML=$(mktemp ${TMPDIR:-/tmp}/cts-cli.tools.xml.XXXXXXXXXX)
	local TMPORIG=$(mktemp ${TMPDIR:-/tmp}/cts-cli.tools.existing.xml.XXXXXXXXXX)
	export CIB_shadow_dir="${shadow_dir}"

	$VALGRIND_CMD crm_shadow --batch --force --create-empty $shadow 2>&1
	export CIB_shadow=$shadow

	desc="Validate CIB"
	cmd="cibadmin -Q"
	test_assert $CRM_EX_OK

	desc="Configure something before erasing"
	cmd="crm_attribute -n cluster-delay -v 60s"
	test_assert $CRM_EX_OK

	desc="Require --force for CIB erasure"
	cmd="cibadmin -E"
	test_assert $CRM_EX_UNSAFE

	desc="Allow CIB erasure with --force"
	cmd="cibadmin -E --force"
	test_assert $CRM_EX_OK

	desc="Query CIB"
	cmd="cibadmin -Q > $TMPORIG"
	test_assert $CRM_EX_OK

	desc="Set cluster option"
	cmd="crm_attribute -n cluster-delay -v 60s"
	test_assert $CRM_EX_OK

	desc="Query new cluster option"
	cmd="cibadmin -Q -o crm_config \| grep cib-bootstrap-options-cluster-delay"
	test_assert $CRM_EX_OK

	desc="Query cluster options"
	cmd="cibadmin -Q -o crm_config > $TMPXML"
	test_assert $CRM_EX_OK

	desc="Set no-quorum policy"
	cmd="crm_attribute -n no-quorum-policy -v ignore"
	test_assert $CRM_EX_OK

	desc="Delete nvpair"
	cmd="cibadmin -D -o crm_config --xml-text '<nvpair id=\"cib-bootstrap-options-cluster-delay\"/>'"
	test_assert $CRM_EX_OK

	desc="Create operation should fail"
	cmd="cibadmin -C -o crm_config --xml-file $TMPXML"
	test_assert $CRM_EX_EXISTS

	desc="Modify cluster options section"
	cmd="cibadmin -M -o crm_config --xml-file $TMPXML"
	test_assert $CRM_EX_OK

	desc="Query updated cluster option"
	cmd="cibadmin -Q -o crm_config \| grep cib-bootstrap-options-cluster-delay"
	test_assert $CRM_EX_OK

	desc="Set duplicate cluster option"
	cmd="crm_attribute -n cluster-delay -v 40s -s duplicate"
	test_assert $CRM_EX_OK

	desc="Setting multiply defined cluster option should fail"
	cmd="crm_attribute -n cluster-delay -v 30s"
	test_assert $CRM_EX_MULTIPLE

	desc="Set cluster option with -s"
	cmd="crm_attribute -n cluster-delay -v 30s -s duplicate"
	test_assert $CRM_EX_OK

	desc="Delete cluster option with -i"
	cmd="crm_attribute -n cluster-delay -D -i cib-bootstrap-options-cluster-delay"
	test_assert $CRM_EX_OK

	desc="Create node1 and bring it online"
	cmd="crm_simulate --live-check --in-place --node-up=node1"
	test_assert $CRM_EX_OK

	desc="Create node attribute"
	cmd="crm_attribute -n ram -v 1024M -N node1 -t nodes"
	test_assert $CRM_EX_OK

	desc="Query new node attribute"
	cmd="cibadmin -Q -o nodes \| grep node1-ram"
	test_assert $CRM_EX_OK

	desc="Set a transient (fail-count) node attribute"
	cmd="crm_attribute -n fail-count-foo -v 3 -N node1 -t status"
	test_assert $CRM_EX_OK

	desc="Query a fail count"
	cmd="crm_failcount --query -r foo -N node1"
	test_assert $CRM_EX_OK

	desc="Delete a transient (fail-count) node attribute"
	cmd="crm_attribute -n fail-count-foo -D -N node1 -t status"
	test_assert $CRM_EX_OK

	desc="Digest calculation"
	cmd="cibadmin -Q \| cibadmin -5 -p 2>&1 > /dev/null"
	test_assert $CRM_EX_OK

	# This update will fail because it has version numbers
	desc="Replace operation should fail"
	cmd="cibadmin -R --xml-file $TMPORIG"
	test_assert $CRM_EX_OLD

	desc="Default standby value"
	cmd="crm_standby -N node1 -G"
	test_assert $CRM_EX_OK

	desc="Set standby status"
	cmd="crm_standby -N node1 -v true"
	test_assert $CRM_EX_OK

	desc="Query standby value"
	cmd="crm_standby -N node1 -G"
	test_assert $CRM_EX_OK

	desc="Delete standby value"
	cmd="crm_standby -N node1 -D"
	test_assert $CRM_EX_OK

	desc="Create a resource"
	cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
	test_assert $CRM_EX_OK

	desc="Create a resource meta attribute"
	cmd="crm_resource -r dummy --meta -p is-managed -v false"
	test_assert $CRM_EX_OK

	desc="Query a resource meta attribute"
	cmd="crm_resource -r dummy --meta -g is-managed"
	test_assert $CRM_EX_OK

	desc="Remove a resource meta attribute"
	cmd="crm_resource -r dummy --meta -d is-managed"
	test_assert $CRM_EX_OK

	desc="Create a resource attribute"
	cmd="crm_resource -r dummy -p delay -v 10s"
	test_assert $CRM_EX_OK

	desc="List the configured resources"
	cmd="crm_resource -L"
	test_assert $CRM_EX_OK

	desc="Require a destination when migrating a resource that is stopped"
	cmd="crm_resource -r dummy -M"
	test_assert $CRM_EX_USAGE

	desc="Don't support migration to non-existent locations"
	cmd="crm_resource -r dummy -M -N i.dont.exist"
	test_assert $CRM_EX_NOSUCH

	desc="Create a fencing resource"
	cmd="cibadmin -C -o resources --xml-text '<primitive id=\"Fence\" class=\"stonith\" type=\"fence_true\"/>'"
	test_assert $CRM_EX_OK

	desc="Bring resources online"
	cmd="crm_simulate --live-check --in-place -S"
	test_assert $CRM_EX_OK

	desc="Try to move a resource to its existing location"
	cmd="crm_resource -r dummy --move --host node1"
	test_assert $CRM_EX_EXISTS

	desc="Move a resource from its existing location"
	cmd="crm_resource -r dummy --move"
	test_assert $CRM_EX_OK

	desc="Clear out constraints generated by --move"
	cmd="crm_resource -r dummy --clear"
	test_assert $CRM_EX_OK

	desc="Default ticket granted state"
	cmd="crm_ticket -t ticketA -G granted -d false"
	test_assert $CRM_EX_OK

	desc="Set ticket granted state"
	cmd="crm_ticket -t ticketA -r --force"
	test_assert $CRM_EX_OK

	desc="Query ticket granted state"
	cmd="crm_ticket -t ticketA -G granted"
	test_assert $CRM_EX_OK

	desc="Delete ticket granted state"
	cmd="crm_ticket -t ticketA -D granted --force"
	test_assert $CRM_EX_OK

	desc="Make a ticket standby"
	cmd="crm_ticket -t ticketA -s"
	test_assert $CRM_EX_OK

	desc="Query ticket standby state"
	cmd="crm_ticket -t ticketA -G standby"
	test_assert $CRM_EX_OK

	desc="Activate a ticket"
	cmd="crm_ticket -t ticketA -a"
	test_assert $CRM_EX_OK

	desc="Delete ticket standby state"
	cmd="crm_ticket -t ticketA -D standby"
	test_assert $CRM_EX_OK

	desc="Ban a resource on unknown node"
	cmd="crm_resource -r dummy -B -N host1"
	test_assert $CRM_EX_NOSUCH

	desc="Create two more nodes and bring them online"
	cmd="crm_simulate --live-check --in-place --node-up=node2 --node-up=node3"
	test_assert $CRM_EX_OK

	desc="Ban dummy from node1"
	cmd="crm_resource -r dummy -B -N node1"
	test_assert $CRM_EX_OK

	desc="Ban dummy from node2"
	cmd="crm_resource -r dummy -B -N node2"
	test_assert $CRM_EX_OK

	desc="Relocate resources due to ban"
	cmd="crm_simulate --live-check --in-place -S"
	test_assert $CRM_EX_OK

	desc="Move dummy to node1"
	cmd="crm_resource -r dummy -M -N node1"
	test_assert $CRM_EX_OK

	desc="Clear implicit constraints for dummy on node2"
	cmd="crm_resource -r dummy -U -N node2"
	test_assert $CRM_EX_OK

	desc="Drop the status section"
	cmd="cibadmin -R -o status --xml-text '<status/>'"
	test_assert $CRM_EX_OK 0

	desc="Create a clone"
	cmd="cibadmin -C -o resources --xml-text '<clone id=\"test-clone\"><primitive id=\"test-primitive\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/></clone>'"
	test_assert $CRM_EX_OK 0

	desc="Create a resource meta attribute"
	cmd="crm_resource -r test-primitive --meta -p is-managed -v false"
	test_assert $CRM_EX_OK

	desc="Create a resource meta attribute in the primitive"
	cmd="crm_resource -r test-primitive --meta -p is-managed -v false --force"
	test_assert $CRM_EX_OK

	desc="Update resource meta attribute with duplicates"
	cmd="crm_resource -r test-clone --meta -p is-managed -v true"
	test_assert $CRM_EX_OK

	desc="Update resource meta attribute with duplicates (force clone)"
	cmd="crm_resource -r test-clone --meta -p is-managed -v true --force"
	test_assert $CRM_EX_OK

	desc="Update child resource meta attribute with duplicates"
	cmd="crm_resource -r test-primitive --meta -p is-managed -v false"
	test_assert $CRM_EX_OK

	desc="Delete resource meta attribute with duplicates"
	cmd="crm_resource -r test-clone --meta -d is-managed"
	test_assert $CRM_EX_OK

	desc="Delete resource meta attribute in parent"
	cmd="crm_resource -r test-primitive --meta -d is-managed"
	test_assert $CRM_EX_OK

	desc="Create a resource meta attribute in the primitive"
	cmd="crm_resource -r test-primitive --meta -p is-managed -v false --force"
	test_assert $CRM_EX_OK

	desc="Update existing resource meta attribute"
	cmd="crm_resource -r test-clone --meta -p is-managed -v true"
	test_assert $CRM_EX_OK

	desc="Create a resource meta attribute in the parent"
	cmd="crm_resource -r test-clone --meta -p is-managed -v true --force"
	test_assert $CRM_EX_OK

	desc="Copy resources"
	cmd="cibadmin -Q -o resources > $TMPXML"
	test_assert $CRM_EX_OK 0

	desc="Delete resource paremt meta attribute (force)"
	cmd="crm_resource -r test-clone --meta -d is-managed --force"
	test_assert $CRM_EX_OK

	desc="Restore duplicates"
	cmd="cibadmin -R -o resources --xml-file $TMPXML"
	test_assert $CRM_EX_OK

	desc="Delete resource child meta attribute"
	cmd="crm_resource -r test-primitive --meta -d is-managed"
	test_assert $CRM_EX_OK

	unset CIB_shadow_dir
	rm -f "$TMPXML" "$TMPORIG"
	}

	function test_dates() {
	desc="2014-01-01 00:30:00 - 1 Hour"
	cmd="iso8601 -d '2014-01-01 00:30:00Z' -D P-1H -E '2013-12-31 23:30:00Z'"
	test_assert $CRM_EX_OK 0

	for y in 06 07 08 09 10 11 12 13 14 15 16 17 18; do
	desc="20$y-W01-7"
	cmd="iso8601 -d '20$y-W01-7 00Z'"
	test_assert $CRM_EX_OK 0

	desc="20$y-W01-7 - round-trip"
	cmd="iso8601 -d '20$y-W01-7 00Z' -W -E '20$y-W01-7 00:00:00Z'"
	test_assert $CRM_EX_OK 0

	desc="20$y-W01-1"
	cmd="iso8601 -d '20$y-W01-1 00Z'"
	test_assert $CRM_EX_OK 0

	desc="20$y-W01-1 - round-trip"
	cmd="iso8601 -d '20$y-W01-1 00Z' -W -E '20$y-W01-1 00:00:00Z'"
	test_assert $CRM_EX_OK 0
	done

	desc="2009-W53-07"
	cmd="iso8601 -d '2009-W53-7 00:00:00Z' -W -E '2009-W53-7 00:00:00Z'"
	test_assert $CRM_EX_OK 0

	desc="2009-01-31 + 1 Month"
	cmd="iso8601 -d '2009-01-31 00:00:00Z' -D P1M -E '2009-02-28 00:00:00Z'"
	test_assert $CRM_EX_OK 0

	desc="2009-01-31 + 2 Months"
	cmd="iso8601 -d '2009-01-31 00:00:00Z' -D P2M -E '2009-03-31 00:00:00Z'"
	test_assert $CRM_EX_OK 0

	desc="2009-01-31 + 3 Months"
	cmd="iso8601 -d '2009-01-31 00:00:00Z' -D P3M -E '2009-04-30 00:00:00Z'"
	test_assert $CRM_EX_OK 0

	desc="2009-03-31 - 1 Month"
	cmd="iso8601 -d '2009-03-31 00:00:00Z' -D P-1M -E '2009-02-28 00:00:00Z'"
	test_assert $CRM_EX_OK 0
	}

	function test_acl_loop() {
	local TMPXML="$1"

	# Make sure we're rejecting things for the right reasons
	export PCMK_trace_functions=__xml_acl_check,__xml_acl_post_process
	export PCMK_stderr=1

	CIB_user=root cibadmin --replace --xml-text '<resources/>'

	export CIB_user=unknownguy
	desc="$CIB_user: Query configuration"
	cmd="cibadmin -Q"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	desc="$CIB_user: Set enable-acl"
	cmd="crm_attribute -n enable-acl -v false"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	desc="$CIB_user: Set stonith-enabled"
	cmd="crm_attribute -n stonith-enabled -v false"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	desc="$CIB_user: Create a resource"
	cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	export CIB_user=l33t-haxor
	desc="$CIB_user: Query configuration"
	cmd="cibadmin -Q"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	desc="$CIB_user: Set enable-acl"
	cmd="crm_attribute -n enable-acl -v false"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	desc="$CIB_user: Set stonith-enabled"
	cmd="crm_attribute -n stonith-enabled -v false"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	desc="$CIB_user: Create a resource"
	cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	export CIB_user=niceguy
	desc="$CIB_user: Query configuration"
	cmd="cibadmin -Q"
	test_assert $CRM_EX_OK 0

	desc="$CIB_user: Set enable-acl"
	cmd="crm_attribute -n enable-acl -v false"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	desc="$CIB_user: Set stonith-enabled"
	cmd="crm_attribute -n stonith-enabled -v false"
	test_assert $CRM_EX_OK

	desc="$CIB_user: Create a resource"
	cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	export CIB_user=root
	desc="$CIB_user: Query configuration"
	cmd="cibadmin -Q"
	test_assert $CRM_EX_OK 0

	desc="$CIB_user: Set stonith-enabled"
	cmd="crm_attribute -n stonith-enabled -v true"
	test_assert $CRM_EX_OK

	desc="$CIB_user: Create a resource"
	cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
	test_assert $CRM_EX_OK

	export CIB_user=l33t-haxor

	desc="$CIB_user: Create a resource meta attribute"
	cmd="crm_resource -r dummy --meta -p target-role -v Stopped"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	desc="$CIB_user: Query a resource meta attribute"
	cmd="crm_resource -r dummy --meta -g target-role"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	desc="$CIB_user: Remove a resource meta attribute"
	cmd="crm_resource -r dummy --meta -d target-role"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	export CIB_user=niceguy

	desc="$CIB_user: Create a resource meta attribute"
	cmd="crm_resource -r dummy --meta -p target-role -v Stopped"
	test_assert $CRM_EX_OK

	desc="$CIB_user: Query a resource meta attribute"
	cmd="crm_resource -r dummy --meta -g target-role"
	test_assert $CRM_EX_OK

	desc="$CIB_user: Remove a resource meta attribute"
	cmd="crm_resource -r dummy --meta -d target-role"
	test_assert $CRM_EX_OK

	desc="$CIB_user: Create a resource meta attribute"
	cmd="crm_resource -r dummy --meta -p target-role -v Started"
	test_assert $CRM_EX_OK

	export CIB_user=badidea
	desc="$CIB_user: Query configuration - implied deny"
	cmd="cibadmin -Q"
	test_assert $CRM_EX_OK 0

	export CIB_user=betteridea
	desc="$CIB_user: Query configuration - explicit deny"
	cmd="cibadmin -Q"
	test_assert $CRM_EX_OK 0

	CIB_user=root cibadmin -Q > "$TMPXML"
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin --delete --xml-text '<acls/>'
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql

	export CIB_user=niceguy
	desc="$CIB_user: Replace - remove acls"
	cmd="cibadmin --replace --xml-file $TMPXML"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	CIB_user=root cibadmin -Q > "$TMPXML"
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -C -o resources --xml-text '<primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy"/>'
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql

	desc="$CIB_user: Replace - create resource"
	cmd="cibadmin --replace --xml-file $TMPXML"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	CIB_user=root cibadmin -Q > "$TMPXML"
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" crm_attribute -n enable-acl -v false
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql

	desc="$CIB_user: Replace - modify attribute (deny)"
	cmd="cibadmin --replace --xml-file $TMPXML"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	CIB_user=root cibadmin -Q > "$TMPXML"
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin --replace --xml-text '<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl"/>'
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql

	desc="$CIB_user: Replace - delete attribute (deny)"
	cmd="cibadmin --replace --xml-file $TMPXML"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	CIB_user=root cibadmin -Q > "$TMPXML"
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin --modify --xml-text '<primitive id="dummy" description="nothing interesting"/>'
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql

	desc="$CIB_user: Replace - create attribute (deny)"
	cmd="cibadmin --replace --xml-file $TMPXML"
	test_assert $CRM_EX_INSUFFICIENT_PRIV 0

	CIB_user=bob
	CIB_user=root cibadmin -Q > "$TMPXML"
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin --modify --xml-text '<primitive id="dummy" description="nothing interesting"/>'
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql

	desc="$CIB_user: Replace - create attribute (allow)"
	cmd="cibadmin --replace -o resources --xml-file $TMPXML"
	test_assert $CRM_EX_OK 0

	CIB_user=root cibadmin -Q > "$TMPXML"
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin --modify --xml-text '<primitive id="dummy" description="something interesting"/>'
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql

	desc="$CIB_user: Replace - modify attribute (allow)"
	cmd="cibadmin --replace -o resources --xml-file $TMPXML"
	test_assert $CRM_EX_OK 0

	CIB_user=root cibadmin -Q > "$TMPXML"
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin --replace -o resources --xml-text '<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy"/>'
	CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql

	desc="$CIB_user: Replace - delete attribute (allow)"
	cmd="cibadmin --replace -o resources --xml-file $TMPXML"
	test_assert $CRM_EX_OK 0
	}

	function test_acls() {
	local SHADOWPATH
	local TMPXML=$(mktemp ${TMPDIR:-/tmp}/cts-cli.acls.xml.XXXXXXXXXX)
	export CIB_shadow_dir="${shadow_dir}"

	$VALGRIND_CMD crm_shadow --batch --force --create-empty $shadow --validate-with pacemaker-1.3 2>&1
	export CIB_shadow=$shadow

	cat <<EOF > "$TMPXML"
	<acls>
	<acl_user id="l33t-haxor">
	<deny id="crook-nothing" xpath="/cib"/>
	</acl_user>
	<acl_user id="niceguy">
	<role_ref id="observer"/>
	</acl_user>
	<acl_user id="bob">
	<role_ref id="admin"/>
	</acl_user>
	<acl_role id="observer">
	<read id="observer-read-1" xpath="/cib"/>
	<write id="observer-write-1" xpath="//nvpair[@name='stonith-enabled']"/>
	<write id="observer-write-2" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_role id="admin">
	<read id="admin-read-1" xpath="/cib"/>
	<write id="admin-write-1" xpath="//resources"/>
	</acl_role>
	</acls>
	EOF

	desc="Configure some ACLs"
	cmd="cibadmin -M -o acls --xml-file $TMPXML"
	test_assert $CRM_EX_OK

	desc="Enable ACLs"
	cmd="crm_attribute -n enable-acl -v true"
	test_assert $CRM_EX_OK

	desc="Set cluster option"
	cmd="crm_attribute -n no-quorum-policy -v ignore"
	test_assert $CRM_EX_OK

	desc="New ACL"
	cmd="cibadmin --create -o acls --xml-text '<acl_user id=\"badidea\"><read id=\"badidea-resources\" xpath=\"//meta_attributes\"/></acl_user>'"
	test_assert $CRM_EX_OK

	desc="Another ACL"
	cmd="cibadmin --create -o acls --xml-text '<acl_user id=\"betteridea\"><read id=\"betteridea-resources\" xpath=\"//meta_attributes\"/></acl_user>'"
	test_assert $CRM_EX_OK

	desc="Updated ACL"
	cmd="cibadmin --replace -o acls --xml-text '<acl_user id=\"betteridea\"><deny id=\"betteridea-nothing\" xpath=\"/cib\"/><read id=\"betteridea-resources\" xpath=\"//meta_attributes\"/></acl_user>'"
	test_assert $CRM_EX_OK

	test_acl_loop "$TMPXML"

	printf "\n\n !#!#!#!#! Upgrading to latest CIB schema and re-testing !#!#!#!#!\n"
	printf "\nUpgrading to latest CIB schema and re-testing\n" 1>&2

	export CIB_user=root
	desc="$CIB_user: Upgrade to latest CIB schema"
	cmd="cibadmin --upgrade --force -V"
	test_assert $CRM_EX_OK

	SHADOWPATH="$(crm_shadow --file)"
	# sed -i isn't portable :-(
	cp -p "$SHADOWPATH" "${SHADOWPATH}.$$" # to keep permissions
	sed -e 's/epoch=.2/epoch=\"6/g' -e 's/admin_epoch=.1/admin_epoch=\"0/g' \
	"$SHADOWPATH" > "${SHADOWPATH}.$$"
	mv -- "${SHADOWPATH}.$$" "$SHADOWPATH"

	test_acl_loop "$TMPXML"

	unset CIB_shadow_dir
	rm -f "$TMPXML"
	}

	function test_validity() {
	local TMPGOOD=$(mktemp ${TMPDIR:-/tmp}/cts-cli.validity.good.xml.XXXXXXXXXX)
	local TMPBAD=$(mktemp ${TMPDIR:-/tmp}/cts-cli.validity.bad.xml.XXXXXXXXXX)
	export CIB_shadow_dir="${shadow_dir}"

	$VALGRIND_CMD crm_shadow --batch --force --create-empty $shadow --validate-with pacemaker-1.2 2>&1
	export CIB_shadow=$shadow
	export PCMK_trace_functions=update_validation,cli_config_update
	export PCMK_stderr=1

	cibadmin -C -o resources --xml-text '<primitive id="dummy1" class="ocf" provider="pacemaker" type="Dummy"/>'
	cibadmin -C -o resources --xml-text '<primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy"/>'
	cibadmin -C -o constraints --xml-text '<rsc_order id="ord_1-2" first="dummy1" first-action="start" then="dummy2"/>'
	cibadmin -Q > "$TMPGOOD"


	desc="Try to make resulting CIB invalid (enum violation)"
	cmd="cibadmin -M -o constraints --xml-text '<rsc_order id=\"ord_1-2\" first=\"dummy1\" first-action=\"break\" then=\"dummy2\"/>'"
	test_assert $CRM_EX_CONFIG

	sed 's\|"start"\|"break"\|' "$TMPGOOD" > "$TMPBAD"
	desc="Run crm_simulate with invalid CIB (enum violation)"
	cmd="crm_simulate -x $TMPBAD -S"
	test_assert $CRM_EX_CONFIG 0


	desc="Try to make resulting CIB invalid (unrecognized validate-with)"
	cmd="cibadmin -M --xml-text '<cib validate-with=\"pacemaker-9999.0\"/>'"
	test_assert $CRM_EX_CONFIG

	sed 's\|"pacemaker-1.2"\|"pacemaker-9999.0"\|' "$TMPGOOD" > "$TMPBAD"
	desc="Run crm_simulate with invalid CIB (unrecognized validate-with)"
	cmd="crm_simulate -x $TMPBAD -S"
	test_assert $CRM_EX_CONFIG 0


	desc="Try to make resulting CIB invalid, but possibly recoverable (valid with X.Y+1)"
	cmd="cibadmin -C -o configuration --xml-text '<tags/>'"
	test_assert $CRM_EX_CONFIG

	- sed 's\|</configuration>\|<tags/>\0\|' "$TMPGOOD" > "$TMPBAD"
	+ sed 's\|</configuration>\|<tags/></configuration>\|' "$TMPGOOD" > "$TMPBAD"
	desc="Run crm_simulate with invalid, but possibly recoverable CIB (valid with X.Y+1)"
	cmd="crm_simulate -x $TMPBAD -S"
	test_assert $CRM_EX_OK 0


	- sed 's\|\s\svalidate-with="[^"]"\|\|' "$TMPGOOD" > "$TMPBAD"
	+ sed 's\|[ ][ ]validate-with="[^"]"\|\|' "$TMPGOOD" > "$TMPBAD"
	desc="Make resulting CIB valid, although without validate-with attribute"
	cmd="cibadmin -R --xml-file $TMPBAD"
	test_assert $CRM_EX_OK

	desc="Run crm_simulate with valid CIB, but without validate-with attribute"
	cmd="crm_simulate -x $TMPBAD -S"
	test_assert $CRM_EX_OK 0


	# this will just disable validation and accept the config, outputting
	# validation errors
	- sed -e 's\|\s\svalidate-with="[^"]"\|\|' \
	- -e 's\|$\s\sepoch="[^"]$"\|\10"\|' -e 's\|"start"\|"break"\|' \
	+ sed -e 's\|[ ][ ]validate-with="[^"]"\|\|' \
	+ -e 's\|$[ ][ ]epoch="[^"]$"\|\10"\|' -e 's\|"start"\|"break"\|' \
	"$TMPGOOD" > "$TMPBAD"
	desc="Make resulting CIB invalid, and without validate-with attribute"
	cmd="cibadmin -R --xml-file $TMPBAD"
	test_assert $CRM_EX_OK

	desc="Run crm_simulate with invalid CIB, also without validate-with attribute"
	cmd="crm_simulate -x $TMPBAD -S"
	test_assert $CRM_EX_OK 0

	unset CIB_shadow_dir
	rm -f "$TMPGOOD" "$TMPBAD"
	}

	# Process command-line arguments
	while [ $# -gt 0 ]; do
	case "$1" in
	-t)
	tests="$2"
	shift 2
	;;
	-V\|--verbose)
	verbose=1
	shift
	;;
	-v\|--valgrind)
	export G_SLICE=always-malloc
	VALGRIND_CMD="valgrind $VALGRIND_OPTS"
	shift
	;;
	-s)
	do_save=1
	shift
	;;
	-p)
	export PATH="$2:$PATH"
	shift
	;;
	--help)
	echo "$USAGE_TEXT"
	exit $CRM_EX_OK
	;;
	*)
	echo "error: unknown option $1"
	echo
	echo "$USAGE_TEXT"
	exit $CRM_EX_USAGE
	;;
	esac
	done

	for t in $tests; do
	case "$t" in
	dates) ;;
	tools) ;;
	acls) ;;
	validity) ;;
	*)
	echo "error: unknown test $t"
	echo
	echo "$USAGE_TEXT"
	exit $CRM_EX_USAGE
	;;
	esac
	done

	# Check whether we're running from source directory
	SRCDIR=$(dirname $test_home)
	if [ -x "$SRCDIR/tools/crm_simulate" ]; then
	export PATH="$SRCDIR/tools:$PATH"
	echo "Using local binaries from: $SRCDIR/tools"

	if [ -x "$SRCDIR/xml" ]; then
	export PCMK_schema_directory="$SRCDIR/xml"
	echo "Using local schemas from: $PCMK_schema_directory"
	fi
	fi

	for t in $tests; do
	echo "Testing $t"
	TMPFILE=$(mktemp ${TMPDIR:-/tmp}/cts-cli.$t.XXXXXXXXXX)
	eval TMPFILE_$t="$TMPFILE"
	test_$t > "$TMPFILE"

	- # sed -i isn't portable :-(
	- # neither is \\| :-(
	sed -e 's/cib-last-written.*>/>/'\
	-e 's/ last-run=\"[0-9]*\"//'\
	-e 's/crm_feature_set="[^"]*" //'\
	-e 's/validate-with="[^"]*" //'\
	-e 's/Created new pacemaker-.* configuration/Created new pacemaker configuration/'\
	-e 's/.$__xml_.$@.\.c:[0-9][0-9])/\1/g' \
	-e 's/.$unpack_.$@.\.c:[0-9][0-9])/\1/g' \
	-e 's/.$update_validation$@.\.c:[0-9][0-9]*)/\1/g' \
	-e 's/ last-rc-change=\"[0-9]*\"//'\
	-e 's\|^/tmp/cts-cli\.validity\.bad.xml\.[^:]*:\|validity.bad.xml:\|'\
	-e 's/^Entity: line [0-9][0-9]*: //'\
	-e 's/$validation ([0-9][0-9]* of $[0-9][0-9]$).$/\1X\2/' \
	"$TMPFILE" > "${TMPFILE}.$$"
	mv -- "${TMPFILE}.$$" "$TMPFILE"

	if [ $do_save -eq 1 ]; then
	cp "$TMPFILE" $test_home/cli/regression.$t.exp
	fi
	done

	rm -rf "${shadow_dir}"

	failed=0

	if [ $verbose -eq 1 ]; then
	echo -e "\n\nResults"
	fi
	for t in $tests; do
	eval TMPFILE="\$TMPFILE_$t"
	if [ $verbose -eq 1 ]; then
	diff -wu $test_home/cli/regression.$t.exp "$TMPFILE"
	else
	diff -w $test_home/cli/regression.$t.exp "$TMPFILE" >/dev/null 2>&1
	fi
	if [ $? -ne 0 ]; then
	failed=1
	fi
	done

	echo -e "\n\nSummary"
	for t in $tests; do
	eval TMPFILE="\$TMPFILE_$t"
	grep -e "^*" "$TMPFILE"
	done

	if [ $num_errors -ne 0 ]; then
	echo "$num_errors tests failed; see output in:"
	for t in $tests; do
	eval TMPFILE="\$TMPFILE_$t"
	echo " $TMPFILE"
	done
	exit $CRM_EX_ERROR

	elif [ $failed -eq 1 ]; then
	echo "$num_passed tests passed but output was unexpected; see output in:"
	for t in $tests; do
	eval TMPFILE="\$TMPFILE_$t"
	echo " $TMPFILE"
	done
	exit $CRM_EX_DIGEST

	else
	echo $num_passed tests passed
	for t in $tests; do
	eval TMPFILE="\$TMPFILE_$t"
	rm -f "$TMPFILE"
	done
	crm_shadow --force --delete $shadow >/dev/null 2>&1
	exit $CRM_EX_OK
	fi
	diff --git a/cts/cts-exec.in b/cts/cts-exec.in
	index 041af1e8a1..da1a279111 100644
	--- a/cts/cts-exec.in
	+++ b/cts/cts-exec.in
	@@ -1,1285 +1,1284 @@
	#!@PYTHON@
	""" Regression tests for Pacemaker's pacemaker-execd
	"""

	# Pacemaker targets compatibility with Python 2.7 and 3.2+
	from __future__ import print_function, unicode_literals, absolute_import, division

	__copyright__ = "Copyright 2012-2018 Andrew Beekhof <andrew@beekhof.net>"
	__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"

	import io
	import os
	import stat
	import sys
	import subprocess
	import shlex
	import time

	# Where to find test binaries
	# Prefer the source tree if available
	BUILD_DIR = "@abs_top_builddir@"
	TEST_DIR = sys.path[0]

	SBIN_DIR = "@sbindir@"

	# File permissions for executable scripts we create
	EXECMODE = stat.S_IRUSR \| stat.S_IXUSR \| stat.S_IRGRP \| stat.S_IXGRP \| stat.S_IROTH \| stat.S_IXOTH

	# These values must be kept in sync with include/crm/crm.h
	class CrmExit:
	OK = 0
	ERROR = 1
	INVALID_PARAM = 2
	UNIMPLEMENT_FEATURE = 3
	INSUFFICIENT_PRIV = 4
	NOT_INSTALLED = 5
	NOT_CONFIGURED = 6
	NOT_RUNNING = 7
	USAGE = 64
	DATAERR = 65
	NOINPUT = 66
	NOUSER = 67
	NOHOST = 68
	UNAVAILABLE = 69
	SOFTWARE = 70
	OSERR = 71
	OSFILE = 72
	CANTCREAT = 73
	IOERR = 74
	TEMPFAIL = 75
	PROTOCOL = 76
	NOPERM = 77
	CONFIG = 78
	FATAL = 100
	PANIC = 101
	DISCONNECT = 102
	SOLO = 103
	DIGEST = 104
	NOSUCH = 105
	QUORUM = 106
	UNSAFE = 107
	EXISTS = 108
	MULTIPLE = 109
	OLD = 110
	TIMEOUT = 124
	MAX = 255


	def update_path():
	""" Set the PATH environment variable appropriately for the tests """

	new_path = os.environ['PATH']
	if os.path.exists("%s/cts-exec.in" % TEST_DIR):
	print("Running tests from the source tree: %s (%s)" % (BUILD_DIR, TEST_DIR))
	# For pacemaker-execd, cts-exec-helper, and pacemaker-remoted
	new_path = "%s/daemons/execd:%s" % (BUILD_DIR, new_path)
	new_path = "%s/tools:%s" % (BUILD_DIR, new_path) # For crm_resource
	# For pacemaker-fenced
	new_path = "%s/daemons/fenced:%s" % (BUILD_DIR, new_path)
	# For cts-support
	new_path = "%s/cts:%s" % (BUILD_DIR, new_path)

	else:
	print("Running tests from the install tree: @CRM_DAEMON_DIR@ (not %s)" % TEST_DIR)
	# For cts-exec-helper, cts-support, pacemaker-execd, pacemaker-fenced,
	# and pacemaker-remoted
	new_path = "@CRM_DAEMON_DIR@:%s" % (new_path)

	print('Using PATH="{}"'.format(new_path))
	os.environ['PATH'] = new_path


	def pipe_output(pipes, stdout=True, stderr=False):
	""" Wrapper to get text output from pipes regardless of Python version """

	output = ""
	pipe_outputs = pipes.communicate()
	if sys.version_info < (3,):
	if stdout:
	output = output + pipe_outputs[0]
	if stderr:
	output = output + pipe_outputs[1]
	else:
	if stdout:
	output = output + pipe_outputs[0].decode(sys.stdout.encoding)
	if stderr:
	output = output + pipe_outputs[1].decode(sys.stderr.encoding)
	return output


	def output_from_command(command):
	""" Run a command, and return its standard output. """

	test = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE)
	test.wait()
	return pipe_output(test).split("\n")


	def write_file(filename, contents, executable=False):
	""" Create a file. """

	f = io.open(filename, "w+")
	f.write(contents)
	f.close()
	if executable:
	os.chmod(filename, EXECMODE)


	class TestError(Exception):
	""" Base class for exceptions in this module """
	pass


	class ExitCodeError(TestError):
	""" Exception raised when command exit status is unexpected """

	def __init__(self, exit_code):
	self.exit_code = exit_code

	def __str__(self):
	return repr(self.exit_code)


	class OutputNotFoundError(TestError):
	""" Exception raised when command output does not contain wanted string """

	def __init__(self, output):
	self.output = output

	def __str__(self):
	return repr(self.output)


	class OutputFoundError(TestError):
	""" Exception raised when command output contains unwanted string """

	def __init__(self, output):
	self.output = output

	def __str__(self):
	return repr(self.output)


	class Test(object):
	""" Executor for a single pacemaker-execd regression test """

	def __init__(self, name, description, verbose=0, tls=0):
	self.name = name
	self.description = description
	self.cmds = []

	if tls:
	self.daemon_location = "pacemaker-remoted"
	else:
	self.daemon_location = "pacemaker-execd"

	self.test_tool_location = "cts-exec-helper"
	self.verbose = verbose
	self.tls = tls

	self.result_txt = ""
	self.cmd_tool_output = ""
	self.result_exitcode = CrmExit.OK

	self.execd_process = None
	self.stonith_process = None

	self.executed = 0

	def __new_cmd(self, cmd, args, exitcode, stdout_match="", no_wait=0, stdout_negative_match="", kill=None):
	""" Add a command to be executed as part of this test """

	if self.verbose and cmd == self.test_tool_location:
	args = args + " -V "

	if (cmd == self.test_tool_location) and self.tls:
	args = args + " -S "

	self.cmds.append(
	{
	"cmd" : cmd,
	"kill" : kill,
	"args" : args,
	"expected_exitcode" : exitcode,
	"stdout_match" : stdout_match,
	"stdout_negative_match" : stdout_negative_match,
	"no_wait" : no_wait,
	"cmd_output" : "",
	}
	)

	def start_environment(self):
	""" Prepare the host for running a test """

	### make sure we are in full control here ###
	cmd = shlex.split("killall -q -9 pacemaker-fenced lt-pacemaker-fenced pacemaker-execd lt-pacemaker-execd cts-exec-helper lt-cts-exec-helper pacemaker-remoted")
	test = subprocess.Popen(cmd, stdout=subprocess.PIPE)
	test.wait()

	additional_args = ""

	if self.tls == 0:
	self.stonith_process = subprocess.Popen(shlex.split("pacemaker-fenced -s"))

	if self.verbose:
	additional_args = additional_args + " -V"

	self.execd_process = subprocess.Popen(shlex.split("%s %s -l /tmp/pacemaker-execd-regression.log"
	% (self.daemon_location, additional_args)))

	time.sleep(1)

	def clean_environment(self):
	""" Clean up the host after running a test """

	if self.execd_process:
	self.execd_process.terminate()
	self.execd_process.wait()

	if self.verbose:
	print("Daemon output")
	logfile = io.open('/tmp/pacemaker-execd-regression.log', 'rt', errors='replace')
	for line in logfile:
	print(line.strip().encode('utf-8', 'replace'))
	os.remove('/tmp/pacemaker-execd-regression.log')

	if self.stonith_process:
	self.stonith_process.terminate()
	self.stonith_process.wait()

	self.execd_process = None
	self.stonith_process = None

	def add_sys_cmd(self, cmd, args):
	""" Add a simple command to be executed as part of this test """

	self.__new_cmd(cmd, args, CrmExit.OK, "")

	def add_cmd_check_stdout(self, args, match, no_match=""):
	""" Add a command with expected output to be executed as part of this test """

	self.__new_cmd(self.test_tool_location, args, CrmExit.OK, match, 0, no_match)

	def add_cmd(self, args):
	""" Add a cts-exec-helper command to be executed as part of this test """

	self.__new_cmd(self.test_tool_location, args, CrmExit.OK, "")

	def add_cmd_and_kill(self, kill_proc, args):
	""" Add a cts-exec-helper command and system command to be executed as part of this test """

	self.__new_cmd(self.test_tool_location, args, CrmExit.OK, "", kill=kill_proc)

	def add_expected_fail_cmd(self, args, exitcode=CrmExit.ERROR):
	""" Add a cts-exec-helper command to be executed as part of this test and expected to fail """

	self.__new_cmd(self.test_tool_location, args, exitcode, "")

	def get_exitcode(self):
	""" Return the exit status of the last test execution """

	return self.result_exitcode

	def print_result(self, filler):
	""" Print the result of the last test execution """

	print("%s%s" % (filler, self.result_txt))

	def run_cmd(self, args):
	""" Execute a command as part of this test """

	cmd = shlex.split(args['args'])
	cmd.insert(0, args['cmd'])
	if self.verbose:
	print("\n\nRunning: "+" ".join(cmd))
	test = subprocess.Popen(cmd, stdout=subprocess.PIPE)

	if args['kill']:
	if self.verbose:
	print("Also running: "+args['kill'])
	### Typically, the kill argument is used to detect some sort of
	### failure. Without yielding for a few seconds here, the process
	### launched earlier that is listening for the failure may not have
	### time to connect to pacemaker-execd.
	time.sleep(2)
	subprocess.Popen(shlex.split(args['kill']))

	if args['no_wait'] == 0:
	test.wait()
	else:
	return CrmExit.OK

	output = pipe_output(test)
	args['cmd_output'] = output

	if test.returncode != args['expected_exitcode']:
	raise ExitCodeError(test.returncode)

	if args['stdout_match'] != "" and output.count(args['stdout_match']) == 0:
	raise OutputNotFoundError(output)

	if args['stdout_negative_match'] != "" and output.count(args['stdout_negative_match']) != 0:
	raise OutputFoundError(output)

	def set_error(self, step, cmd):
	""" Record failure of this test """

	msg = "FAILURE - '%s' failed at step %d. Command: %s %s"
	self.result_txt = msg % (self.name, step, cmd['cmd'], cmd['args'])
	self.result_exitcode = CrmExit.ERROR

	def run(self):
	""" Execute this test. """

	res = 0
	i = 1

	if self.tls and self.name.count("stonith") != 0:
	self.result_txt = "SKIPPED - '%s' - disabled when testing pacemaker_remote" % (self.name)
	print(self.result_txt)
	return res

	self.start_environment()

	if self.verbose:
	print("\n--- START TEST - %s" % self.name)

	self.result_txt = "SUCCESS - '%s'" % (self.name)
	self.result_exitcode = CrmExit.OK
	for cmd in self.cmds:
	try:
	self.run_cmd(cmd)
	except ExitCodeError as e:
	print(cmd['cmd_output'])
	print("Step %d FAILED - command returned %s, expected %d" % (i, e, cmd['expected_exitcode']))
	self.set_error(i, cmd);
	break
	except OutputNotFoundError as e:
	print("Step %d FAILED - '%s' was not found in command output: %s" % (i, cmd['stdout_match'], e))
	self.set_error(i, cmd);
	break
	except OutputFoundError as e:
	print("Step %d FAILED - '%s' was found in command output: %s" % (i, cmd['stdout_negative_match'], e))
	self.set_error(i, cmd);
	break
	if self.verbose:
	print(cmd['cmd_output'].strip())
	print("Step %d SUCCESS" % (i))
	i = i + 1
	self.clean_environment()

	print(self.result_txt)
	if self.verbose:
	print("--- END TEST - %s\n" % self.name)

	self.executed = 1
	return res

	class Tests(object):
	""" Collection of all pacemaker-execd regression tests """

	def __init__(self, verbose=0, tls=0):
	self.tests = []
	self.verbose = verbose
	self.tls = tls
	self.rsc_classes = output_from_command("crm_resource --list-standards")
	self.rsc_classes = self.rsc_classes[:-1] # Strip trailing empty line
	self.need_authkey = 0
	self.action_timeout = " -t 9000 "
	if self.tls:
	self.rsc_classes.remove("stonith")
	if "systemd" in self.rsc_classes:
	try:
	# This code doesn't need this import, but pacemaker-cts-dummyd
	# does, so ensure the dependency is available rather than cause
	# all systemd tests to fail.
	import systemd.daemon
	except ImportError:
	print("Fatal error: python systemd bindings not found. Is package installed?",
	file=sys.stderr)
	sys.exit(CrmExit.ERROR)

	print("Testing resource classes", repr(self.rsc_classes))

	self.common_cmds = {
	"ocf_reg_line" : "-c register_rsc -r ocf_test_rsc "+self.action_timeout+" -C ocf -P pacemaker -T Dummy",
	"ocf_reg_event" : "-l \"NEW_EVENT event_type:register rsc_id:ocf_test_rsc action:none rc:ok op_status:complete\"",
	"ocf_unreg_line" : "-c unregister_rsc -r \"ocf_test_rsc\" "+self.action_timeout,
	"ocf_unreg_event" : "-l \"NEW_EVENT event_type:unregister rsc_id:ocf_test_rsc action:none rc:ok op_status:complete\"",
	"ocf_start_line" : "-c exec -r \"ocf_test_rsc\" -a \"start\" "+self.action_timeout,
	"ocf_start_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:start rc:ok op_status:complete\" ",
	"ocf_stop_line" : "-c exec -r \"ocf_test_rsc\" -a \"stop\" "+self.action_timeout,
	"ocf_stop_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:stop rc:ok op_status:complete\" ",
	"ocf_monitor_line" : '-c exec -r ocf_test_rsc -a monitor -i 2s ' + self.action_timeout,
	"ocf_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout,
	"ocf_cancel_line" : '-c cancel -r ocf_test_rsc -a monitor -i 2s ' + self.action_timeout,
	"ocf_cancel_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:monitor rc:ok op_status:Cancelled\" ",

	"systemd_reg_line" : "-c register_rsc -r systemd_test_rsc " +
	self.action_timeout +
	" -C systemd -T pacemaker-cts-dummyd@3",
	"systemd_reg_event" : "-l \"NEW_EVENT event_type:register rsc_id:systemd_test_rsc action:none rc:ok op_status:complete\"",
	"systemd_unreg_line" : "-c unregister_rsc -r \"systemd_test_rsc\" "+self.action_timeout,
	"systemd_unreg_event" : "-l \"NEW_EVENT event_type:unregister rsc_id:systemd_test_rsc action:none rc:ok op_status:complete\"",
	"systemd_start_line" : "-c exec -r \"systemd_test_rsc\" -a \"start\" "+self.action_timeout,
	"systemd_start_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:start rc:ok op_status:complete\" ",
	"systemd_stop_line" : "-c exec -r \"systemd_test_rsc\" -a \"stop\" "+self.action_timeout,
	"systemd_stop_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:stop rc:ok op_status:complete\" ",
	"systemd_monitor_line" : '-c exec -r systemd_test_rsc -a monitor -i 2s ' + self.action_timeout,
	"systemd_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:monitor rc:ok op_status:complete\" -t 15000 ",
	"systemd_cancel_line" : '-c cancel -r systemd_test_rsc -a monitor -i 2s ' + self.action_timeout,
	"systemd_cancel_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:monitor rc:ok op_status:Cancelled\" ",

	"upstart_reg_line" : "-c register_rsc -r upstart_test_rsc "+self.action_timeout+" -C upstart -T pacemaker-cts-dummyd",
	"upstart_reg_event" : "-l \"NEW_EVENT event_type:register rsc_id:upstart_test_rsc action:none rc:ok op_status:complete\"",
	"upstart_unreg_line" : "-c unregister_rsc -r \"upstart_test_rsc\" "+self.action_timeout,
	"upstart_unreg_event" : "-l \"NEW_EVENT event_type:unregister rsc_id:upstart_test_rsc action:none rc:ok op_status:complete\"",
	"upstart_start_line" : "-c exec -r \"upstart_test_rsc\" -a \"start\" "+self.action_timeout,
	"upstart_start_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:upstart_test_rsc action:start rc:ok op_status:complete\" ",
	"upstart_stop_line" : "-c exec -r \"upstart_test_rsc\" -a \"stop\" "+self.action_timeout,
	"upstart_stop_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:upstart_test_rsc action:stop rc:ok op_status:complete\" ",
	"upstart_monitor_line" : '-c exec -r upstart_test_rsc -a monitor -i 2s ' + self.action_timeout,
	"upstart_monitor_event" : '-l "NEW_EVENT event_type:exec_complete rsc_id:upstart_test_rsc action:monitor rc:ok op_status:complete" -t 15000',
	"upstart_cancel_line" : '-c cancel -r upstart_test_rsc -a monitor -i 2s ' + self.action_timeout,
	"upstart_cancel_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:upstart_test_rsc action:monitor rc:ok op_status:Cancelled\" ",

	"service_reg_line" : "-c register_rsc -r service_test_rsc "+self.action_timeout+" -C service -T LSBDummy",
	"service_reg_event" : "-l \"NEW_EVENT event_type:register rsc_id:service_test_rsc action:none rc:ok op_status:complete\"",
	"service_unreg_line" : "-c unregister_rsc -r \"service_test_rsc\" "+self.action_timeout,
	"service_unreg_event" : "-l \"NEW_EVENT event_type:unregister rsc_id:service_test_rsc action:none rc:ok op_status:complete\"",
	"service_start_line" : "-c exec -r \"service_test_rsc\" -a \"start\" "+self.action_timeout,
	"service_start_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:start rc:ok op_status:complete\" ",
	"service_stop_line" : "-c exec -r \"service_test_rsc\" -a \"stop\" "+self.action_timeout,
	"service_stop_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:stop rc:ok op_status:complete\" ",
	"service_monitor_line" : '-c exec -r service_test_rsc -a monitor -i 2s ' + self.action_timeout,
	"service_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout,
	"service_cancel_line" : '-c cancel -r service_test_rsc -a monitor -i 2s ' + self.action_timeout,
	"service_cancel_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:monitor rc:ok op_status:Cancelled\" ",

	"lsb_reg_line" : "-c register_rsc -r lsb_test_rsc "+self.action_timeout+" -C lsb -T LSBDummy",
	"lsb_reg_event" : "-l \"NEW_EVENT event_type:register rsc_id:lsb_test_rsc action:none rc:ok op_status:complete\" ",
	"lsb_unreg_line" : "-c unregister_rsc -r \"lsb_test_rsc\" "+self.action_timeout,
	"lsb_unreg_event" : "-l \"NEW_EVENT event_type:unregister rsc_id:lsb_test_rsc action:none rc:ok op_status:complete\"",
	"lsb_start_line" : "-c exec -r \"lsb_test_rsc\" -a \"start\" "+self.action_timeout,
	"lsb_start_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:start rc:ok op_status:complete\" ",
	"lsb_stop_line" : "-c exec -r \"lsb_test_rsc\" -a \"stop\" "+self.action_timeout,
	"lsb_stop_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:stop rc:ok op_status:complete\" ",
	"lsb_monitor_line" : '-c exec -r lsb_test_rsc -a status -i 2s ' + self.action_timeout,
	"lsb_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:status rc:ok op_status:complete\" "+self.action_timeout,
	"lsb_cancel_line" : '-c cancel -r lsb_test_rsc -a status -i 2s ' + self.action_timeout,
	"lsb_cancel_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:status rc:ok op_status:Cancelled\" ",

	"stonith_reg_line" : "-c register_rsc -r stonith_test_rsc "+self.action_timeout+" -C stonith -P pacemaker -T fence_dummy_monitor",
	"stonith_reg_event" : "-l \"NEW_EVENT event_type:register rsc_id:stonith_test_rsc action:none rc:ok op_status:complete\" ",
	"stonith_unreg_line" : "-c unregister_rsc -r \"stonith_test_rsc\" "+self.action_timeout,
	"stonith_unreg_event" : "-l \"NEW_EVENT event_type:unregister rsc_id:stonith_test_rsc action:none rc:ok op_status:complete\"",
	"stonith_start_line" : '-c exec -r stonith_test_rsc -a start ' + self.action_timeout,
	"stonith_start_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:start rc:ok op_status:complete\" ",
	"stonith_stop_line" : "-c exec -r \"stonith_test_rsc\" -a \"stop\" "+self.action_timeout,
	"stonith_stop_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:stop rc:ok op_status:complete\" ",
	"stonith_monitor_line" : '-c exec -r stonith_test_rsc -a monitor -i 2s ' + self.action_timeout,
	"stonith_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout,
	"stonith_cancel_line" : '-c cancel -r stonith_test_rsc -a monitor -i 2s ' + self.action_timeout,
	"stonith_cancel_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:ok op_status:Cancelled\" ",
	}

	def new_test(self, name, description):
	""" Create a named test """

	test = Test(name, description, self.verbose, self.tls)
	self.tests.append(test)
	return test

	def setup_test_environment(self):
	""" Prepare the host before executing any tests """

	os.system("service pacemaker_remote stop")
	self.cleanup_test_environment()

	if self.tls and not os.path.isfile("/etc/pacemaker/authkey"):
	self.need_authkey = 1
	os.system("mkdir -p /etc/pacemaker")
	os.system("dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1")

	dummy_upstart_job = ("""
	description "Dummy service for regression tests"
	exec dd if=/dev/random of=/dev/null
	""")

	dummy_fence_sleep_agent = ("""#!@PYTHON@
	import sys
	import time
	def main():
	for line in sys.stdin:
	if line.count("monitor") > 0:
	time.sleep(30000)
	sys.exit(0)
	sys.exit(1)
	if __name__ == "__main__":
	main()
	""")
	dummy_fence_agent = ("""#!/bin/sh
	while read line; do
	case ${line} in
	monitor) exit 0;;
	metadata)
	echo '<resource-agent name="fence_dummy_monitor" shortdesc="Dummy Fence agent for testing">'
	echo ' <longdesc>dummy description.</longdesc>'
	echo ' <vendor-url>http://www.example.com</vendor-url>'
	echo ' <parameters>'
	echo ' <parameter name="action" unique="0" required="1">'
	echo ' <getopt mixed="-o, --action=[action]"/>'
	echo ' <content type="string" default="reboot"/>'
	echo ' <shortdesc lang="en">Fencing Action</shortdesc>'
	echo ' </parameter>'
	echo ' <parameter name="port" unique="0" required="0">'
	echo ' <getopt mixed="-n, --plug=[id]"/>'
	echo ' <content type="string"/>'
	echo ' <shortdesc lang="en">Physical plug number or name of virtual machine</shortdesc>'
	echo ' </parameter>'
	echo ' </parameters>'
	echo ' <actions>'
	echo ' <action name="on"/>'
	echo ' <action name="off"/>'
	echo ' <action name="monitor"/>'
	echo ' <action name="metadata"/>'
	echo ' </actions>'
	echo '</resource-agent>'
	exit 0;;
	esac
	exit 1
	done
	""")

	if os.path.isdir("/etc/init"):
	write_file("/etc/init/pacemaker-cts-dummyd.conf", dummy_upstart_job);
	write_file(SBIN_DIR + "/fence_dummy_sleep", dummy_fence_sleep_agent, executable=True);
	write_file(SBIN_DIR + "/fence_dummy_monitor", dummy_fence_agent, executable=True);

	if os.path.exists("%s/cts/LSBDummy" % BUILD_DIR):
	os.system("cp %s/cts/LSBDummy /etc/init.d/LSBDummy" % BUILD_DIR)

	if not os.path.exists("@OCF_RA_DIR@/pacemaker"):
	os.system("mkdir -p @OCF_RA_DIR@/pacemaker/")

	# Install helper OCF agents
	for agent in ["Dummy", "Stateful", "ping"]:
	os.system("cp %s/extra/resources/%s @OCF_RA_DIR@/pacemaker/%s" % (BUILD_DIR, agent, agent))
	os.system("chmod a+x @OCF_RA_DIR@/pacemaker/%s" % (agent))

	else:
	# Assume it's installed
	os.system("cp @datadir@/@PACKAGE@/tests/cts/LSBDummy /etc/init.d/LSBDummy")

	os.system("chmod a+x /etc/init.d/LSBDummy")
	- os.system("mkdir -p @CRM_CORE_DIR@/root")
	subprocess.call(["cts-support", "install"])

	def cleanup_test_environment(self):
	""" Clean up the host after executing desired tests """

	if self.need_authkey:
	os.system("rm -f /etc/pacemaker/authkey")

	os.system("rm -f /etc/init.d/LSBDummy")
	os.system("rm -f " + SBIN_DIR + "/fence_dummy_monitor")
	os.system("rm -f " + SBIN_DIR + "/fence_dummy_sleep")
	subprocess.call(["cts-support", "uninstall"])

	if os.path.exists("/bin/systemctl"):
	os.system("systemctl daemon-reload")

	def build_generic_tests(self):
	""" Register tests that apply to all resource classes """

	common_cmds = self.common_cmds

	### register/unregister tests ###
	for rsc in self.rsc_classes:
	test = self.new_test("generic_registration_%s" % (rsc),
	"Simple resource registration test for %s standard" % (rsc))
	test.add_cmd(common_cmds["%s_reg_line" % (rsc)] + " " + common_cmds["%s_reg_event" % (rsc)])
	test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])

	### start/stop tests ###
	for rsc in self.rsc_classes:
	test = self.new_test("generic_start_stop_%s" % (rsc), "Simple start and stop test for %s standard" % (rsc))
	test.add_cmd(common_cmds["%s_reg_line" % (rsc)] + " " + common_cmds["%s_reg_event" % (rsc)])
	test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
	test.add_cmd(common_cmds["%s_stop_line" % (rsc)] + " " + common_cmds["%s_stop_event" % (rsc)])
	test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])

	### monitor cancel test ###
	for rsc in self.rsc_classes:
	test = self.new_test("generic_monitor_cancel_%s" % (rsc),
	"Simple monitor cancel test for %s standard" % (rsc))
	test.add_cmd(common_cmds["%s_reg_line" % (rsc)] + " " + common_cmds["%s_reg_event" % (rsc)])
	test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
	test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
	### If this fails, that means the monitor may not be getting rescheduled ####
	test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
	### If this fails, that means the monitor may not be getting rescheduled ####
	test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
	test.add_cmd(common_cmds["%s_cancel_line" % (rsc)] + " " + common_cmds["%s_cancel_event" % (rsc)])
	### If this happens the monitor did not actually cancel correctly. ###
	test.add_expected_fail_cmd(common_cmds["%s_monitor_event" % (rsc)], CrmExit.TIMEOUT)
	### If this happens the monitor did not actually cancel correctly. ###
	test.add_expected_fail_cmd(common_cmds["%s_monitor_event" % (rsc)], CrmExit.TIMEOUT)
	test.add_cmd(common_cmds["%s_stop_line" % (rsc)] + " " + common_cmds["%s_stop_event" % (rsc)])
	test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])

	### monitor duplicate test ###
	for rsc in self.rsc_classes:
	test = self.new_test("generic_monitor_duplicate_%s" % (rsc),
	"Test creation and canceling of duplicate monitors for %s standard" % (rsc))
	test.add_cmd(common_cmds["%s_reg_line" % (rsc)] + " " + common_cmds["%s_reg_event" % (rsc)])
	test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
	test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
	### If this fails, that means the monitor may not be getting rescheduled ####
	test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
	### If this fails, that means the monitor may not be getting rescheduled ####
	test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])

	# Add the duplicate monitors
	test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
	test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
	test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
	test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
	# verify we still get update events
	### If this fails, that means the monitor may not be getting rescheduled ####
	test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])

	# cancel the monitor, if the duplicate merged with the original, we should no longer see monitor updates
	test.add_cmd(common_cmds["%s_cancel_line" % (rsc)] + " " + common_cmds["%s_cancel_event" % (rsc)])
	### If this happens the monitor did not actually cancel correctly. ###
	test.add_expected_fail_cmd(common_cmds["%s_monitor_event" % (rsc)], CrmExit.TIMEOUT)
	### If this happens the monitor did not actually cancel correctly. ###
	test.add_expected_fail_cmd(common_cmds["%s_monitor_event" % (rsc)], CrmExit.TIMEOUT)
	test.add_cmd(common_cmds["%s_stop_line" % (rsc)] + " " + common_cmds["%s_stop_event" % (rsc)])
	test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])

	### stop implies cancel test ###
	for rsc in self.rsc_classes:
	test = self.new_test("generic_stop_implies_cancel_%s" % (rsc),
	"Verify stopping a resource implies cancel of recurring ops for %s standard" % (rsc))
	test.add_cmd(common_cmds["%s_reg_line" % (rsc)] + " " + common_cmds["%s_reg_event" % (rsc)])
	test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
	test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
	### If this fails, that means the monitor may not be getting rescheduled ####
	test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
	### If this fails, that means the monitor may not be getting rescheduled ####
	test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
	test.add_cmd(common_cmds["%s_stop_line" % (rsc)] + " " + common_cmds["%s_stop_event" % (rsc)])
	### If this happens the monitor did not actually cancel correctly. ###
	test.add_expected_fail_cmd(common_cmds["%s_monitor_event" % (rsc)], CrmExit.TIMEOUT)
	### If this happens the monitor did not actually cancel correctly. ###
	test.add_expected_fail_cmd(common_cmds["%s_monitor_event" % (rsc)], CrmExit.TIMEOUT)
	test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])


	def build_multi_rsc_tests(self):
	""" Register complex tests that involve managing multiple resouces of different types """

	common_cmds = self.common_cmds
	# do not use service and systemd at the same time, it is the same resource.

	### register start monitor stop unregister resources of each type at the same time. ###
	test = self.new_test("multi_rsc_start_stop_all",
	"Start, monitor, and stop resources of multiple types and classes")
	for rsc in self.rsc_classes:
	test.add_cmd(common_cmds["%s_reg_line" % (rsc)] + " " + common_cmds["%s_reg_event" % (rsc)])
	for rsc in self.rsc_classes:
	test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
	for rsc in self.rsc_classes:
	test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
	for rsc in self.rsc_classes:
	### If this fails, that means the monitor is not being rescheduled ####
	test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
	for rsc in self.rsc_classes:
	test.add_cmd(common_cmds["%s_cancel_line" % (rsc)] + " " + common_cmds["%s_cancel_event" % (rsc)])
	for rsc in self.rsc_classes:
	test.add_cmd(common_cmds["%s_stop_line" % (rsc)] + " " + common_cmds["%s_stop_event" % (rsc)])
	for rsc in self.rsc_classes:
	test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])

	def build_negative_tests(self):
	""" Register tests related to how pacemaker-execd handles failures """

	### ocf start timeout test ###
	test = self.new_test("ocf_start_timeout", "Force start timeout to occur, verify start failure.")
	test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "
	+ self.action_timeout +
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
	# -t must be less than self.action_timeout
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" -k \"op_sleep\" -v \"5\" -t 1000 -w")
	test.add_cmd('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:unknown error op_status:Timed Out" '
	+ self.action_timeout)
	test.add_cmd("-c exec -r test_rsc -a stop " + self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\" ")
	test.add_cmd("-c unregister_rsc -r test_rsc " + self.action_timeout +
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	### stonith start timeout test ###
	test = self.new_test("stonith_start_timeout", "Force start timeout to occur, verify start failure.")
	test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"stonith\" -P \"pacemaker\" -T \"fence_dummy_sleep\" "
	+ self.action_timeout +
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" -t 1000 -w") # -t must be less than self.action_timeout
	test.add_cmd('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:unknown error op_status:Timed Out" '
	+ self.action_timeout)
	test.add_cmd("-c exec -r test_rsc -a stop " + self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\" ")
	test.add_cmd("-c unregister_rsc -r test_rsc " + self.action_timeout +
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	### stonith component fail ###
	common_cmds = self.common_cmds
	test = self.new_test("stonith_component_fail", "Kill stonith component after pacemaker-execd connects")
	test.add_cmd(common_cmds["stonith_reg_line"] + " " + common_cmds["stonith_reg_event"])
	test.add_cmd(common_cmds["stonith_start_line"] + " " + common_cmds["stonith_start_event"])

	test.add_cmd('-c exec -r stonith_test_rsc -a monitor -i 600s '
	'-l "NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:ok op_status:complete" '
	+ self.action_timeout)

	test.add_cmd_and_kill("killall -9 -q pacemaker-fenced lt-pacemaker-fenced",
	'-l "NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:unknown error op_status:error" -t 15000')
	test.add_cmd(common_cmds["stonith_unreg_line"] + " " + common_cmds["stonith_unreg_event"])


	### monitor fail for ocf resources ###
	test = self.new_test("monitor_fail_ocf", "Force ocf monitor to fail, verify failure is reported.")
	test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "
	+ self.action_timeout +
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" " + self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" " + self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
	test.add_cmd('-c exec -r test_rsc -a monitor -i 1s '
	+ self.action_timeout +
	'-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"')
	test.add_cmd('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"'
	+ self.action_timeout)
	test.add_cmd('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"'
	+ self.action_timeout)
	test.add_cmd_and_kill("rm -f @localstatedir@/run/Dummy-test_rsc.state",
	'-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete" ' + self.action_timeout)
	test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s ' + self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
	test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete\" "
	+ self.action_timeout, CrmExit.TIMEOUT)
	test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "
	+ self.action_timeout, CrmExit.TIMEOUT)
	test.add_cmd("-c unregister_rsc -r \"test_rsc\" "
	+ self.action_timeout +
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	### verify notify changes only for monitor operation. ###
	test = self.new_test("monitor_changes_only", "Verify when flag is set, only monitor changes are notified.")
	test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+" -o "
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
	test.add_cmd('-c exec -r test_rsc -a monitor -i 1s '
	+ self.action_timeout +
	' -o -l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete" ')
	test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
	test.add_cmd_and_kill('rm -f @localstatedir@/run/Dummy-test_rsc.state', '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete"' + self.action_timeout)
	test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s'
	+ self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
	test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
	test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
	test.add_cmd('-c unregister_rsc -r "test_rsc" ' + self.action_timeout +
	'-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete"')

	### monitor fail for systemd resource ###
	if "systemd" in self.rsc_classes:
	test = self.new_test("monitor_fail_systemd", "Force systemd monitor to fail, verify failure is reported..")
	test.add_cmd("-c register_rsc -r \"test_rsc\" -C systemd -T pacemaker-cts-dummyd@3 " +
	self.action_timeout +
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
	test.add_cmd('-c exec -r test_rsc -a monitor -i 1s '
	+ self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" ")
	test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout)
	test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout)
	test.add_cmd_and_kill("killall -9 -q pacemaker-cts-dummyd",
	'-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete"' + self.action_timeout)
	test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s' + self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
	test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
	test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
	test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	### monitor fail for upstart resource ###
	if "upstart" in self.rsc_classes:
	test = self.new_test("monitor_fail_upstart", "Force upstart monitor to fail, verify failure is reported..")
	test.add_cmd("-c register_rsc -r \"test_rsc\" -C upstart -T pacemaker-cts-dummyd "+self.action_timeout+
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
	test.add_cmd('-c exec -r test_rsc -a monitor -i 1s ' + self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" ")
	test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout)
	test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout)
	test.add_cmd_and_kill('killall -9 -q dd', '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete"' + self.action_timeout)
	test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s'
	+ self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
	test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
	test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
	test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	### Cancel non-existent operation on a resource ###
	test = self.new_test("cancel_non_existent_op", "Attempt to cancel the wrong monitor operation, verify expected failure")
	test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
	test.add_cmd('-c exec -r test_rsc -a monitor -i 1s '
	+ self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" ")
	test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout)
	### interval is wrong, should fail
	test.add_expected_fail_cmd('-c cancel -r test_rsc -a monitor -i 2s' + self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
	### action name is wrong, should fail
	test.add_expected_fail_cmd('-c cancel -r test_rsc -a stop -i 1s' + self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
	test.add_cmd("-c unregister_rsc -r \"test_rsc\" " + self.action_timeout +
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	### Attempt to invoke non-existent rsc id ###
	test = self.new_test("invoke_non_existent_rsc", "Attempt to perform operations on a non-existent rsc id.")
	test.add_expected_fail_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:unknown error op_status:complete\" ")
	test.add_expected_fail_cmd("-c exec -r test_rsc -a stop "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\" ")
	test.add_expected_fail_cmd('-c exec -r test_rsc -a monitor -i 6s '
	+ self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" ")
	test.add_expected_fail_cmd("-c cancel -r test_rsc -a start "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Cancelled\" ")
	test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	### Register and start a resource that doesn't exist, systemd ###
	if "systemd" in self.rsc_classes:
	test = self.new_test("start_uninstalled_systemd", "Register uninstalled systemd agent, try to start, verify expected failure")
	test.add_cmd("-c register_rsc -r \"test_rsc\" -C systemd -T this_is_fake1234 "+self.action_timeout+
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed\" ")
	test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	if "upstart" in self.rsc_classes:
	test = self.new_test("start_uninstalled_upstart", "Register uninstalled upstart agent, try to start, verify expected failure")
	test.add_cmd("-c register_rsc -r \"test_rsc\" -C upstart -T this_is_fake1234 "+self.action_timeout+
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed\" ")
	test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	### Register and start a resource that doesn't exist, ocf ###
	test = self.new_test("start_uninstalled_ocf", "Register uninstalled ocf agent, try to start, verify expected failure.")
	test.add_cmd("-c register_rsc -r \"test_rsc\" -C ocf -P pacemaker -T this_is_fake1234 "+self.action_timeout+
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed\" ")
	test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	### Register ocf with non-existent provider ###
	test = self.new_test("start_ocf_bad_provider", "Register ocf agent with a non-existent provider, verify expected failure.")
	test.add_cmd("-c register_rsc -r \"test_rsc\" -C ocf -P pancakes -T Dummy "+self.action_timeout+
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed\" ")
	test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	### Register ocf with empty provider field ###
	test = self.new_test("start_ocf_no_provider", "Register ocf agent with a no provider, verify expected failure.")
	test.add_expected_fail_cmd("-c register_rsc -r \"test_rsc\" -C ocf -T Dummy "+self.action_timeout+
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
	test.add_expected_fail_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Error\" ")
	test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	def build_stress_tests(self):
	""" Register stress tests """

	timeout = "-t 20000"

	iterations = 25
	test = self.new_test("ocf_stress", "Verify OCF agent handling works under load")
	for i in range(iterations):
	test.add_cmd("-c register_rsc -r rsc_%s %s -C ocf -P heartbeat -T Dummy -l \"NEW_EVENT event_type:register rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))
	test.add_cmd("-c exec -r rsc_%s -a start %s -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:start rc:ok op_status:complete\"" % (i, timeout, i))
	test.add_cmd('-c exec -r rsc_%s -a monitor %s -i 1s '
	'-l "NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:monitor rc:ok op_status:complete"' % (i, timeout, i))
	for i in range(iterations):
	test.add_cmd("-c exec -r rsc_%s -a stop %s -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:stop rc:ok op_status:complete\"" % (i, timeout, i))
	test.add_cmd("-c unregister_rsc -r rsc_%s %s -l \"NEW_EVENT event_type:unregister rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))


	if "systemd" in self.rsc_classes:
	test = self.new_test("systemd_stress", "Verify systemd dbus connection works under load")
	for i in range(iterations):
	test.add_cmd("-c register_rsc -r rsc_%s %s -C systemd -T pacemaker-cts-dummyd@3 -l \"NEW_EVENT event_type:register rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))
	test.add_cmd("-c exec -r rsc_%s -a start %s -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:start rc:ok op_status:complete\"" % (i, timeout, i))
	test.add_cmd('-c exec -r rsc_%s -a monitor %s -i 1s '
	'-l "NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:monitor rc:ok op_status:complete"' % (i, timeout, i))

	for i in range(iterations):
	test.add_cmd("-c exec -r rsc_%s -a stop %s -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:stop rc:ok op_status:complete\"" % (i, timeout, i))
	test.add_cmd("-c unregister_rsc -r rsc_%s %s -l \"NEW_EVENT event_type:unregister rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))

	iterations = 9
	timeout = "-t 30000"
	### Verify recurring op in-flight collision is handled in series properly
	test = self.new_test("rsc_inflight_collision", "Verify recurring ops do not collide with other operations for the same rsc.")
	test.add_cmd("-c register_rsc -r test_rsc -P pacemaker -C ocf -T Dummy "
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" "+self.action_timeout)
	test.add_cmd("-c exec -r test_rsc -a start %s -k op_sleep -v 1 -l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\"" % (timeout))
	for i in range(iterations):
	test.add_cmd('-c exec -r test_rsc -a monitor %s -i 100%dms '
	'-k op_sleep -v 2 '
	'-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"' % (timeout, i))

	test.add_cmd("-c exec -r test_rsc -a stop %s -l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\"" % (timeout))
	test.add_cmd("-c unregister_rsc -r test_rsc %s -l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\"" % (timeout))

	def build_custom_tests(self):
	""" Register tests that target specific cases """

	### verify resource temporary folder is created and used by OCF agents. ###
	test = self.new_test("rsc_tmp_dir", "Verify creation and use of rsc temporary state directory")
	test.add_sys_cmd("ls", "-al @CRM_RSCTMP_DIR@")
	test.add_cmd("-c register_rsc -r test_rsc -P heartbeat -C ocf -T Dummy "
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" "+self.action_timeout)
	test.add_cmd("-c exec -r test_rsc -a start -t 4000")
	test.add_sys_cmd("ls", "-al @CRM_RSCTMP_DIR@")
	test.add_sys_cmd("ls", "@CRM_RSCTMP_DIR@/Dummy-test_rsc.state")
	test.add_cmd("-c exec -r test_rsc -a stop -t 4000")
	test.add_cmd("-c unregister_rsc -r test_rsc "+self.action_timeout+
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	### start delay then stop test ###
	test = self.new_test("start_delay", "Verify start delay works as expected.")
	test.add_cmd("-c register_rsc -r test_rsc -P pacemaker -C ocf -T Dummy "
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" "+self.action_timeout)
	test.add_cmd("-c exec -r test_rsc -s 6000 -a start -w -t 6000")
	test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" -t 2000", CrmExit.TIMEOUT)
	test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" -t 6000")
	test.add_cmd("-c exec -r test_rsc -a stop " + self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\" ")
	test.add_cmd("-c unregister_rsc -r test_rsc " + self.action_timeout +
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	### start delay, but cancel before it gets a chance to start. ###
	test = self.new_test("start_delay_cancel", "Using start_delay, start a rsc, but cancel the start op before execution.")
	test.add_cmd("-c register_rsc -r test_rsc -P pacemaker -C ocf -T Dummy "
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" "+self.action_timeout)
	test.add_cmd("-c exec -r test_rsc -s 5000 -a start -w -t 4000")
	test.add_cmd("-c cancel -r test_rsc -a start " + self.action_timeout +
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Cancelled\" ")
	test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" -t 5000", CrmExit.TIMEOUT)
	test.add_cmd("-c unregister_rsc -r test_rsc " + self.action_timeout +
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	### Register a bunch of resources, verify we can get info on them ###
	test = self.new_test("verify_get_rsc_info", "Register multiple resources, verify retrieval of rsc info.")
	if "systemd" in self.rsc_classes:
	test.add_cmd("-c register_rsc -r rsc1 -C systemd -T pacemaker-cts-dummyd@3 "+self.action_timeout)
	test.add_cmd("-c get_rsc_info -r rsc1 ")
	test.add_cmd("-c unregister_rsc -r rsc1 "+self.action_timeout)
	test.add_expected_fail_cmd("-c get_rsc_info -r rsc1 ")

	if "upstart" in self.rsc_classes:
	test.add_cmd("-c register_rsc -r rsc1 -C upstart -T pacemaker-cts-dummyd "+self.action_timeout)
	test.add_cmd("-c get_rsc_info -r rsc1 ")
	test.add_cmd("-c unregister_rsc -r rsc1 "+self.action_timeout)
	test.add_expected_fail_cmd("-c get_rsc_info -r rsc1 ")

	test.add_cmd("-c register_rsc -r rsc2 -C ocf -T Dummy -P pacemaker "+self.action_timeout)
	test.add_cmd("-c get_rsc_info -r rsc2 ")
	test.add_cmd("-c unregister_rsc -r rsc2 "+self.action_timeout)
	test.add_expected_fail_cmd("-c get_rsc_info -r rsc2 ")

	### Register duplicate, verify only one entry exists and can still be removed.
	test = self.new_test("duplicate_registration", "Register resource multiple times, verify only one entry exists and can be removed.")
	test.add_cmd("-c register_rsc -r rsc2 -C ocf -T Dummy -P pacemaker "+self.action_timeout)
	test.add_cmd_check_stdout("-c get_rsc_info -r rsc2 ", "id:rsc2 class:ocf provider:pacemaker type:Dummy")
	test.add_cmd("-c register_rsc -r rsc2 -C ocf -T Dummy -P pacemaker "+self.action_timeout)
	test.add_cmd_check_stdout("-c get_rsc_info -r rsc2 ", "id:rsc2 class:ocf provider:pacemaker type:Dummy")
	test.add_cmd("-c register_rsc -r rsc2 -C ocf -T Stateful -P pacemaker "+self.action_timeout)
	test.add_cmd_check_stdout("-c get_rsc_info -r rsc2 ", "id:rsc2 class:ocf provider:pacemaker type:Stateful")
	test.add_cmd("-c unregister_rsc -r rsc2 "+self.action_timeout)
	test.add_expected_fail_cmd("-c get_rsc_info -r rsc2 ")

	### verify the option to only send notification to the original client. ###
	test = self.new_test("notify_orig_client_only", "Verify option to only send notifications to the client originating the action.")
	test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
	test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
	test.add_cmd('-c exec -r \"test_rsc\" -a \"monitor\" -i 1s '
	+ self.action_timeout + ' -n '
	'-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"')
	# this will fail because the monitor notifications should only go to the original caller, which no longer exists.
	test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
	test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s -t 6000 ')
	test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
	"-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")

	### get metadata ###
	test = self.new_test("get_ocf_metadata", "Retrieve metadata for a resource")
	test.add_cmd_check_stdout("-c metadata -C \"ocf\" -P \"pacemaker\" -T \"Dummy\"",
	"resource-agent name=\"Dummy\"")
	test.add_cmd("-c metadata -C \"ocf\" -P \"pacemaker\" -T \"Stateful\"")
	test.add_expected_fail_cmd("-c metadata -P \"pacemaker\" -T \"Stateful\"")
	test.add_expected_fail_cmd("-c metadata -C \"ocf\" -P \"pacemaker\" -T \"fake_agent\"")

	### get metadata ###
	test = self.new_test("get_lsb_metadata", "Retrieve metadata for a resource")
	test.add_cmd_check_stdout("-c metadata -C \"lsb\" -T \"LSBDummy\"",
	"resource-agent name='LSBDummy'")

	### get stonith metadata ###
	test = self.new_test("get_stonith_metadata", "Retrieve stonith metadata for a resource")
	test.add_cmd_check_stdout("-c metadata -C \"stonith\" -P \"pacemaker\" -T \"fence_dummy_monitor\"",
	"resource-agent name=\"fence_dummy_monitor\"")

	### get metadata ###
	if "systemd" in self.rsc_classes:
	test = self.new_test("get_systemd_metadata", "Retrieve metadata for a resource")
	test.add_cmd_check_stdout("-c metadata -C \"systemd\" -T \"pacemaker-cts-dummyd@\"",
	"resource-agent name=\"pacemaker-cts-dummyd@\"")

	### get metadata ###
	if "upstart" in self.rsc_classes:
	test = self.new_test("get_upstart_metadata", "Retrieve metadata for a resource")
	test.add_cmd_check_stdout("-c metadata -C \"upstart\" -T \"pacemaker-cts-dummyd\"",
	"resource-agent name=\"pacemaker-cts-dummyd\"")

	### get ocf providers ###
	test = self.new_test("list_ocf_providers",
	"Retrieve list of available resource providers, verifies pacemaker is a provider.")
	test.add_cmd_check_stdout("-c list_ocf_providers ", "pacemaker")
	test.add_cmd_check_stdout("-c list_ocf_providers -T ping", "pacemaker")

	### Verify agents only exist in their lists ###
	test = self.new_test("verify_agent_lists", "Verify the agent lists contain the right data.")
	test.add_cmd_check_stdout("-c list_agents ", "Stateful") ### ocf ###
	test.add_cmd_check_stdout("-c list_agents -C ocf", "Stateful")
	test.add_cmd_check_stdout("-c list_agents -C lsb", "", "Stateful") ### should not exist
	test.add_cmd_check_stdout("-c list_agents -C service", "", "Stateful") ### should not exist
	test.add_cmd_check_stdout("-c list_agents ", "LSBDummy") ### init.d ###
	test.add_cmd_check_stdout("-c list_agents -C lsb", "LSBDummy")
	test.add_cmd_check_stdout("-c list_agents -C service", "LSBDummy")
	test.add_cmd_check_stdout("-c list_agents -C ocf", "", "pacemaker-cts-dummyd@") ### should not exist

	test.add_cmd_check_stdout("-c list_agents -C ocf", "", "pacemaker-cts-dummyd@") ### should not exist
	test.add_cmd_check_stdout("-c list_agents -C lsb", "", "fence_dummy_monitor") ### should not exist
	test.add_cmd_check_stdout("-c list_agents -C service", "", "fence_dummy_monitor") ### should not exist
	test.add_cmd_check_stdout("-c list_agents -C ocf", "", "fence_dummy_monitor") ### should not exist

	if "systemd" in self.rsc_classes:
	test.add_cmd_check_stdout("-c list_agents ", "pacemaker-cts-dummyd@") ### systemd ###
	test.add_cmd_check_stdout("-c list_agents -C service", "LSBDummy")
	test.add_cmd_check_stdout("-c list_agents -C systemd", "", "Stateful") ### should not exist
	test.add_cmd_check_stdout("-c list_agents -C systemd", "pacemaker-cts-dummyd@")
	test.add_cmd_check_stdout("-c list_agents -C systemd", "", "fence_dummy_monitor") ### should not exist

	if "upstart" in self.rsc_classes:
	test.add_cmd_check_stdout("-c list_agents ", "pacemaker-cts-dummyd") ### upstart ###
	test.add_cmd_check_stdout("-c list_agents -C service", "LSBDummy")
	test.add_cmd_check_stdout("-c list_agents -C upstart", "", "Stateful") ### should not exist
	test.add_cmd_check_stdout("-c list_agents -C upstart", "pacemaker-cts-dummyd")
	test.add_cmd_check_stdout("-c list_agents -C upstart", "", "fence_dummy_monitor") ### should not exist

	if "stonith" in self.rsc_classes:
	test.add_cmd_check_stdout("-c list_agents -C stonith", "fence_dummy_monitor") ### stonith ###
	test.add_cmd_check_stdout("-c list_agents -C stonith", "", "pacemaker-cts-dummyd@") ### should not exist
	test.add_cmd_check_stdout("-c list_agents -C stonith", "", "Stateful") ### should not exist
	test.add_cmd_check_stdout("-c list_agents ", "fence_dummy_monitor")

	def print_list(self):
	""" List all registered tests """

	print("\n==== %d TESTS FOUND ====" % (len(self.tests)))
	print("%35s - %s" % ("TEST NAME", "TEST DESCRIPTION"))
	print("%35s - %s" % ("--------------------", "--------------------"))
	for test in self.tests:
	print("%35s - %s" % (test.name, test.description))
	print("==== END OF LIST ====\n")

	def run_single(self, name):
	""" Run a single named test """

	for test in self.tests:
	if test.name == name:
	test.run()
	break

	def run_tests_matching(self, pattern):
	""" Run all tests whose name matches a pattern """

	for test in self.tests:
	if test.name.count(pattern) != 0:
	test.run()

	def run_tests(self):
	""" Run all tests """

	for test in self.tests:
	test.run()

	def exit(self):
	""" Exit (with error status code if any test failed) """

	for test in self.tests:
	if test.executed == 0:
	continue

	if test.get_exitcode() != CrmExit.OK:
	sys.exit(CrmExit.ERROR)

	sys.exit(CrmExit.OK)

	def print_results(self):
	""" Print summary of results of executed tests """

	failures = 0
	success = 0
	print("\n\n======= FINAL RESULTS ==========")
	print("\n--- FAILURE RESULTS:")
	for test in self.tests:
	if test.executed == 0:
	continue

	if test.get_exitcode() != CrmExit.OK:
	failures = failures + 1
	test.print_result(" ")
	else:
	success = success + 1

	if failures == 0:
	print(" None")

	print("\n--- TOTALS\n Pass:%d\n Fail:%d\n" % (success, failures))


	class TestOptions(object):
	""" Option handler """

	def __init__(self):
	self.options = {}
	self.options['list-tests'] = 0
	self.options['run-all'] = 1
	self.options['run-only'] = ""
	self.options['run-only-pattern'] = ""
	self.options['verbose'] = 0
	self.options['invalid-arg'] = ""
	self.options['show-usage'] = 0
	self.options['pacemaker-remote'] = 0

	def build_options(self, argv):
	""" Set options based on command-line arguments """

	args = argv[1:]
	skip = 0
	for i in range(0, len(args)):
	if skip:
	skip = 0
	continue
	elif args[i] == "-h" or args[i] == "--help":
	self.options['show-usage'] = 1
	elif args[i] == "-l" or args[i] == "--list-tests":
	self.options['list-tests'] = 1
	elif args[i] == "-V" or args[i] == "--verbose":
	self.options['verbose'] = 1
	elif args[i] == "-R" or args[i] == "--pacemaker-remote":
	self.options['pacemaker-remote'] = 1
	elif args[i] == "-r" or args[i] == "--run-only":
	self.options['run-only'] = args[i+1]
	skip = 1
	elif args[i] == "-p" or args[i] == "--run-only-pattern":
	self.options['run-only-pattern'] = args[i+1]
	skip = 1

	def show_usage(self):
	""" Show command usage """

	print("usage: " + sys.argv[0] + " [options]")
	print("If no options are provided, all tests will run")
	print("Options:")
	print("\t [--help \| -h] Show usage")
	print("\t [--list-tests \| -l] Print out all registered tests.")
	print("\t [--run-only \| -r 'testname'] Run a specific test")
	print("\t [--verbose \| -V] Verbose output")
	print("\t [--pacemaker-remote \| -R Test pacemaker-remoted binary instead of pacemaker-execd")
	print("\t [--run-only-pattern \| -p 'string'] Run only tests containing the string value")
	print("\n\tExample: Run only the test 'start_stop'")
	print("\t\t " + sys.argv[0] + " --run-only start_stop")
	print("\n\tExample: Run only the tests with the string 'systemd' present in them")
	print("\t\t " + sys.argv[0] + " --run-only-pattern systemd")


	def main(argv):
	""" Run pacemaker-execd regression tests as specified by arguments """

	update_path()

	opts = TestOptions()
	opts.build_options(argv)

	tests = Tests(opts.options['verbose'], opts.options['pacemaker-remote'])

	tests.build_generic_tests()
	tests.build_multi_rsc_tests()
	tests.build_negative_tests()
	tests.build_custom_tests()
	tests.build_stress_tests()

	tests.setup_test_environment()

	print("Starting ...")

	if opts.options['list-tests']:
	tests.print_list()
	elif opts.options['show-usage']:
	opts.show_usage()
	elif opts.options['run-only-pattern'] != "":
	tests.run_tests_matching(opts.options['run-only-pattern'])
	tests.print_results()
	elif opts.options['run-only'] != "":
	tests.run_single(opts.options['run-only'])
	tests.print_results()
	else:
	tests.run_tests()
	tests.print_results()

	tests.cleanup_test_environment()
	tests.exit()


	if __name__ == "__main__":
	main(sys.argv)
	diff --git a/extra/resources/ClusterMon b/extra/resources/ClusterMon.in
	similarity index 99%
	rename from extra/resources/ClusterMon
	rename to extra/resources/ClusterMon.in
	index 0604a26b5f..04fb43f1e6 100755
	--- a/extra/resources/ClusterMon
	+++ b/extra/resources/ClusterMon.in
	@@ -1,276 +1,276 @@
	-#!/bin/bash
	+#!@BASH_PATH@
	#
	#
	# ClusterMon OCF RA.
	# Starts crm_mon in background which logs cluster status as
	# html to the specified file.
	#
	# Copyright (c) 2004 SUSE LINUX AG, Lars Marowsky-Br馥
	# All Rights Reserved.
	#
	# This program is free software; you can redistribute it and/or modify
	# it under the terms of version 2 of the GNU General Public License as
	# published by the Free Software Foundation.
	#
	# This program is distributed in the hope that it would be useful, but
	# WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
	#
	# Further, this software is distributed without any warranty that it is
	# free of the rightful claim of any third person regarding infringement
	# or the like. Any license provided herein, whether implied or
	# otherwise, applies only to this software file. Patent licenses, if
	# any, provided herein do not apply to combinations of this program with
	# other software, or any other product whatsoever.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write the Free Software Foundation,
	# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
	#
	# OCF instance parameters:
	# OCF_RESKEY_user
	# OCF_RESKEY_pidfile
	# OCF_RESKEY_update
	# OCF_RESKEY_extra_options
	# OCF_RESKEY_htmlfile

	#######################################################################
	# Initialization:
	: ${OCF_FUNCTIONS=${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs}
	. ${OCF_FUNCTIONS}
	: ${__OCF_ACTION=$1}

	#######################################################################

	meta_data() {
	cat <<END
	<?xml version="1.0"?>
	<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
	<resource-agent name="ClusterMon" version="1.0">
	<version>1.0</version>

	<longdesc lang="en">
	This is a ClusterMon Resource Agent.
	It outputs current cluster status to the html.
	</longdesc>
	<shortdesc lang="en">Runs crm_mon in the background, recording the cluster status to an HTML file</shortdesc>

	<parameters>

	<parameter name="user" unique="0">
	<longdesc lang="en">
	The user we want to run crm_mon as
	</longdesc>
	<shortdesc lang="en">The user we want to run crm_mon as</shortdesc>
	<content type="string" default="root" />
	</parameter>

	<parameter name="update" unique="0">
	<longdesc lang="en">
	How frequently should we update the cluster status (in milliseconds).
	For compatibility with old documentation, values less than 1000 will be treated
	as seconds.
	</longdesc>
	<shortdesc lang="en">Update interval in milliseconds</shortdesc>
	<content type="integer" default="15000" />
	</parameter>

	<parameter name="extra_options" unique="0">
	<longdesc lang="en">
	Additional options to pass to crm_mon. Eg. -n -r
	</longdesc>
	<shortdesc lang="en">Extra options</shortdesc>
	<content type="string" default="" />
	</parameter>

	<parameter name="pidfile" unique="1">
	<longdesc lang="en">
	PID file location to ensure only one instance is running
	</longdesc>
	<shortdesc lang="en">PID file</shortdesc>
	<content type="string" default="/tmp/ClusterMon_${OCF_RESOURCE_INSTANCE}.pid" />
	</parameter>

	<parameter name="htmlfile" unique="1" required="0">
	<longdesc lang="en">
	Location to write HTML output to.
	</longdesc>
	<shortdesc lang="en">HTML output</shortdesc>
	<content type="string" default="/tmp/ClusterMon_${OCF_RESOURCE_INSTANCE}.html" />
	</parameter>
	</parameters>

	<actions>
	<action name="start" timeout="20" />
	<action name="stop" timeout="20" />
	<action name="monitor" depth="0" timeout="20" interval="10" />
	<action name="meta-data" timeout="5" />
	<action name="validate-all" timeout="30" />
	</actions>
	</resource-agent>
	END
	}

	#######################################################################

	ClusterMon_usage() {
	cat <<END
	usage: $0 {start\|stop\|monitor\|validate-all\|meta-data}

	Expects to have a fully populated OCF RA-compliant environment set.
	END
	}

	ClusterMon_exit() {
	if [ $1 != 0 ]; then
	exit $OCF_ERR_GENERIC
	else
	exit $OCF_SUCCESS
	fi
	}

	ClusterMon_start() {
	cmd_prefix=""
	cmd_suffix=""
	if [ ! -z $OCF_RESKEY_user ]; then
	su - $OCF_RESKEY_user -c "$CMON_CMD"
	else
	$CMON_CMD
	fi
	ClusterMon_exit $?
	}

	ClusterMon_stop() {
	if [ -f $OCF_RESKEY_pidfile ]; then
	pid=`cat $OCF_RESKEY_pidfile`
	if [ ! -z $pid ]; then
	kill -s 9 $pid
	rm -f $OCF_RESKEY_pidfile
	fi
	fi
	ClusterMon_exit 0
	}

	ClusterMon_monitor() {
	if [ -f $OCF_RESKEY_pidfile ]; then
	pid=`cat $OCF_RESKEY_pidfile`
	if [ ! -z $pid ]; then
	str=$(echo "su - $OCF_RESKEY_user -c \"$CMON_CMD\"" \| tr 'crmon, \t' 'xxxxxxxx')
	ps -o "args=${str}" -p $pid 2>/dev/null \| \
	grep -qE "[c]rm_mon.*${OCF_RESKEY_pidfile}"
	rc=$?
	case $rc in
	0) exit $OCF_SUCCESS;;
	1) exit $OCF_NOT_RUNNING;;
	*) exit $OCF_ERR_GENERIC;;
	esac
	fi
	fi
	exit $OCF_NOT_RUNNING
	}

	CheckOptions() {
	while getopts Vi:nrh:cdp: OPTION
	do
	case $OPTION in
	V\|n\|r\|c\|d);;
	i) ocf_log warn "You should not have specified the -i option, since OCF_RESKEY_update is set already!";;
	h) ocf_log warn "You should not have specified the -h option, since OCF_RESKEY_htmlfile is set already!";;
	p) ocf_log warn "You should not have specified the -p option, since OCF_RESKEY_pidfile is set already!";;
	*) return $OCF_ERR_ARGS;;
	esac
	done

	if [ $? -ne 0 ]; then
	return $OCF_ERR_ARGS
	fi

	# We should have eaten all options at this stage
	shift $(($OPTIND -1))
	if [ $# -gt 0 ]; then
	false
	else
	true
	fi
	}

	ClusterMon_validate() {
	# Existence of the user
	if [ ! -z $OCF_RESKEY_user ]; then
	getent passwd "$OCF_RESKEY_user" >/dev/null
	if [ $? -eq 0 ]; then
	: Yes, user exists. We can further check his permission on crm_mon if necessary
	else
	ocf_log err "The user $OCF_RESKEY_user does not exist!"
	exit $OCF_ERR_ARGS
	fi
	fi

	# Pidfile better be an absolute path
	case $OCF_RESKEY_pidfile in
	/*) ;;
	*) ocf_log warn "You should have pidfile($OCF_RESKEY_pidfile) of absolute path!" ;;
	esac

	# Check the update interval
	if ocf_is_decimal "$OCF_RESKEY_update" && [ $OCF_RESKEY_update -gt 0 ]; then
	:
	else
	ocf_log err "Invalid update interval $OCF_RESKEY_update. It should be positive integer!"
	exit $OCF_ERR_ARGS
	fi

	if CheckOptions $OCF_RESKEY_extra_options; then
	:
	else
	ocf_log err "Invalid options $OCF_RESKEY_extra_options!"
	exit $OCF_ERR_ARGS
	fi

	# Htmlfile better be an absolute path
	case $OCF_RESKEY_htmlfile in
	/*) ;;
	*) ocf_log warn "You should have htmlfile($OCF_RESKEY_htmlfile) of absolute path!" ;;
	esac


	echo "Validate OK"
	return $OCF_SUCCESS
	}

	if [ $# -ne 1 ]; then
	ClusterMon_usage
	exit $OCF_ERR_ARGS
	fi

	: ${OCF_RESKEY_update:="15000"}
	: ${OCF_RESKEY_pidfile:="/tmp/ClusterMon_${OCF_RESOURCE_INSTANCE}.pid"}
	: ${OCF_RESKEY_htmlfile:="/tmp/ClusterMon_${OCF_RESOURCE_INSTANCE}.html"}

	if [ ${OCF_RESKEY_update} -ge 1000 ]; then
	OCF_RESKEY_update=$(( $OCF_RESKEY_update / 1000 ))
	fi

	CMON_CMD="${HA_SBIN_DIR}/crm_mon -p $OCF_RESKEY_pidfile -d -i $OCF_RESKEY_update $OCF_RESKEY_extra_options -h $OCF_RESKEY_htmlfile"

	case $__OCF_ACTION in
	meta-data) meta_data
	exit $OCF_SUCCESS
	;;
	start) ClusterMon_start
	;;
	stop) ClusterMon_stop
	;;
	monitor) ClusterMon_monitor
	;;
	validate-all) ClusterMon_validate
	;;
	usage\|help) ClusterMon_usage
	exit $OCF_SUCCESS
	;;
	*) ClusterMon_usage
	exit $OCF_ERR_UNIMPLEMENTED
	;;
	esac

	exit $?
	diff --git a/extra/resources/HealthSMART b/extra/resources/HealthSMART.in
	similarity index 99%
	rename from extra/resources/HealthSMART
	rename to extra/resources/HealthSMART.in
	index 3db92adcac..9e91298a9e 100755
	--- a/extra/resources/HealthSMART
	+++ b/extra/resources/HealthSMART.in
	@@ -1,328 +1,328 @@
	-#!/bin/bash
	+#!@BASH_PATH@
	#
	#
	# HealthSMART OCF RA. Checks the S.M.A.R.T. status of all given
	# drives and writes the #health-smart status into the CIB
	#
	# Copyright (c) 2009 Michael Schwartzkopff, 2010 Matthew Richardson
	#
	# All Rights Reserved.
	#
	# This program is free software; you can redistribute it and/or modify
	# it under the terms of version 2 of the GNU General Public License as
	# published by the Free Software Foundation.
	#
	# This program is distributed in the hope that it would be useful, but
	# WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
	#
	# Further, this software is distributed without any warranty that it is
	# free of the rightful claim of any third person regarding infringement
	# or the like. Any license provided herein, whether implied or
	# otherwise, applies only to this software file. Patent licenses, if
	# any, provided herein do not apply to combinations of this program with
	# other software, or any other product whatsoever.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write the Free Software Foundation,
	# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
	#
	#######################################################################

	#######################################################################
	# Initialization:

	: ${OCF_FUNCTIONS=${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs}
	. ${OCF_FUNCTIONS}
	: ${__OCF_ACTION=$1}
	#
	SMARTCTL=/usr/sbin/smartctl
	ATTRDUP=/usr/sbin/attrd_updater

	#######################################################################

	meta_data() {
	cat <<END
	<?xml version="1.0"?>
	<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
	<resource-agent name="HealthSMART" version="0.1">
	<version>1.0</version>

	<longdesc lang="en">
	Systhem health agent that checks the S.M.A.R.T. status of the given drives and
	updates the #health-smart attribute.
	</longdesc>
	<shortdesc lang="en">SMART health status</shortdesc>

	<parameters>
	<parameter name="state" unique="1">
	<longdesc lang="en">
	Location to store the resource state in.
	</longdesc>
	<shortdesc lang="en">State file</shortdesc>
	<content type="string" default="${HA_VARRUN%%/}/HealthSMART-${OCF_RESOURCE_INSTANCE}.state" />
	</parameter>

	<parameter name="drives" unique="0">
	<longdesc lang="en">
	The drive(s) to check as a SPACE separated list. Enter the full path to the device, e.g. "/dev/sda".
	</longdesc>
	<shortdesc lang="en">Drives to check</shortdesc>
	<content type="string" default="/dev/sda" />
	</parameter>

	<parameter name="devices" unique="0">
	<longdesc lang="en">
	The device type(s) to assume for the drive(s) being tested as a SPACE separated list.
	</longdesc>
	<shortdesc lang="en">Device types</shortdesc>
	<content type="string" />
	</parameter>

	<parameter name="temp_lower_limit" unique="0">
	<longdesc lang="en">
	Lower limit of the temperature in deg C of the drive(s). Below this limit the status will be red.
	</longdesc>
	<shortdesc lang="en">Lower limit for the red smart attribute</shortdesc>
	<content type="string" default="0"/>
	</parameter>

	<parameter name="temp_upper_limit" unique="0">
	<longdesc lang="en">
	Upper limit of the temperature if deg C of the drives(s). If the drive reports
	a temperature higher than this value the status of #health-smart will be red.
	</longdesc>
	<shortdesc lang="en">Upper limit for red smart attribute</shortdesc>
	<content type="string" default="60"/>
	</parameter>

	<parameter name="temp_warning" unique="0">
	<longdesc lang="en">
	Number of deg C below/above the upper/lower temp limits at which point the status of #health-smart will change to yellow.
	</longdesc>
	<shortdesc lang="en">Deg C below/above the upper limits for yellow smart attribute</shortdesc>
	<content type="string" default="5"/>
	</parameter>

	</parameters>

	<actions>
	<action name="start" timeout="10" />
	<action name="stop" timeout="10" />
	<action name="monitor" timeout="10" interval="10" start-delay="0" />
	<action name="meta-data" timeout="5" />
	<action name="validate-all" timeout="10" />
	</actions>
	</resource-agent>
	END
	}

	#######################################################################

	check_temperature() {

	if [ $1 -lt ${lower_red_limit} ] ; then
	ocf_log info "Drive ${DRIVE} ${DEVICE} too cold: ${1} C"
	$ATTRDUP -n "#health-smart" -U "red" -d "5s"
	return 1
	fi

	if [ $1 -gt ${upper_red_limit} ] ; then
	ocf_log info "Drive ${DRIVE} ${DEVICE} too hot: ${1} C"
	$ATTRDUP -n "#health-smart" -U "red" -d "5s"
	return 1
	fi

	if [ $1 -lt ${lower_yellow_limit} ] ; then
	ocf_log info "Drive ${DRIVE} ${DEVICE} quite cold: ${1} C"
	$ATTRDUP -n "#health-smart" -U "yellow" -d "5s"
	return 1
	fi

	if [ $1 -gt ${upper_yellow_limit} ] ; then
	ocf_log info "Drive ${DRIVE} ${DEVICE} quite hot: ${1} C"
	$ATTRDUP -n "#health-smart" -U "yellow" -d "5s"
	return 1
	fi
	}


	init_smart() {
	#Set temperature defaults
	if [ -z ${OCF_RESKEY_temp_warning} ]; then
	yellow_threshold=5
	else
	yellow_threshold=${OCF_RESKEY_temp_warning}
	fi

	if [ -z ${OCF_RESKEY_temp_lower_limit} ] ; then
	lower_red_limit=0
	else
	lower_red_limit=${OCF_RESKEY_temp_lower_limit}
	fi
	lower_yellow_limit=$((${lower_red_limit}+${yellow_threshold}))

	if [ -z ${OCF_RESKEY_temp_upper_limit} ] ; then
	upper_red_limit=60
	else
	upper_red_limit=${OCF_RESKEY_temp_upper_limit}
	fi
	upper_yellow_limit=$((${upper_red_limit}-${yellow_threshold}))

	#Set disk defaults
	if [ -z "${OCF_RESKEY_drives}" ] ; then
	DRIVES="/dev/sda"
	else
	DRIVES=${OCF_RESKEY_drives}
	fi

	#Test for presence of smartctl
	if [ ! -x $SMARTCTL ] ; then
	ocf_log err "${SMARTCTL} not installed."
	exit $OCF_ERR_INSTALLED
	fi

	for DRIVE in $DRIVES; do
	if [ "${OCF_RESKEY_devices}" ]; then
	for DEVICE in ${OCF_RESKEY_devices}; do
	$SMARTCTL -d $DEVICE -i ${DRIVE} \| grep -q "SMART support is: Enabled"
	if [ $? -ne "0" ] ; then
	ocf_log err "S.M.A.R.T. not enabled for drive "${DRIVE}
	exit $OCF_ERR_INSTALLED
	fi
	done
	else
	$SMARTCTL -i ${DRIVE} \| grep -q "SMART support is: Enabled"
	if [ $? -ne "0" ] ; then
	ocf_log err "S.M.A.R.T. not enabled for drive "${DRIVE}
	exit $OCF_ERR_INSTALLED
	fi
	fi
	done
	}

	HealthSMART_usage() {
	cat <<END
	usage: $0 {start\|stop\|monitor\|validate-all\|meta-data}

	Expects to have a fully populated OCF RA-compliant environment set.
	END
	}

	HealthSMART_start() {
	HealthSMART_monitor
	if [ $? = $OCF_SUCCESS ]; then
	return $OCF_SUCCESS
	fi
	touch ${OCF_RESKEY_state}
	}

	HealthSMART_stop() {
	HealthSMART_monitor
	if [ $? = $OCF_SUCCESS ]; then
	rm ${OCF_RESKEY_state}
	fi
	return $OCF_SUCCESS
	}

	HealthSMART_monitor() {

	init_smart

	# Monitor _MUST!_ differentiate correctly between running
	# (SUCCESS), failed (ERROR) or _cleanly_ stopped (NOT RUNNING).
	# That is THREE states, not just yes/no.

	if [ -f ${OCF_RESKEY_state} ]; then

	# Check overall S.M.A.R.T. status
	for DRIVE in $DRIVES; do
	if [ "${OCF_RESKEY_devices}" ]; then
	for DEVICE in ${OCF_RESKEY_devices}; do
	$SMARTCTL -d $DEVICE -H ${DRIVE} \| grep -q "SMART overall-health self-assessment test result: PASSED"
	if [ $? -ne "0" ]; then
	$ATTRDUP -n "#health-smart" -U "red" -d "5s"
	return $OCF_SUCCESS
	fi
	done
	else
	$SMARTCTL -H ${DRIVE} \| grep -q "SMART overall-health self-assessment test result: PASSED"
	if [ $? -ne "0" ]; then
	$ATTRDUP -n "#health-smart" -U "red" -d "5s"
	return $OCF_SUCCESS
	fi
	fi

	# Check drive temperature(s)
	if [ "${OCF_RESKEY_devices}" ]; then
	for DEVICE in ${OCF_RESKEY_devices}; do
	check_temperature `$SMARTCTL -d $DEVICE -A ${DRIVE} \| awk '/^194/ { print $10 }'`
	if [ $? != 0 ]; then
	return $OCF_SUCCESS
	fi
	done
	else
	check_temperature `$SMARTCTL -A ${DRIVE} \| awk '/^194/ { print $10 }'`
	if [ $? != 0 ]; then
	return $OCF_SUCCESS
	fi
	fi
	done

	$ATTRDUP -n "#health-smart" -U "green" -d "5s"
	return $OCF_SUCCESS
	fi

	return $OCF_NOT_RUNNING

	}

	HealthSMART_validate() {

	init_smart

	# Is the state directory writable?
	state_dir=`dirname "$OCF_RESKEY_state"`
	touch "$state_dir/$$"
	if [ $? != 0 ]; then
	return $OCF_ERR_ARGS
	fi
	rm "$state_dir/$$"

	return $OCF_SUCCESS
	}

	: ${OCF_RESKEY_CRM_meta_interval=0}
	: ${OCF_RESKEY_CRM_meta_globally_unique:="true"}

	if [ "x$OCF_RESKEY_state" = "x" ]; then
	if [ ${OCF_RESKEY_CRM_meta_globally_unique} = "false" ]; then
	state="${HA_VARRUN%%/}/HealthSMART-${OCF_RESOURCE_INSTANCE}.state"

	# Strip off the trailing clone marker
	OCF_RESKEY_state=`echo $state \| sed s/:[0-9][0-9]*\.state/.state/`
	else
	OCF_RESKEY_state="${HA_VARRUN%%/}/HealthSMART-${OCF_RESOURCE_INSTANCE}.state"
	fi
	fi

	case $__OCF_ACTION in
	start) HealthSMART_start;;
	stop) HealthSMART_stop;;
	monitor) HealthSMART_monitor;;
	validate-all) HealthSMART_validate;;
	meta-data)
	meta_data
	exit $OCF_SUCCESS
	;;
	usage\|help)
	HealthSMART_usage
	exit $OCF_SUCCESS
	;;
	*) HealthSMART_usage
	exit $OCF_ERR_UNIMPLEMENTED
	;;
	esac
	rc=$?
	ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc"
	exit $rc
	diff --git a/extra/resources/SysInfo b/extra/resources/SysInfo.in
	similarity index 99%
	rename from extra/resources/SysInfo
	rename to extra/resources/SysInfo.in
	index 7d1c0a3054..24c259cc1b 100755
	--- a/extra/resources/SysInfo
	+++ b/extra/resources/SysInfo.in
	@@ -1,386 +1,386 @@
	-#!/bin/bash
	+#!@BASH_PATH@
	#
	#
	# SysInfo OCF Resource Agent
	# It records (in the CIB) various attributes of a node
	#
	# Copyright (c) 2004 SUSE LINUX AG, Lars Marowsky-Br馥
	# All Rights Reserved.
	#
	# This program is free software; you can redistribute it and/or modify
	# it under the terms of version 2 of the GNU General Public License as
	# published by the Free Software Foundation.
	#
	# This program is distributed in the hope that it would be useful, but
	# WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
	#
	# Further, this software is distributed without any warranty that it is
	# free of the rightful claim of any third person regarding infringement
	# or the like. Any license provided herein, whether implied or
	# otherwise, applies only to this software file. Patent licenses, if
	# any, provided herein do not apply to combinations of this program with
	# other software, or any other product whatsoever.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write the Free Software Foundation,
	# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
	#
	#######################################################################
	# Initialization:

	: ${OCF_FUNCTIONS=${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs}
	. ${OCF_FUNCTIONS}
	: ${__OCF_ACTION=$1}

	#######################################################################

	meta_data() {
	cat <<END
	<?xml version="1.0"?>
	<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
	<resource-agent name="SysInfo" version="1.0">
	<version>1.0</version>

	<longdesc lang="en">
	This is a SysInfo Resource Agent.
	It records (in the CIB) various attributes of a node
	Sample Linux output:
	arch: i686
	os: Linux-2.4.26-gentoo-r14
	free_swap: 1999
	cpu_info: Intel(R) Celeron(R) CPU 2.40GHz
	cpu_speed: 4771.02
	cpu_cores: 1
	cpu_load: 0.00
	ram_total: 513
	ram_free: 117
	root_free: 2.4
	#health_disk: red

	Sample Darwin output:
	arch: i386
	os: Darwin-8.6.2
	cpu_info: Intel Core Duo
	cpu_speed: 2.16
	cpu_cores: 2
	cpu_load: 0.18
	ram_total: 2016
	ram_free: 787
	root_free: 13
	#health_disk: green

	Units:
	free_swap: MB
	ram_*: MB
	cpu_speed (Linux): bogomips
	cpu_speed (Darwin): GHz
	*_free: GB (or user-defined: disk_unit)

	</longdesc>
	<shortdesc lang="en">SysInfo resource agent</shortdesc>

	<parameters>

	<parameter name="pidfile" unique="1">
	<longdesc lang="en">PID file</longdesc>
	<shortdesc lang="en">PID file</shortdesc>
	<content type="string" default="$OCF_RESKEY_pidfile" />
	</parameter>

	<parameter name="delay" unique="0">
	<longdesc lang="en">Interval to allow values to stabilize</longdesc>
	<shortdesc lang="en">Dampening Delay</shortdesc>
	<content type="string" default="0s" />
	</parameter>

	<parameter name="disks" unique="0">
	<longdesc lang="en">
	Filesystems or Paths to be queried for free disk space as a SPACE
	separated list - e.g "/dev/sda1 /tmp".
	Results will be written to an attribute with leading slashes
	removed, and other slashes replaced with underscore, and the word
	'free' appended - e.g for /dev/sda1 it would be 'dev_sda1_free'.
	Note: The root filesystem '/' is always queried to an attribute
	named 'root_free'
	</longdesc>
	<shortdesc lang="en">List of Filesytems/Paths to query for free disk space</shortdesc>
	<content type="string" />
	</parameter>

	<parameter name="disk_unit" unique="0">
	<longdesc lang="en">
	Unit to report disk free space in.
	Can be one of: B, K, M, G, T, P (case-insensitive)
	</longdesc>
	<shortdesc lang="en">Unit to report disk free space in</shortdesc>
	<content type="string" default="G"/>
	</parameter>

	<parameter name="min_disk_free" unique="0">
	<longdesc lang="en">
	The amount of free space required in monitored disks. If any
	of the monitored disks has less than this amount of free space,
	, with the node attribute "#health_disk" changing to "red",
	all resources will move away from the node. Set the node-health-strategy
	property appropriately for this to take effect.
	If the unit is not specified, it defaults to disk_unit.
	</longdesc>
	<shortdesc lang="en">minimum disk free space required</shortdesc>
	<content type="string" default=""/>
	</parameter>


	</parameters>
	<actions>
	<action name="start" timeout="20s" />
	<action name="stop" timeout="20s" />
	<action name="monitor" timeout="20s" interval="60s"/>
	<action name="meta-data" timeout="5" />
	<action name="validate-all" timeout="30" />
	</actions>
	</resource-agent>
	END
	}

	#######################################################################

	UpdateStat() {
	name=$1; shift
	value="$*"
	printf "%s:\t%s\n" "$name" "$value"
	if [ "$__OCF_ACTION" = "start" ] ; then
	${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -S status -n $name -B "$value"
	else
	${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -S status -n $name -v "$value"
	fi
	}

	SysInfoStats() {

	UpdateStat arch "`uname -m`"
	UpdateStat os "`uname -s`-`uname -r`"

	case `uname -s` in
	"Darwin")
	mem=`top -l 1 \| grep Mem: \| awk '{print $10}'`
	mem_used=`top -l 1 \| grep Mem: \| awk '{print $8}'`
	mem=`SysInfo_mem_units $mem`
	mem_used=`SysInfo_mem_units $mem_used`
	mem_total=`expr $mem_used + $mem`
	cpu_type=`system_profiler SPHardwareDataType \| awk -F': ' '/^CPU Type/ {print $2; exit}'`
	cpu_speed=`system_profiler SPHardwareDataType \| awk -F': ' '/^CPU Speed/ {print $2; exit}'`
	cpu_cores=`system_profiler SPHardwareDataType \| awk -F': ' '/^Number Of/ {print $2; exit}'`
	cpu_load=`uptime \| awk '{ print $10 }'`
	;;
	"FreeBSD")
	cpu_type=`sysctl -in hw.model`
	cpu_speed=`sysctl -in dev.cpu.0.freq`
	cpu_cores=`sysctl -in hw.ncpu`
	cpu_load=`sysctl -in vm.loadavg \| awk '{ print $4 }'`

	free_pages=`sysctl -in vm.stats.vm.v_free_count`
	page_count=`sysctl -in vm.stats.vm.v_page_count`
	page_size=`sysctl -in vm.stats.vm.v_page_size`

	mem=`expr $free_pages \* $page_size / 1024 / 1024`M
	mem_total=`expr $page_count \* $page_size / 1024 / 1024`M
	;;
	"Linux")
	if [ -f /proc/cpuinfo ]; then
	cpu_type=`awk -F': ' '/model name/ {print $2; exit}' /proc/cpuinfo`
	cpu_speed=`awk -F': ' '/bogomips/ {print $2; exit}' /proc/cpuinfo`
	cpu_cores=`grep "^processor" /proc/cpuinfo \| wc -l`
	fi
	cpu_load=`uptime \| awk '{ print $10 }'`

	if [ -f /proc/meminfo ]; then
	# meminfo results are in kB
	mem=`grep "SwapFree" /proc/meminfo \| awk '{print $2"k"}'`
	if [ ! -z $mem ]; then
	UpdateStat free_swap `SysInfo_mem_units $mem`
	fi
	mem=`grep "Inactive" /proc/meminfo \| awk '{print $2"k"}'`
	mem_total=`grep "MemTotal" /proc/meminfo \| awk '{print $2"k"}'`
	else
	mem=`top -n 1 \| grep Mem: \| awk '{print $7}'`
	fi
	;;
	*)
	esac

	if [ x != x"$cpu_type" ]; then
	UpdateStat cpu_info "$cpu_type"
	fi

	if [ x != x"$cpu_speed" ]; then
	UpdateStat cpu_speed "$cpu_speed"
	fi

	if [ x != x"$cpu_cores" ]; then
	UpdateStat cpu_cores "$cpu_cores"
	fi

	if [ x != x"$cpu_load" ]; then
	UpdateStat cpu_load "$cpu_load"
	fi

	if [ ! -z "$mem" ]; then
	# Massage the memory values
	UpdateStat ram_total `SysInfo_mem_units $mem_total`
	UpdateStat ram_free `SysInfo_mem_units $mem`
	fi

	# Portability notes:
	# o tail: explicit "-n" not available in Solaris; instead simplify
	# 'tail -n <c>' to the equivalent 'tail -<c>'.
	for disk in "/" ${OCF_RESKEY_disks}; do
	unset disk_free disk_label
	disk_free=`df -h ${disk} \| tail -1 \| awk '{print $4}'`
	if [ x != x"$disk_free" ]; then
	disk_label=`echo $disk \| sed -e 's#^/$#root#;s#^/*##;s#/#_#g'`
	disk_free=`SysInfo_hdd_units $disk_free`
	UpdateStat ${disk_label}_free $disk_free
	if [ -n "$MIN_FREE" ]; then
	if [ $disk_free -le $MIN_FREE ]; then
	UpdateStat "#health_disk" "red"
	else
	UpdateStat "#health_disk" "green"
	fi
	fi
	fi
	done
	}

	SysInfo_megabytes() {
	# Size in megabytes
	echo $1 \| awk '{ n = $0;
	sub(/[0-9]+(.[0-9]+)?/, "");
	split(n, a, $0);
	n=a[1];
	if ($0 == "G" \|\| $0 == "") { n *= 1024 };
	if (/^kB?/) { n /= 1024 };
	printf "%d\n", n }' # Intentionally round to an integer
	}

	SysInfo_mem_units() {
	mem=$1

	if [ -z $1 ]; then
	return
	fi

	mem=$(SysInfo_megabytes "$1")
	# Round to the next multiple of 50
	r=$(($mem % 50))
	if [ $r != 0 ]; then
	mem=$(($mem + 50 - $r))
	fi

	echo $mem
	}

	SysInfo_hdd_units() {
	# Defauts to size in gigabytes

	case $OCF_RESKEY_disk_unit in
	[Pp]) echo $(($(SysInfo_megabytes "$1") / 1024 / 1024 / 1024));;
	[Tt]) echo $(($(SysInfo_megabytes "$1") / 1024 / 1024));;
	[Gg]) echo $(($(SysInfo_megabytes "$1") / 1024));;
	[Mm]) echo $(SysInfo_megabytes "$1");;
	[Kk]) echo $(($(SysInfo_megabytes "$1") * 1024));;
	[Bb]) echo $(($(SysInfo_megabytes "$1") * 1024 * 1024));;
	*)
	ocf_log err "Invalid value for disk_unit: $OCF_RESKEY_disk_unit"
	echo $(($(SysInfo_megabytes "$1") / 1024));;
	esac
	}

	SysInfo_usage() {
	cat <<END
	usage: $0 {start\|stop\|monitor\|validate-all\|meta-data}

	Expects to have a fully populated OCF RA-compliant environment set.
	END
	}

	SysInfo_start() {
	echo $OCF_RESKEY_clone > $OCF_RESKEY_pidfile
	SysInfoStats
	exit $OCF_SUCCESS
	}

	SysInfo_stop() {
	rm $OCF_RESKEY_pidfile
	exit $OCF_SUCCESS
	}

	SysInfo_monitor() {
	if [ -f $OCF_RESKEY_pidfile ]; then
	clone=`cat $OCF_RESKEY_pidfile`
	fi

	if [ x$clone = x ]; then
	rm $OCF_RESKEY_pidfile
	exit $OCF_NOT_RUNNING

	elif [ $clone = $OCF_RESKEY_clone ]; then
	SysInfoStats
	exit $OCF_SUCCESS

	elif [ x$OCF_RESKEY_CRM_meta_globally_unique = xtrue
	-o x$OCF_RESKEY_CRM_meta_globally_unique = xTrue
	-o x$OCF_RESKEY_CRM_meta_globally_unique = xyes
	-o x$OCF_RESKEY_CRM_meta_globally_unique = xYes
	]; then
	SysInfoStats
	exit $OCF_SUCCESS
	fi
	exit $OCF_NOT_RUNNING
	}

	SysInfo_validate() {
	return $OCF_SUCCESS
	}

	if [ $# -ne 1 ]; then
	SysInfo_usage
	exit $OCF_ERR_ARGS
	fi

	: ${OCF_RESKEY_pidfile:="${HA_VARRUN%%/}/SysInfo-${OCF_RESOURCE_INSTANCE}"}
	: ${OCF_RESKEY_disk_unit:="G"}
	: ${OCF_RESKEY_clone:="0"}
	if [ x != x${OCF_RESKEY_delay} ]; then
	OCF_RESKEY_delay="-d ${OCF_RESKEY_delay}"
	else
	OCF_RESKEY_delay="-d 0"
	fi
	MIN_FREE=""
	if [ -n "$OCF_RESKEY_min_disk_free" ]; then
	ocf_is_decimal "$OCF_RESKEY_min_disk_free" &&
	OCF_RESKEY_min_disk_free="$OCF_RESKEY_min_disk_free$OCF_RESKEY_disk_unit"
	MIN_FREE=`SysInfo_hdd_units $OCF_RESKEY_min_disk_free`
	fi

	case $__OCF_ACTION in
	meta-data) meta_data
	exit $OCF_SUCCESS
	;;
	start) SysInfo_start
	;;
	stop) SysInfo_stop
	;;
	monitor) SysInfo_monitor
	;;
	validate-all) SysInfo_validate
	;;
	usage\|help) SysInfo_usage
	exit $OCF_SUCCESS
	;;
	*) SysInfo_usage
	exit $OCF_ERR_UNIMPLEMENTED
	;;
	esac

	exit $?
	diff --git a/extra/resources/ifspeed b/extra/resources/ifspeed.in
	similarity index 99%
	rename from extra/resources/ifspeed
	rename to extra/resources/ifspeed.in
	index 12c6135e4d..d5eee6c5f8 100755
	--- a/extra/resources/ifspeed
	+++ b/extra/resources/ifspeed.in
	@@ -1,541 +1,541 @@
	-#!/bin/bash
	+#!@BASH_PATH@
	#
	# OCF resource agent which monitors state of network interface and records it
	# as a node attribute in the CIB based on the sum of speeds of its active (up,
	# link detected, not blocked) underlying interfaces.
	#
	# Copyright (c) 2011 Vladislav Bogdanov <bubble@hoster-ok.com>
	# Partially based on 'ping' RA by Andrew Beekhof
	#
	# Change on 2017 by Tomer Azran <tomerazran@gmail.com>:
	# Add "ip" parameter to detect network interface name by ip address:
	# http://lists.clusterlabs.org/pipermail/users/2017-August/006224.html
	#
	# OCF instance parameters:
	# OCF_RESKEY_name: name of attribute to set in CIB
	# OCF_RESKEY_ip ip address to check
	# OCF_RESKEY_iface: network interface to monitor
	# OCF_RESKEY_bridge_ports: if not null and OCF_RESKEY_iface is a bridge, list of
	# bridge ports to consider.
	# Default is all ports which have designated_bridge=root_id
	# OCF_RESKEY_weight_base: Relative weight of 1Gbps. This can be used to tune
	# value of resulting CIB attribute.
	#
	# Initialization:

	: ${OCF_FUNCTIONS=${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs}

	# If these aren't available, we can still show help,
	# which is all that is needed to build the man pages.
	[ -r "${OCF_FUNCTIONS}" ] && . "${OCF_FUNCTIONS}"
	[ -r "${OCF_FUNCTIONS_DIR}/findif.sh" ] && . "${OCF_FUNCTIONS_DIR}/findif.sh"
	: ${OCF_SUCCESS=0}

	: ${__OCF_ACTION=$1}

	FINDIF=findif

	# Defaults
	OCF_RESKEY_name_default="ifspeed"
	OCF_RESKEY_bridge_ports_default="detect"
	OCF_RESKEY_weight_base_default=1000
	OCF_RESKEY_dampen_default=5

	: ${OCF_RESKEY_name=${OCF_RESKEY_name_default}}
	: ${OCF_RESKEY_bridge_ports=${OCF_RESKEY_bridge_ports_default}}
	: ${OCF_RESKEY_weight_base=${OCF_RESKEY_weight_base_default}}
	: ${OCF_RESKEY_dampen=${OCF_RESKEY_dampen_default}}

	meta_data() {
	cat <<END
	<?xml version="1.0"?>
	<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
	<resource-agent name="ifspeed" version="1.0">
	<version>1.0</version>

	<longdesc lang="en">
	Every time the monitor action is run, this resource agent records (in the CIB)
	(relative) speed of network interface it monitors.

	This RA can monitor physical interfaces, bonds, bridges, vlans and (hopefully)
	any combination of them.

	Examples:
	*) Bridge on top of one 10Gbps interface (eth2) and 802.3ad bonding (bond0) built
	on two 1Gbps interfaces (eth0 and eth1).
	*) Active-backup bonding built on top of one physical interface and one vlan on
	another interface.

	For STP-enabled bridges this RA tries to some-how guess network topology and by
	default looks only on ports which are connected to upstream switch. This can be
	overridden by 'bridge_ports' parameter. Active interfaces in this case are those
	in "forwarding" state.

	For balancing bonds this RA summs speeds of underlying "up" slave interfaces
	(and applies coefficient 0.8 to result).

	For non-balancing bonds ('active-backup' and probably 'broadcast'), only the
	speed of the currently active slave is used.
	</longdesc>
	<shortdesc lang="en">Network interface speed monitor</shortdesc>

	<parameters>

	<parameter name="name" unique="1">
	<longdesc lang="en">
	The name of the attribute to set. This is the name to be used in the constraints.
	</longdesc>
	<shortdesc lang="en">Attribute name</shortdesc>
	<content type="string" default="${OCF_RESKEY_name_default}"/>
	</parameter>

	<parameter name="iface" unique="0" required="1">
	<longdesc lang="en">
	Network interface to monitor.
	</longdesc>
	<shortdesc lang="en">Network interface</shortdesc>
	<content type="string" default=""/>
	</parameter>

	<parameter name="ip" unique="0" required="0">
	<longdesc lang="en">
	Try to detect interface name by detecting the interface that holds the IP address.
	The IPv4 (dotted quad notation) or IPv6 address (colon hexadecimal notation)
	example IPv4 "192.168.1.1".
	example IPv6 "2001:db8:DC28:0:0:FC57:D4C8:1FFF".
	</longdesc>
	<shortdesc lang="en">IPv4 or IPv6 address</shortdesc>
	<content type="string" default="" />
	</parameter>

	<parameter name="bridge_ports" unique="0">
	<longdesc lang="en">
	If not null and OCF_RESKEY_iface is a bridge, list of bridge ports to consider.
	Default is all ports which have designated_bridge=root_id.
	</longdesc>
	<shortdesc lang="en">Bridge ports</shortdesc>
	<content type="string" default="${OCF_RESKEY_bridge_ports_default}"/>
	</parameter>

	<parameter name="weight_base" unique="0">
	<longdesc lang="en">
	Relative weight of 1Gbps in interface speed.
	Can be used to tune how big attribute value will be.
	</longdesc>
	<shortdesc lang="en">Weight of 1Gbps</shortdesc>
	<content type="integer" default="${OCF_RESKEY_weight_base_default}"/>
	</parameter>

	<parameter name="dampen" unique="0">
	<longdesc lang="en">
	The time to wait (dampening) for further changes to occur.
	</longdesc>
	<shortdesc lang="en">Dampening interval</shortdesc>
	<content type="integer" default="${OCF_RESKEY_dampen_default}"/>
	</parameter>

	<parameter name="debug" unique="0">
	<longdesc lang="en">
	Log what have been done more verbosely.
	</longdesc>
	<shortdesc lang="en">Verbose logging</shortdesc>
	<content type="string" default="false"/>
	</parameter>

	</parameters>

	<actions>
	<action name="start" timeout="30" />
	<action name="stop" timeout="30" />
	<action name="monitor" depth="0" timeout="30" interval="10"/>
	<action name="meta-data" timeout="5" />
	<action name="validate-all" timeout="30" />
	</actions>
	</resource-agent>
	END
	}

	usage() {
	cat <<END
	usage: $0 {start\|stop\|monitor\|validate-all\|meta-data}

	Expects to have a fully populated OCF RA-compliant environment set.
	END
	}

	start() {
	monitor
	if [ $? -eq $OCF_SUCCESS ]; then
	return $OCF_SUCCESS
	fi
	ha_pseudo_resource ${ha_pseudo_resource_name} start
	update
	return $?
	}

	stop() {
	ha_pseudo_resource ${ha_pseudo_resource_name} stop
	attrd_updater -D -n ${OCF_RESKEY_name} -d ${OCF_RESKEY_dampen} ${attrd_options}
	return $OCF_SUCCESS
	}

	monitor() {
	local ret
	ha_pseudo_resource ${ha_pseudo_resource_name} monitor
	ret=$?
	if [ ${ret} -eq $OCF_SUCCESS ] ; then
	update
	fi
	return ${ret}
	}

	# This function tries to guess nic interface by IP
	get_nic_name_by_ip(){
	# $FINDIF takes its parameters from the environment.
	# Its output is as follows:
	# [NIC_NAME] netmask [NETMASK] broadcast [BROADCAST}
	NICINFO=$( ${FINDIF} )
	rc=$?
	if [ $rc -eq 0 ];then
	# Get NIC_NAME part of findif function output.
	echo "${NICINFO%% *}"
	else
	echo ""
	fi
	}

	validate() {
	# Check the interface parameter
	if [ -z "${OCF_RESKEY_iface}" ]; then
	if [ -z "${OCF_RESKEY_ip}" ]; then
	ocf_log err "Empty iface and ip parameters. Please specify either an interface name or valid ip address."
	exit $OCF_ERR_CONFIGURED
	else
	ipcheck_ipv4 "${OCF_RESKEY_ip}"
	if [ $? -eq 1 ] ; then
	ipcheck_ipv6 "${OCF_RESKEY_ip}"
	if [ $? -eq 1 ] ; then
	ocf_exit_reason "ip parameter [${OCF_RESKEY_ip}] is not a valid ip address."
	exit $OCF_ERR_CONFIGURED
	fi
	fi
	fi
	fi
	return $OCF_SUCCESS
	}

	iface_get_speed() {
	local iface=$1
	local operstate
	local carrier
	local bridge_iface_speed
	local bond_iface_speed
	local vlan_iface_speed
	local speed

	if [ ! -e "/sys/class/net/${iface}" ] ; then
	echo "0"
	elif iface_is_bridge ${iface} ; then # bridges do not have operstate
	read carrier < "/sys/class/net/${iface}/carrier"

	if [ "${carrier}" != "1" ] ; then
	echo "0"
	else
	bridge_get_speed ${iface}
	fi
	else
	read operstate < "/sys/class/net/${iface}/operstate"
	read carrier < "/sys/class/net/${iface}/carrier"

	if [ "${operstate}" != "up" ] \|\| [ "${carrier}" != "1" ] ; then
	echo "0"
	elif iface_is_bond ${iface} ; then
	bond_get_speed ${iface}
	elif iface_is_vlan ${iface} ; then
	iface_get_speed $( vlan_get_phy ${iface} )
	elif iface_is_hfi1 "${iface}" ; then
	hfi1_get_speed "${iface}"
	else
	read speed < "/sys/class/net/${iface}/speed"
	echo ${speed}
	fi
	fi
	}

	iface_is_vlan() {
	local iface=$1
	[ -e "/proc/net/vlan/${iface}" ] && return 0 \|\| return 1
	}

	iface_is_bridge() {
	local iface=$1
	[ -e "/sys/class/net/${iface}/bridge" ] && return 0 \|\| return 1
	}

	iface_is_bond() {
	local iface=$1
	[ -e "/sys/class/net/${iface}/bonding" ] && return 0 \|\| return 1
	}

	iface_is_hfi1() {
	local iface=$1
	driver=$(readlink /sys/class/net/${iface}/device/driver)
	[[ $(basename ${driver}) =~ "hfi1" ]] && return 0 \|\| return 1
	}

	vlan_get_phy() {
	local iface=$1
	sed -ne "s/^${iface} .\| //p" < /proc/net/vlan/config
	}

	bridge_is_stp_enabled() {
	local iface=$1
	local stp
	read stp < "/sys/class/net/${iface}/bridge/stp_state"
	[ "${stp}" = "1" ] && return 0 \|\| return 1
	}

	bridge_get_root_ports() {
	local bridge=$1
	local root_id
	local root_ports=""
	local bridge_id

	read root_id < "/sys/class/net/${bridge}/bridge/root_id"

	for port in /sys/class/net/${bridge}/brif/* ; do
	read bridge_id < "${port}/designated_bridge"
	if [ "${bridge_id}" = "${root_id}" ] ; then
	root_ports="${root_ports} ${port##*/}"
	fi
	done

	root_ports=${root_ports# }

	if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
	eval $2=\${root_ports}
	else # Expect sub-shell
	echo ${root_ports}
	fi
	}

	# From /inlude/linux/if_bridge.h:
	#define BR_STATE_DISABLED 0
	#define BR_STATE_LISTENING 1
	#define BR_STATE_LEARNING 2
	#define BR_STATE_FORWARDING 3
	#define BR_STATE_BLOCKING 4

	bridge_get_active_ports() {
	local bridge=$1
	shift 1
	local ports="$*"
	local active_ports=""
	local port_state
	local stp_state
	local warn=0

	bridge_is_stp_enabled ${bridge}
	stp_state=$?

	if [ -z "${ports}" ] \|\| [ "${ports}" = "detect" ] ; then
	bridge_get_root_ports ${bridge} ports
	fi

	for port in $ports ; do
	if [ ! -e "/sys/class/net/${bridge}/brif/${port}" ] ; then
	ocf_log warning "Port ${port} doesn't belong to bridge ${bridge}"
	continue
	fi
	read port_state < "/sys/class/net/${bridge}/brif/${port}/state"
	if [ "${port_state}" = "3" ] ; then
	if [ -n "${active_ports}" ] && ${stp_state} ; then
	warn=1
	fi
	active_ports="${active_ports} ${port}"
	fi
	done
	if [ ${warn} -eq 1 ] ; then
	ocf_log warning "More then one upstream port in bridge '${bridge}' is in forwarding state while STP is enabled: ${active_ports}"
	fi
	echo "${active_ports# }"
	}

	bridge_get_speed() {
	local iface=$1
	local bridge_port_speed
	local aggregate_speed=0

	if ! iface_is_bridge ${iface} ; then
	echo 0
	return
	fi

	local ports=$( bridge_get_active_ports ${iface} ${OCF_RESKEY_bridge_ports} )
	for port in ${ports} ; do
	: $(( aggregate_speed += $( iface_get_speed ${port} ) ))
	done
	if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
	eval $2=\${aggregate_speed}
	else # Expect sub-shell
	echo ${aggregate_speed}
	fi
	}

	hfi1_get_speed() {
	local iface=$1
	local hfi1_speed
	local hfi1_value
	local hfi1_desc

	# Currently (9/14/2017 Intel Omni Path v10.5.0.0.155) Intel doesn't have Dual/Multiple ports Host Channel Adapters
	# and it's save to use such method to get a speed.
	# Example of output:
	# [root@es-host0 ~]# cat /sys/class/net/ib0/device/infiniband//ports//rate
	# 100 Gb/sec (4X EDR)
	read hfi1_speed hfi1_value hfi1_desc < /sys/class/net/${iface}/device/infiniband//ports//rate

	# hfi1_value always in Gb/sec, so we need to convert hfi1_speed in Mb/sec
	echo $(( hfi1_speed * 1000 ))
	}

	bond_get_slaves() {
	local iface=$1
	local slaves
	read slaves < "/sys/class/net/${iface}/bonding/slaves"
	if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
	eval $2=\${slaves}
	else # Expect sub-shell
	echo ${slaves}
	fi
	}

	bond_get_active_iface() {
	local iface=$1
	local active
	read active < "/sys/class/net/${iface}/bonding/active_slave"
	if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
	eval $2=\${active}
	else # Expect sub-shell
	echo ${active}
	fi
	}

	bond_is_balancing() {
	local iface=$1
	read mode mode_index < "/sys/class/net/${iface}/bonding/mode"
	case ${mode} in
	"balance-rr"\|"balance-xor"\|"802.3ad"\|"balance-tlb"\|"balance-alb")
	return 0
	;;
	*)
	return 1
	;;
	esac
	}

	bond_get_speed() {
	local iface=$1
	local aggregate_speed=0
	local bond_slave_speed
	local active_iface
	local bond_slaves

	if ! iface_is_bond ${iface} ; then
	echo 0
	return
	fi

	bond_get_slaves ${iface} bond_slaves

	if bond_is_balancing ${iface} ; then
	for slave in ${bond_slaves} ; do
	: $(( aggregate_speed += $( iface_get_speed ${slave} ) ))
	done
	# Bonding is unable to get speed*n
	: $(( aggregate_speed = aggregate_speed * 8 / 10 ))
	else
	bond_get_active_iface ${iface} active_iface
	aggregate_speed=$( iface_get_speed $active_iface )
	fi
	if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
	eval $2=\${aggregate_speed}
	else # Expect sub-shell
	echo ${aggregate_speed}
	fi
	}

	update() {
	local speed;
	local nic=${OCF_RESKEY_iface};
	if [ -z "${OCF_RESKEY_iface}" ]; then
	nic=$( get_nic_name_by_ip )
	if [ -z "${nic}" ];then
	ocf_log err "Could not retrieve network interface name from ip address (${OCF_RESKEY_ip})"
	exit $OCF_ERR_GENERIC
	fi
	fi
	speed=$( iface_get_speed ${nic} )

	: $(( score = speed * ${OCF_RESKEY_weight_base} / 1000 ))
	if [ "$__OCF_ACTION" = "start" ] ; then
	attrd_updater -n ${OCF_RESKEY_name} -B ${score} -d ${OCF_RESKEY_dampen} ${attrd_options}
	else
	attrd_updater -n ${OCF_RESKEY_name} -v ${score} -d ${OCF_RESKEY_dampen} ${attrd_options}
	fi
	rc=$?
	case ${rc} in
	0)
	ocf_is_true ${OCF_RESKEY_debug} && ocf_log debug "Updated ${OCF_RESKEY_name} = ${score}"
	;;
	*)
	ocf_log warn "Could not update ${OCF_RESKEY_name} = ${score}: rc=${rc}"
	;;
	esac
	return ${rc}
	}

	case $__OCF_ACTION in
	meta-data)
	meta_data
	exit $OCF_SUCCESS
	;;
	usage\|help)
	usage
	exit $OCF_SUCCESS
	;;
	esac

	if [ `uname` != "Linux" ] ; then
	ocf_log err "This RA works only on linux."
	exit $OCF_ERR_INSTALLED
	fi

	: ${ha_pseudo_resource_name:="ifspeed-${OCF_RESOURCE_INSTANCE}"}

	attrd_options='-q'
	if ocf_is_true ${OCF_RESKEY_debug} ; then
	attrd_options=''
	fi

	validate \|\| exit $?

	case $__OCF_ACTION in
	start)
	start
	;;
	stop)
	stop
	;;
	monitor)
	monitor
	;;
	validate-all)
	;;
	*)
	usage
	exit $OCF_ERR_UNIMPLEMENTED
	;;
	esac

	exit $?
	diff --git a/extra/resources/o2cb b/extra/resources/o2cb.in
	similarity index 99%
	rename from extra/resources/o2cb
	rename to extra/resources/o2cb.in
	index d5aba16d73..4cf4b8ffb4 100755
	--- a/extra/resources/o2cb
	+++ b/extra/resources/o2cb.in
	@@ -1,452 +1,452 @@
	-#!/bin/bash
	+#!@BASH_PATH@
	# Copyright (c) 2005,2008 Oracle
	# Copyright (c) 2008 Andrew Beekhof
	# All Rights Reserved.
	#
	# This program is free software; you can redistribute it and/or modify
	# it under the terms of version 2 of the GNU General Public License as
	# published by the Free Software Foundation.
	#
	# This program is distributed in the hope that it would be useful, but
	# WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
	#
	# Further, this software is distributed without any warranty that it is
	# free of the rightful claim of any third person regarding infringement
	# or the like. Any license provided herein, whether implied or
	# otherwise, applies only to this software file. Patent licenses, if
	# any, provided herein do not apply to combinations of this program with
	# other software, or any other product whatsoever.
	#
	#######################################################################

	: ${OCF_FUNCTIONS=${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs}
	. ${OCF_FUNCTIONS}
	: ${__OCF_ACTION=$1}

	: ${OCF_RESKEY_stack:="pcmk"}
	: ${OCF_RESKEY_sysfs:="/sys/fs"}
	: ${OCF_RESKEY_configfs:="/sys/kernel/config"}
	: ${OCF_RESKEY_daemon_timeout:="10"} # How long to wait for things to start
	: ${OCF_RESKEY_CRM_meta_globally_unique:="false"}

	DAEMON="/usr/sbin/ocfs2_controld.${OCF_RESKEY_stack}"
	CLUSTER_STACK_FILE="${OCF_RESKEY_sysfs}/ocfs2/cluster_stack"
	LOADED_PLUGINS_FILE="${OCF_RESKEY_sysfs}/ocfs2/loaded_cluster_plugins"

	#
	# Check to see if a filesystem driver is loaded.
	# 0 is loaded, 1 is not.
	#
	driver_filesystem() {
	if [ -z "$1" ]
	then
	ocf_log err "driver_filesystem(): Missing an argument"
	exit 1
	fi
	FSNAME="$1"

	FSOUT="$(awk '(NF == 1 && $1 ~ /^'$FSNAME'$/) \|\| $2 ~ /^'$FSNAME'$/{
	print $1;exit
	}' /proc/filesystems 2>/dev/null)"

	test -n "$FSOUT"
	return $?
	}

	#
	# Check to see if a filesystem of type $1 is mounted at $2.
	#
	# 0 is mounted, 1 is not.
	#
	check_filesystem()
	{
	if [ "$#" != "2" -o -z "$1" -o -z "$2" ]
	then
	ocf_log err "check_filesystem(): Missing arguments"
	exit 4
	fi
	FSNAME="$1"
	MOUNTPOINT="$2"

	FULL_MOUNTSEARCH="`echo "$MOUNTPOINT" \| sed -e 's/\//\\\\\//g'`"
	MOUNTOUT="`awk '$2 ~ /^'$FULL_MOUNTSEARCH'$/ && $3 ~ /^'$FSNAME'$/{print $2; exit}' < /proc/mounts 2>/dev/null`"
	test -n "$MOUNTOUT"
	return $?
	}

	#
	# Unload a filesystem driver.
	# Be careful to notice if the driver is built-in and do nothing.
	#
	# 0 is success, 1 is error, 2 is already unloaded.
	#
	unload_filesystem()
	{
	if [ "$#" != "1" -o -z "$1" ]
	then
	ocf_log err "unload_filesystem(): Missing an argument"
	return 1
	fi
	FSNAME="$1"

	driver_filesystem "$FSNAME" \|\| return 2

	MODOUT="`awk '$1 ~ /^'$FSNAME'$/{print $1,$3;exit}' < /proc/modules 2>/dev/null`"
	if [ -z "$MODOUT" ]; then
	# The driver is built in, we can't unload it.
	return 0
	fi

	case "$MODOUT" in
	$FSNAME\ 0)
	;;
	$FSNAME\ *)
	# The driver is busy, leave it alone
	ocf_log err "Module $FSNAME is still in use"
	return 1
	;;
	*)
	ocf_log err "Invalid module parsing! "
	return 1
	;;
	esac

	modprobe -rs "$FSNAME"
	if [ "$?" != 0 ]; then
	ocf_log err "Unable to unload module: $FSNAME"
	return 1
	fi

	return 0
	}

	status_daemon()
	{
	PID=`pidof "$DAEMON"`
	if [ -n "$PID" ]; then
	return $OCF_SUCCESS
	fi
	return $OCF_NOT_RUNNING
	}

	bringup_daemon()
	{
	if [ ! -e "$DAEMON" ]; then
	ocf_log err "Required binary not found: $DAEMON"
	return $OCF_ERR_INSTALLED
	fi

	"$DAEMON"; rc=$?
	if [ $rc != 0 ]; then
	ocf_log err "Could not start $DAEMON"
	return $OCF_ERR_GENERIC
	fi

	sleep 1
	COUNT=0
	rc=$OCF_NOT_RUNNING

	while [ $rc = $OCF_NOT_RUNNING ]; do
	COUNT=`expr $COUNT + 1`
	if [ $COUNT -gt $OCF_RESKEY_daemon_timeout ]; then
	ocf_log err "`basename $DAEMON` did not come up"
	return $OCF_ERR_GENERIC
	fi
	status_daemon; rc=$?
	sleep 1
	done

	return $rc
	}

	kill_daemon()
	{
	status_daemon; rc=$?
	if [ $rc != $OCF_SUCCESS ]; then
	return $rc
	fi

	ocf_log info "Stopping `basename "$DAEMON"`"
	killproc "$DAEMON"

	while [ $rc = $OCF_NOT_RUNNING ]; do
	sleep 1
	status_daemon; rc=$?
	done

	return $OCF_SUCCESS
	}

	#
	# Unload a module
	# 0 is success, 1 is error, 2 is not loaded
	#
	unload_module()
	{
	if [ "$#" -lt "1" -o -z "$1" ]
	then
	ocf_log err "unload_module(): Requires an argument"
	return 1
	fi
	MODNAME="$1"

	MODOUT="`awk '$1 ~ /^'$MODNAME'$/{print $1,$3;exit}' < /proc/modules 2>/dev/null`"
	if [ -z "$MODOUT" ]
	then
	return 2
	fi

	case "$MODOUT" in
	$MODNAME\ 0)
	;;
	$MODNAME\ *)
	return 2
	;;
	*)
	ocf_log err "Invalid module parsing!"
	return 1
	;;
	esac

	modprobe -rs "$MODNAME"
	if [ "$?" != 0 ]; then
	ocf_log err "Unable to unload module \"$MODNAME\""
	return 1
	fi

	return 0
	}

	o2cb_start() {

	o2cb_monitor; rc=$?
	if [ $rc != $OCF_NOT_RUNNING ]; then
	return $rc
	fi

	ocf_log info "Starting $OCF_RESOURCE_INSTANCE"

	if [ ! -e "$CLUSTER_STACK_FILE" ]; then
	modprobe -s ocfs2_stackglue
	if [ $? != 0 ]; then
	ocf_log err "Could not load ocfs2_stackglue"
	return $OCF_ERR_INSTALLED
	fi
	fi

	SP_OUT="$(awk '/^'user'$/{print; exit}' "$LOADED_PLUGINS_FILE" 2>/dev/null)"
	if [ -z "$SP_OUT" ]
	then
	modprobe -s ocfs2_stack_user
	if [ $? != 0 ]; then
	ocf_log err "Could not load ocfs2_stack_user"
	return $OCF_ERR_INSTALLED
	fi
	fi

	SP_OUT="$(awk '/^'user'$/{print; exit}' "$LOADED_PLUGINS_FILE" 2>/dev/null)"
	if [ -z "$SP_OUT" ]; then
	ocf_log err "Switch to userspace stack unsuccessful"
	return $OCF_ERR_INSTALLED
	fi

	if [ -f "$CLUSTER_STACK_FILE" ]; then
	echo "$OCF_RESKEY_stack" >"$CLUSTER_STACK_FILE"
	if [ $? != 0 ]; then
	ocf_log err "Userspace stack '$OCF_RESKEY_stack' not supported"
	return $OCF_ERR_INSTALLED
	fi
	else
	ocf_log err "Switch to userspace stack not supported"
	return $OCF_ERR_INSTALLED
	fi

	driver_filesystem ocfs2; rc=$?
	if [ $rc != 0 ]; then
	modprobe -s ocfs2
	if [ "$?" != 0 ]; then
	ocf_log err "Unable to load ocfs2 module"
	return $OCF_ERR_INSTALLED
	fi
	fi

	bringup_daemon
	return $?
	}

	o2cb_stop() {
	o2cb_monitor; rc=$?
	case $rc in
	$OCF_NOT_RUNNING) return $OCF_SUCCESS;;
	esac

	ocf_log info "Stopping $OCF_RESOURCE_INSTANCE"

	kill_daemon
	if [ $? != 0 ]; then
	ocf_log err "Unable to unload modules: the cluster is still online"
	return $OCF_ERR_GENERIC
	fi

	unload_filesystem ocfs2
	if [ $? = 1 ]; then
	ocf_log err "Unable to unload ocfs2 module"
	return $OCF_ERR_GENERIC
	fi

	# If we can't find the stack glue, we have nothing to do.
	[ ! -e "$LOADED_PLUGINS_FILE" ] && return $OCF_SUCCESS

	while read plugin
	do
	unload_module "ocfs2_stack_${plugin}"
	if [ $? = 1 ]; then
	ocf_log err "Unable to unload ocfs2_stack_${plugin}"
	return $OCF_ERR_GENERIC
	fi
	done <"$LOADED_PLUGINS_FILE"

	unload_module "ocfs2_stackglue"
	if [ $? = 1 ]; then
	ocf_log err "Unable to unload ocfs2_stackglue"
	return $OCF_ERR_GENERIC
	fi

	# Don't unmount configfs - it's always in use by libdlm
	}

	o2cb_monitor() {
	o2cb_validate

	# Assume that ocfs2_controld will terminate if any of the conditions below are met

	driver_filesystem configfs; rc=$?
	if [ $rc != 0 ]; then
	ocf_log info "configfs not loaded"
	return $OCF_NOT_RUNNING
	fi

	check_filesystem configfs "${OCF_RESKEY_configfs}"; rc=$?
	if [ $rc != 0 ]; then
	ocf_log info "configfs not mounted"
	return $OCF_NOT_RUNNING
	fi

	if [ ! -e "$LOADED_PLUGINS_FILE" ]; then
	ocf_log info "Stack glue driver not loaded"
	return $OCF_NOT_RUNNING
	fi

	grep user "$LOADED_PLUGINS_FILE" >/dev/null 2>&1; rc=$?
	if [ $rc != 0 ]; then
	ocf_log err "Wrong stack `cat $LOADED_PLUGINS_FILE`"
	return $OCF_ERR_INSTALLED
	fi

	driver_filesystem ocfs2; rc=$?
	if [ $rc != 0 ]; then
	ocf_log info "ocfs2 not loaded"
	return $OCF_NOT_RUNNING
	fi

	status_daemon
	return $?
	}

	o2cb_usage() {
	echo "usage: $0 {start\|stop\|monitor\|validate-all\|meta-data}"
	echo " Expects to have a fully populated OCF RA-compliant environment set."
	echo " In particualr, a value for OCF_ROOT"
	}

	o2cb_validate() {
	check_binary ${DAEMON}

	case ${OCF_RESKEY_CRM_meta_globally_unique} in
	yes\|Yes\|true\|True\|1)
	ocf_log err "$OCF_RESOURCE_INSTANCE must be configured with the globally_unique=false meta attribute"
	exit $OCF_ERR_CONFIGURED
	;;
	esac

	return $OCF_SUCCESS
	}

	meta_data() {
	cat <<END
	<?xml version="1.0"?>
	<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
	<resource-agent name="o2cb" version="1.0">
	<version>1.0</version>
	<shortdesc lang="en">OCFS2 daemon resource agent</shortdesc>
	<longdesc lang="en">
	This Resource Agent controls the userspace daemon needed by OCFS2.
	</longdesc>
	<parameters>

	<parameter name="sysfs" unique="0">
	<longdesc lang="en">
	Location where sysfs is mounted
	</longdesc>
	<shortdesc lang="en">Sysfs location</shortdesc>
	<content type="string" default="/sys/fs"/>
	</parameter>

	<parameter name="configfs" unique="0">
	<longdesc lang="en">
	Location where configfs is mounted
	</longdesc>
	<shortdesc lang="en">Configfs location</shortdesc>
	<content type="string" default="/sys/kernel/config"/>
	</parameter>

	<parameter name="stack" unique="0">
	<longdesc lang="en">
	Which userspace stack to use. Known values: pcmk
	</longdesc>
	<shortdesc lang="en">Userspace stack</shortdesc>
	<content type="string" default="pcmk"/>
	</parameter>

	<parameter name="daemon_timeout" unique="0">
	<longdesc lang="en">
	Number of seconds to allow the control daemon to come up
	</longdesc>
	<shortdesc lang="en">Daemon Timeout</shortdesc>
	<content type="string" default="10"/>
	</parameter>

	</parameters>
	<actions>
	<action name="start" timeout="90" />
	<action name="stop" timeout="100" />
	<action name="monitor" timeout="20" depth="0"/>
	<action name="meta-data" timeout="5" />
	<action name="validate-all" timeout="30" />
	</actions>
	</resource-agent>
	END
	}

	case $__OCF_ACTION in
	meta-data) meta_data
	exit $OCF_SUCCESS
	;;
	start) o2cb_start
	;;
	stop) o2cb_stop
	;;
	monitor) o2cb_monitor
	;;
	validate-all) o2cb_validate
	;;
	usage\|help) o2cb_usage
	exit $OCF_SUCCESS
	;;
	*) o2cb_usage
	exit $OCF_ERR_UNIMPLEMENTED
	;;
	esac

	exit $?
	diff --git a/include/crm/common/results.h b/include/crm/common/results.h
	index 5569d348a0..b6a586beea 100644
	--- a/include/crm/common/results.h
	+++ b/include/crm/common/results.h
	@@ -1,135 +1,138 @@
	/*
	- * Copyright (C) 2012-2018 Andrew Beekhof <andrew@beekhof.net>
	+ * Copyright 2012-2018 Andrew Beekhof <andrew@beekhof.net>
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/
	#ifndef CRM_RESULTS__H
	# define CRM_RESULTS__H

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*!
	* \file
	* \brief Function and executable result codes
	* \ingroup core
	*/

	# define CRM_ASSERT(expr) do { \
	if(__unlikely((expr) == FALSE)) { \
	crm_abort(__FILE__, __FUNCTION__, __LINE__, #expr, TRUE, FALSE); \
	abort(); /* Redundant but it makes static analyzers happy */ \
	} \
	} while(0)

	/*
	* Function return codes
	*
	* For system error codes, see:
	* - /usr/include/asm-generic/errno.h
	* - /usr/include/asm-generic/errno-base.h
	*/

	# define pcmk_ok 0
	# define PCMK_ERROR_OFFSET 190 /* Replacements on non-linux systems, see include/portability.h */
	# define PCMK_CUSTOM_OFFSET 200 /* Purely custom codes */
	# define pcmk_err_generic 201
	# define pcmk_err_no_quorum 202
	# define pcmk_err_schema_validation 203
	# define pcmk_err_transform_failed 204
	# define pcmk_err_old_data 205
	# define pcmk_err_diff_failed 206
	# define pcmk_err_diff_resync 207
	# define pcmk_err_cib_modified 208
	# define pcmk_err_cib_backup 209
	# define pcmk_err_cib_save 210
	# define pcmk_err_schema_unchanged 211
	# define pcmk_err_cib_corrupt 212
	+# define pcmk_err_multiple 213
	+# define pcmk_err_node_unknown 214
	+# define pcmk_err_already 215

	/*
	* Exit status codes
	*
	* We want well-specified (i.e. OS-invariant) exit status codes for our daemons
	* and applications so they can be relied on by callers. (Function return codes
	* and errno's do not make good exit statuses.)
	*
	* The only hard rule is that exit statuses must be between 0 and 255; all else
	* is convention. Universally, 0 is success, and 1 is generic error (excluding
	* OSes we don't support -- for example, OpenVMS considers 1 success!).
	*
	* For init scripts, the LSB gives meaning to 0-7, and sets aside 150-199 for
	* application use. OCF adds 8-9 and 189-199.
	*
	* sysexits.h was an attempt to give additional meanings, but never really
	* caught on. It uses 0 and 64-78.
	*
	* Bash reserves 2 ("incorrect builtin usage") and 126-255 (126 is "command
	* found but not executable", 127 is "command not found", 128 + n is
	* "interrupted by signal n").
	*
	* tldp.org recommends 64-113 for application use.
	*
	* We try to overlap with the above conventions when practical.
	*/
	typedef enum crm_exit_e {
	// Common convention
	CRM_EX_OK = 0,
	CRM_EX_ERROR = 1,

	// LSB + OCF
	CRM_EX_INVALID_PARAM = 2,
	CRM_EX_UNIMPLEMENT_FEATURE = 3,
	CRM_EX_INSUFFICIENT_PRIV = 4,
	CRM_EX_NOT_INSTALLED = 5,
	CRM_EX_NOT_CONFIGURED = 6,
	CRM_EX_NOT_RUNNING = 7,

	// sysexits.h
	CRM_EX_USAGE = 64, // command line usage error
	CRM_EX_DATAERR = 65, // user-supplied data incorrect
	CRM_EX_NOINPUT = 66, // input file not available
	CRM_EX_NOUSER = 67, // user does not exist
	CRM_EX_NOHOST = 68, // host unknown
	CRM_EX_UNAVAILABLE = 69, // needed service unavailable
	CRM_EX_SOFTWARE = 70, // internal software bug
	CRM_EX_OSERR = 71, // external (OS/environmental) problem
	CRM_EX_OSFILE = 72, // system file not usable
	CRM_EX_CANTCREAT = 73, // file couldn't be created
	CRM_EX_IOERR = 74, // file I/O error
	CRM_EX_TEMPFAIL = 75, // try again
	CRM_EX_PROTOCOL = 76, // protocol violated
	CRM_EX_NOPERM = 77, // non-file permission issue
	CRM_EX_CONFIG = 78, // misconfiguration

	// Custom
	CRM_EX_FATAL = 100, // do not respawn
	CRM_EX_PANIC = 101, // panic the local host
	CRM_EX_DISCONNECT = 102, // lost connection to something
	CRM_EX_OLD = 103, // update older than existing config
	CRM_EX_DIGEST = 104, // digest comparison failed
	CRM_EX_NOSUCH = 105, // requested item does not exist
	CRM_EX_QUORUM = 106, // local partition does not have quorum
	CRM_EX_UNSAFE = 107, // requires --force or new conditions
	CRM_EX_EXISTS = 108, // requested item already exists
	CRM_EX_MULTIPLE = 109, // requested item has multiple matches

	// Other
	CRM_EX_TIMEOUT = 124, // convention from timeout(1)
	CRM_EX_MAX = 255, // ensure crm_exit_t can hold this
	} crm_exit_t;

	const char *pcmk_strerror(int rc);
	const char *pcmk_errorname(int rc);
	const char *bz2_strerror(int rc);
	crm_exit_t crm_errno2exit(int rc);
	const char *crm_exit_name(crm_exit_t exit_code);
	const char *crm_exit_str(crm_exit_t exit_code);
	crm_exit_t crm_exit(crm_exit_t rc);

	#ifdef __cplusplus
	}
	#endif

	#endif
	diff --git a/lib/common/results.c b/lib/common/results.c
	index 75f7c428bd..1a77d15e90 100644
	--- a/lib/common/results.c
	+++ b/lib/common/results.c
	@@ -1,474 +1,485 @@
	/*
	* Copyright 2004-2018 Andrew Beekhof <andrew@beekhof.net>
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <bzlib.h>
	#include <errno.h>
	#include <stdlib.h>
	#include <string.h>
	#include <qb/qbdefs.h>

	#include <crm/common/mainloop.h>
	#include <crm/common/xml.h>

	const char *
	pcmk_errorname(int rc)
	{
	int error = abs(rc);

	switch (error) {
	case E2BIG: return "E2BIG";
	case EACCES: return "EACCES";
	case EADDRINUSE: return "EADDRINUSE";
	case EADDRNOTAVAIL: return "EADDRNOTAVAIL";
	case EAFNOSUPPORT: return "EAFNOSUPPORT";
	case EAGAIN: return "EAGAIN";
	case EALREADY: return "EALREADY";
	case EBADF: return "EBADF";
	case EBADMSG: return "EBADMSG";
	case EBUSY: return "EBUSY";
	case ECANCELED: return "ECANCELED";
	case ECHILD: return "ECHILD";
	case ECOMM: return "ECOMM";
	case ECONNABORTED: return "ECONNABORTED";
	case ECONNREFUSED: return "ECONNREFUSED";
	case ECONNRESET: return "ECONNRESET";
	/* case EDEADLK: return "EDEADLK"; */
	case EDESTADDRREQ: return "EDESTADDRREQ";
	case EDOM: return "EDOM";
	case EDQUOT: return "EDQUOT";
	case EEXIST: return "EEXIST";
	case EFAULT: return "EFAULT";
	case EFBIG: return "EFBIG";
	case EHOSTDOWN: return "EHOSTDOWN";
	case EHOSTUNREACH: return "EHOSTUNREACH";
	case EIDRM: return "EIDRM";
	case EILSEQ: return "EILSEQ";
	case EINPROGRESS: return "EINPROGRESS";
	case EINTR: return "EINTR";
	case EINVAL: return "EINVAL";
	case EIO: return "EIO";
	case EISCONN: return "EISCONN";
	case EISDIR: return "EISDIR";
	case ELIBACC: return "ELIBACC";
	case ELOOP: return "ELOOP";
	case EMFILE: return "EMFILE";
	case EMLINK: return "EMLINK";
	case EMSGSIZE: return "EMSGSIZE";
	#ifdef EMULTIHOP // Not available on OpenBSD
	case EMULTIHOP: return "EMULTIHOP";
	#endif
	case ENAMETOOLONG: return "ENAMETOOLONG";
	case ENETDOWN: return "ENETDOWN";
	case ENETRESET: return "ENETRESET";
	case ENETUNREACH: return "ENETUNREACH";
	case ENFILE: return "ENFILE";
	case ENOBUFS: return "ENOBUFS";
	case ENODATA: return "ENODATA";
	case ENODEV: return "ENODEV";
	case ENOENT: return "ENOENT";
	case ENOEXEC: return "ENOEXEC";
	case ENOKEY: return "ENOKEY";
	case ENOLCK: return "ENOLCK";
	#ifdef ENOLINK // Not available on OpenBSD
	case ENOLINK: return "ENOLINK";
	#endif
	case ENOMEM: return "ENOMEM";
	case ENOMSG: return "ENOMSG";
	case ENOPROTOOPT: return "ENOPROTOOPT";
	case ENOSPC: return "ENOSPC";
	case ENOSR: return "ENOSR";
	case ENOSTR: return "ENOSTR";
	case ENOSYS: return "ENOSYS";
	case ENOTBLK: return "ENOTBLK";
	case ENOTCONN: return "ENOTCONN";
	case ENOTDIR: return "ENOTDIR";
	case ENOTEMPTY: return "ENOTEMPTY";
	case ENOTSOCK: return "ENOTSOCK";
	/* case ENOTSUP: return "ENOTSUP"; */
	case ENOTTY: return "ENOTTY";
	case ENOTUNIQ: return "ENOTUNIQ";
	case ENXIO: return "ENXIO";
	case EOPNOTSUPP: return "EOPNOTSUPP";
	case EOVERFLOW: return "EOVERFLOW";
	case EPERM: return "EPERM";
	case EPFNOSUPPORT: return "EPFNOSUPPORT";
	case EPIPE: return "EPIPE";
	case EPROTO: return "EPROTO";
	case EPROTONOSUPPORT: return "EPROTONOSUPPORT";
	case EPROTOTYPE: return "EPROTOTYPE";
	case ERANGE: return "ERANGE";
	case EREMOTE: return "EREMOTE";
	case EREMOTEIO: return "EREMOTEIO";

	case EROFS: return "EROFS";
	case ESHUTDOWN: return "ESHUTDOWN";
	case ESPIPE: return "ESPIPE";
	case ESOCKTNOSUPPORT: return "ESOCKTNOSUPPORT";
	case ESRCH: return "ESRCH";
	case ESTALE: return "ESTALE";
	case ETIME: return "ETIME";
	case ETIMEDOUT: return "ETIMEDOUT";
	case ETXTBSY: return "ETXTBSY";
	case EUNATCH: return "EUNATCH";
	case EUSERS: return "EUSERS";
	/* case EWOULDBLOCK: return "EWOULDBLOCK"; */
	case EXDEV: return "EXDEV";

	#ifdef EBADE
	/* Not available on OSX */
	case EBADE: return "EBADE";
	case EBADFD: return "EBADFD";
	case EBADSLT: return "EBADSLT";
	case EDEADLOCK: return "EDEADLOCK";
	case EBADR: return "EBADR";
	case EBADRQC: return "EBADRQC";
	case ECHRNG: return "ECHRNG";
	#ifdef EISNAM /* Not available on Illumos/Solaris */
	case EISNAM: return "EISNAM";
	case EKEYEXPIRED: return "EKEYEXPIRED";
	case EKEYREJECTED: return "EKEYREJECTED";
	case EKEYREVOKED: return "EKEYREVOKED";
	#endif
	case EL2HLT: return "EL2HLT";
	case EL2NSYNC: return "EL2NSYNC";
	case EL3HLT: return "EL3HLT";
	case EL3RST: return "EL3RST";
	case ELIBBAD: return "ELIBBAD";
	case ELIBMAX: return "ELIBMAX";
	case ELIBSCN: return "ELIBSCN";
	case ELIBEXEC: return "ELIBEXEC";
	#ifdef ENOMEDIUM /* Not available on Illumos/Solaris */
	case ENOMEDIUM: return "ENOMEDIUM";
	case EMEDIUMTYPE: return "EMEDIUMTYPE";
	#endif
	case ENONET: return "ENONET";
	case ENOPKG: return "ENOPKG";
	case EREMCHG: return "EREMCHG";
	case ERESTART: return "ERESTART";
	case ESTRPIPE: return "ESTRPIPE";
	#ifdef EUCLEAN /* Not available on Illumos/Solaris */
	case EUCLEAN: return "EUCLEAN";
	#endif
	case EXFULL: return "EXFULL";
	#endif

	case pcmk_err_generic: return "pcmk_err_generic";
	case pcmk_err_no_quorum: return "pcmk_err_no_quorum";
	case pcmk_err_schema_validation: return "pcmk_err_schema_validation";
	case pcmk_err_transform_failed: return "pcmk_err_transform_failed";
	case pcmk_err_old_data: return "pcmk_err_old_data";
	case pcmk_err_diff_failed: return "pcmk_err_diff_failed";
	case pcmk_err_diff_resync: return "pcmk_err_diff_resync";
	case pcmk_err_cib_modified: return "pcmk_err_cib_modified";
	case pcmk_err_cib_backup: return "pcmk_err_cib_backup";
	case pcmk_err_cib_save: return "pcmk_err_cib_save";
	case pcmk_err_cib_corrupt: return "pcmk_err_cib_corrupt";
	+ case pcmk_err_multiple: return "pcmk_err_multiple";
	+ case pcmk_err_node_unknown: return "pcmk_err_node_unknown";
	+ case pcmk_err_already: return "pcmk_err_already";
	}
	return "Unknown";
	}

	const char *
	pcmk_strerror(int rc)
	{
	int error = abs(rc);

	if (error == 0) {
	return "OK";
	} else if (error < PCMK_ERROR_OFFSET) {
	return strerror(error);
	}

	switch (error) {
	case pcmk_err_generic:
	return "Generic Pacemaker error";
	case pcmk_err_no_quorum:
	return "Operation requires quorum";
	case pcmk_err_schema_validation:
	return "Update does not conform to the configured schema";
	case pcmk_err_transform_failed:
	return "Schema transform failed";
	case pcmk_err_old_data:
	return "Update was older than existing configuration";
	case pcmk_err_diff_failed:
	return "Application of an update diff failed";
	case pcmk_err_diff_resync:
	return "Application of an update diff failed, requesting a full refresh";
	case pcmk_err_cib_modified:
	return "The on-disk configuration was manually modified";
	case pcmk_err_cib_backup:
	return "Could not archive the previous configuration";
	case pcmk_err_cib_save:
	return "Could not save the new configuration to disk";
	case pcmk_err_cib_corrupt:
	return "Could not parse on-disk configuration";
	-
	+ case pcmk_err_multiple:
	+ return "Resource active on multiple nodes";
	+ case pcmk_err_node_unknown:
	+ return "Node not found";
	+ case pcmk_err_already:
	+ return "Situation already as requested";
	case pcmk_err_schema_unchanged:
	return "Schema is already the latest available";

	/* The following cases will only be hit on systems for which they are non-standard */
	/* coverity[dead_error_condition] False positive on non-Linux */
	case ENOTUNIQ:
	return "Name not unique on network";
	/* coverity[dead_error_condition] False positive on non-Linux */
	case ECOMM:
	return "Communication error on send";
	/* coverity[dead_error_condition] False positive on non-Linux */
	case ELIBACC:
	return "Can not access a needed shared library";
	/* coverity[dead_error_condition] False positive on non-Linux */
	case EREMOTEIO:
	return "Remote I/O error";
	/* coverity[dead_error_condition] False positive on non-Linux */
	case EUNATCH:
	return "Protocol driver not attached";
	/* coverity[dead_error_condition] False positive on non-Linux */
	case ENOKEY:
	return "Required key not available";
	}

	crm_err("Unknown error code: %d", rc);
	return "Unknown error";
	}

	const char *
	crm_exit_name(crm_exit_t exit_code)
	{
	switch (exit_code) {
	case CRM_EX_OK: return "CRM_EX_OK";
	case CRM_EX_ERROR: return "CRM_EX_ERROR";
	case CRM_EX_INVALID_PARAM: return "CRM_EX_INVALID_PARAM";
	case CRM_EX_UNIMPLEMENT_FEATURE: return "CRM_EX_UNIMPLEMENT_FEATURE";
	case CRM_EX_INSUFFICIENT_PRIV: return "CRM_EX_INSUFFICIENT_PRIV";
	case CRM_EX_NOT_INSTALLED: return "CRM_EX_NOT_INSTALLED";
	case CRM_EX_NOT_CONFIGURED: return "CRM_EX_NOT_CONFIGURED";
	case CRM_EX_NOT_RUNNING: return "CRM_EX_NOT_RUNNING";
	case CRM_EX_USAGE: return "CRM_EX_USAGE";
	case CRM_EX_DATAERR: return "CRM_EX_DATAERR";
	case CRM_EX_NOINPUT: return "CRM_EX_NOINPUT";
	case CRM_EX_NOUSER: return "CRM_EX_NOUSER";
	case CRM_EX_NOHOST: return "CRM_EX_NOHOST";
	case CRM_EX_UNAVAILABLE: return "CRM_EX_UNAVAILABLE";
	case CRM_EX_SOFTWARE: return "CRM_EX_SOFTWARE";
	case CRM_EX_OSERR: return "CRM_EX_OSERR";
	case CRM_EX_OSFILE: return "CRM_EX_OSFILE";
	case CRM_EX_CANTCREAT: return "CRM_EX_CANTCREAT";
	case CRM_EX_IOERR: return "CRM_EX_IOERR";
	case CRM_EX_TEMPFAIL: return "CRM_EX_TEMPFAIL";
	case CRM_EX_PROTOCOL: return "CRM_EX_PROTOCOL";
	case CRM_EX_NOPERM: return "CRM_EX_NOPERM";
	case CRM_EX_CONFIG: return "CRM_EX_CONFIG";
	case CRM_EX_FATAL: return "CRM_EX_FATAL";
	case CRM_EX_PANIC: return "CRM_EX_PANIC";
	case CRM_EX_DISCONNECT: return "CRM_EX_DISCONNECT";
	case CRM_EX_DIGEST: return "CRM_EX_DIGEST";
	case CRM_EX_NOSUCH: return "CRM_EX_NOSUCH";
	case CRM_EX_QUORUM: return "CRM_EX_QUORUM";
	case CRM_EX_UNSAFE: return "CRM_EX_UNSAFE";
	case CRM_EX_EXISTS: return "CRM_EX_EXISTS";
	case CRM_EX_MULTIPLE: return "CRM_EX_MULTIPLE";
	case CRM_EX_OLD: return "CRM_EX_OLD";
	case CRM_EX_TIMEOUT: return "CRM_EX_TIMEOUT";
	case CRM_EX_MAX: return "CRM_EX_UNKNOWN";
	}
	return "CRM_EX_UNKNOWN";
	}

	const char *
	crm_exit_str(crm_exit_t exit_code)
	{
	switch (exit_code) {
	case CRM_EX_OK: return "OK";
	case CRM_EX_ERROR: return "Error occurred";
	case CRM_EX_INVALID_PARAM: return "Invalid parameter";
	case CRM_EX_UNIMPLEMENT_FEATURE: return "Unimplemented";
	case CRM_EX_INSUFFICIENT_PRIV: return "Insufficient privileges";
	case CRM_EX_NOT_INSTALLED: return "Not installed";
	case CRM_EX_NOT_CONFIGURED: return "Not configured";
	case CRM_EX_NOT_RUNNING: return "Not running";
	case CRM_EX_USAGE: return "Incorrect usage";
	case CRM_EX_DATAERR: return "Invalid data given";
	case CRM_EX_NOINPUT: return "Input file not available";
	case CRM_EX_NOUSER: return "User does not exist";
	case CRM_EX_NOHOST: return "Host does not exist";
	case CRM_EX_UNAVAILABLE: return "Necessary service unavailable";
	case CRM_EX_SOFTWARE: return "Internal software bug";
	case CRM_EX_OSERR: return "Operating system error occurred";
	case CRM_EX_OSFILE: return "System file not available";
	case CRM_EX_CANTCREAT: return "Cannot create output file";
	case CRM_EX_IOERR: return "I/O error occurred";
	case CRM_EX_TEMPFAIL: return "Temporary failure, try again";
	case CRM_EX_PROTOCOL: return "Protocol violated";
	case CRM_EX_NOPERM: return "Insufficient privileges";
	case CRM_EX_CONFIG: return "Invalid configuration";
	case CRM_EX_FATAL: return "Fatal error occurred, will not respawn";
	case CRM_EX_PANIC: return "System panic required";
	case CRM_EX_DISCONNECT: return "Not connected";
	case CRM_EX_DIGEST: return "Digest mismatch";
	case CRM_EX_NOSUCH: return "No such object";
	case CRM_EX_QUORUM: return "Quorum required";
	case CRM_EX_UNSAFE: return "Operation not safe";
	case CRM_EX_EXISTS: return "Requested item already exists";
	case CRM_EX_MULTIPLE: return "Multiple items match request";
	case CRM_EX_OLD: return "Update was older than existing configuration";
	case CRM_EX_TIMEOUT: return "Timeout occurred";
	case CRM_EX_MAX: return "Error occurred";
	}
	if (exit_code > 128) {
	return "Interrupted by signal";
	}
	return "Unknown exit status";
	}

	/*!
	* \brief Map an errno to a similar exit status
	*
	* \param[in] errno Error number to map
	*
	* \return Exit status corresponding to errno
	*/
	crm_exit_t
	crm_errno2exit(int rc)
	{
	rc = abs(rc); // Convenience for functions that return -errno
	if (rc == EOPNOTSUPP) {
	rc = ENOTSUP; // Values are same on Linux, can't use both in case
	}
	switch (rc) {
	case pcmk_ok:
	return CRM_EX_OK;

	case pcmk_err_no_quorum:
	return CRM_EX_QUORUM;

	case pcmk_err_old_data:
	return CRM_EX_OLD;

	case pcmk_err_schema_validation:
	case pcmk_err_transform_failed:
	return CRM_EX_CONFIG;

	case EACCES:
	return CRM_EX_INSUFFICIENT_PRIV;

	case EBADF:
	case EINVAL:
	case EFAULT:
	case ENOSYS:
	case EOVERFLOW:
	return CRM_EX_SOFTWARE;

	case EBADMSG:
	case EMSGSIZE:
	case ENOMSG:
	case ENOPROTOOPT:
	case EPROTO:
	case EPROTONOSUPPORT:
	case EPROTOTYPE:
	return CRM_EX_PROTOCOL;

	case ECOMM:
	case ENOMEM:
	return CRM_EX_OSERR;

	case ECONNABORTED:
	case ECONNREFUSED:
	case ECONNRESET:
	case ENOTCONN:
	return CRM_EX_DISCONNECT;

	case EEXIST:
	+ case pcmk_err_already:
	return CRM_EX_EXISTS;

	case EIO:
	return CRM_EX_IOERR;

	case ENOTSUP:
	return CRM_EX_UNIMPLEMENT_FEATURE;

	case ENOTUNIQ:
	+ case pcmk_err_multiple:
	return CRM_EX_MULTIPLE;

	case ENXIO:
	+ case pcmk_err_node_unknown:
	return CRM_EX_NOSUCH;

	case ETIME:
	case ETIMEDOUT:
	return CRM_EX_TIMEOUT;

	default:
	return CRM_EX_ERROR;
	}
	}

	const char *
	bz2_strerror(int rc)
	{
	/* http://www.bzip.org/1.0.3/html/err-handling.html */
	switch (rc) {
	case BZ_OK:
	case BZ_RUN_OK:
	case BZ_FLUSH_OK:
	case BZ_FINISH_OK:
	case BZ_STREAM_END:
	return "Ok";
	case BZ_CONFIG_ERROR:
	return "libbz2 has been improperly compiled on your platform";
	case BZ_SEQUENCE_ERROR:
	return "library functions called in the wrong order";
	case BZ_PARAM_ERROR:
	return "parameter is out of range or otherwise incorrect";
	case BZ_MEM_ERROR:
	return "memory allocation failed";
	case BZ_DATA_ERROR:
	return "data integrity error is detected during decompression";
	case BZ_DATA_ERROR_MAGIC:
	return "the compressed stream does not start with the correct magic bytes";
	case BZ_IO_ERROR:
	return "error reading or writing in the compressed file";
	case BZ_UNEXPECTED_EOF:
	return "compressed file finishes before the logical end of stream is detected";
	case BZ_OUTBUFF_FULL:
	return "output data will not fit into the buffer provided";
	}
	return "Unknown error";
	}

	crm_exit_t
	crm_exit(crm_exit_t rc)
	{
	/* A compiler could theoretically use any type for crm_exit_t, but an int
	* should always hold it, so cast to int to keep static analysis happy.
	*/
	if ((((int) rc) < 0) \|\| (((int) rc) > CRM_EX_MAX)) {
	rc = CRM_EX_ERROR;
	}

	mainloop_cleanup();
	crm_xml_cleanup();

	qb_log_fini();
	crm_args_fini();

	if (crm_system_name) {
	crm_info("Exiting %s " CRM_XS " with status %d", crm_system_name, rc);
	free(crm_system_name);
	} else {
	crm_trace("Exiting with status %d", rc);
	}

	exit(rc);
	return rc; /* Can never happen, but allows return crm_exit(rc)
	* where "return rc" was used previously - which
	* keeps compilers happy.
	*/
	}
	diff --git a/lib/services/services_linux.c b/lib/services/services_linux.c
	index 6263cb14dc..bd22777b92 100644
	--- a/lib/services/services_linux.c
	+++ b/lib/services/services_linux.c
	@@ -1,932 +1,939 @@
	/*
	* Copyright (C) 2010-2016 Andrew Beekhof <andrew@beekhof.net>
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <sys/types.h>
	#include <sys/stat.h>
	#include <sys/wait.h>
	#include <errno.h>
	#include <unistd.h>
	#include <dirent.h>
	#include <grp.h>
	#include <string.h>
	#include <sys/time.h>
	#include <sys/resource.h>

	#ifdef HAVE_SYS_SIGNALFD_H
	#include <sys/signalfd.h>
	#endif

	#include "crm/crm.h"
	#include "crm/common/mainloop.h"
	#include "crm/services.h"

	#include "services_private.h"

	#if SUPPORT_CIBSECRETS
	# include "crm/common/cib_secrets.h"
	#endif

	static gboolean
	svc_read_output(int fd, svc_action_t * op, bool is_stderr)
	{
	char *data = NULL;
	int rc = 0, len = 0;
	char buf[500];
	static const size_t buf_read_len = sizeof(buf) - 1;


	if (fd < 0) {
	crm_trace("No fd for %s", op->id);
	return FALSE;
	}

	if (is_stderr && op->stderr_data) {
	len = strlen(op->stderr_data);
	data = op->stderr_data;
	crm_trace("Reading %s stderr into offset %d", op->id, len);

	} else if (is_stderr == FALSE && op->stdout_data) {
	len = strlen(op->stdout_data);
	data = op->stdout_data;
	crm_trace("Reading %s stdout into offset %d", op->id, len);

	} else {
	crm_trace("Reading %s %s into offset %d", op->id, is_stderr?"stderr":"stdout", len);
	}

	do {
	rc = read(fd, buf, buf_read_len);
	if (rc > 0) {
	crm_trace("Got %d chars: %.80s", rc, buf);
	buf[rc] = 0;
	data = realloc_safe(data, len + rc + 1);
	len += sprintf(data + len, "%s", buf);

	} else if (errno != EINTR) {
	/* error or EOF
	* Cleanup happens in pipe_done()
	*/
	rc = FALSE;
	break;
	}

	} while (rc == buf_read_len \|\| rc < 0);

	if (is_stderr) {
	op->stderr_data = data;
	} else {
	op->stdout_data = data;
	}

	return rc;
	}

	static int
	dispatch_stdout(gpointer userdata)
	{
	svc_action_t op = (svc_action_t ) userdata;

	return svc_read_output(op->opaque->stdout_fd, op, FALSE);
	}

	static int
	dispatch_stderr(gpointer userdata)
	{
	svc_action_t op = (svc_action_t ) userdata;

	return svc_read_output(op->opaque->stderr_fd, op, TRUE);
	}

	static void
	pipe_out_done(gpointer user_data)
	{
	svc_action_t op = (svc_action_t ) user_data;

	crm_trace("%p", op);

	op->opaque->stdout_gsource = NULL;
	if (op->opaque->stdout_fd > STDOUT_FILENO) {
	close(op->opaque->stdout_fd);
	}
	op->opaque->stdout_fd = -1;
	}

	static void
	pipe_err_done(gpointer user_data)
	{
	svc_action_t op = (svc_action_t ) user_data;

	op->opaque->stderr_gsource = NULL;
	if (op->opaque->stderr_fd > STDERR_FILENO) {
	close(op->opaque->stderr_fd);
	}
	op->opaque->stderr_fd = -1;
	}

	static struct mainloop_fd_callbacks stdout_callbacks = {
	.dispatch = dispatch_stdout,
	.destroy = pipe_out_done,
	};

	static struct mainloop_fd_callbacks stderr_callbacks = {
	.dispatch = dispatch_stderr,
	.destroy = pipe_err_done,
	};

	static void
	set_ocf_env(const char key, const char value, gpointer user_data)
	{
	if (setenv(key, value, 1) != 0) {
	crm_perror(LOG_ERR, "setenv failed for key:%s and value:%s", key, value);
	}
	}

	static void
	set_ocf_env_with_prefix(gpointer key, gpointer value, gpointer user_data)
	{
	char buffer[500];

	snprintf(buffer, sizeof(buffer), "OCF_RESKEY_%s", (char *)key);
	set_ocf_env(buffer, value, user_data);
	}

	/*!
	* \internal
	* \brief Add environment variables suitable for an action
	*
	* \param[in] op Action to use
	*/
	static void
	add_action_env_vars(const svc_action_t *op)
	{
	if (safe_str_eq(op->standard, PCMK_RESOURCE_CLASS_OCF) == FALSE) {
	return;
	}

	if (op->params) {
	g_hash_table_foreach(op->params, set_ocf_env_with_prefix, NULL);
	}

	set_ocf_env("OCF_RA_VERSION_MAJOR", "1", NULL);
	set_ocf_env("OCF_RA_VERSION_MINOR", "0", NULL);
	set_ocf_env("OCF_ROOT", OCF_ROOT_DIR, NULL);
	set_ocf_env("OCF_EXIT_REASON_PREFIX", PCMK_OCF_REASON_PREFIX, NULL);

	if (op->rsc) {
	set_ocf_env("OCF_RESOURCE_INSTANCE", op->rsc, NULL);
	}

	if (op->agent != NULL) {
	set_ocf_env("OCF_RESOURCE_TYPE", op->agent, NULL);
	}

	/* Notes: this is not added to specification yet. Sept 10,2004 */
	if (op->provider != NULL) {
	set_ocf_env("OCF_RESOURCE_PROVIDER", op->provider, NULL);
	}
	}

	gboolean
	recurring_action_timer(gpointer data)
	{
	svc_action_t *op = data;

	crm_debug("Scheduling another invocation of %s", op->id);

	/* Clean out the old result */
	free(op->stdout_data);
	op->stdout_data = NULL;
	free(op->stderr_data);
	op->stderr_data = NULL;
	op->opaque->repeat_timer = 0;

	services_action_async(op, NULL);
	return FALSE;
	}

	/* Returns FALSE if 'op' should be free'd by the caller */
	gboolean
	operation_finalize(svc_action_t * op)
	{
	int recurring = 0;

	if (op->interval_ms) {
	if (op->cancel) {
	op->status = PCMK_LRM_OP_CANCELLED;
	cancel_recurring_action(op);
	} else {
	recurring = 1;
	op->opaque->repeat_timer = g_timeout_add(op->interval_ms,
	recurring_action_timer, (void *)op);
	}
	}

	if (op->opaque->callback) {
	op->opaque->callback(op);
	}

	op->pid = 0;

	services_untrack_op(op);

	if (!recurring && op->synchronous == FALSE) {
	/*
	* If this is a recurring action, do not free explicitly.
	* It will get freed whenever the action gets cancelled.
	*/
	services_action_free(op);
	return TRUE;
	}

	services_action_cleanup(op);
	return FALSE;
	}

	static void
	operation_finished(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode)
	{
	svc_action_t *op = mainloop_child_userdata(p);
	char *prefix = crm_strdup_printf("%s:%d", op->id, op->pid);

	mainloop_clear_child_userdata(p);
	op->status = PCMK_LRM_OP_DONE;
	CRM_ASSERT(op->pid == pid);

	crm_trace("%s %p %p", prefix, op->opaque->stderr_gsource, op->opaque->stdout_gsource);
	if (op->opaque->stderr_gsource) {
	/* Make sure we have read everything from the buffer.
	* Depending on the priority mainloop gives the fd, operation_finished
	* could occur before all the reads are done. Force the read now.*/
	crm_trace("%s dispatching stderr", prefix);
	dispatch_stderr(op);
	crm_trace("%s: %p", op->id, op->stderr_data);
	mainloop_del_fd(op->opaque->stderr_gsource);
	op->opaque->stderr_gsource = NULL;
	}

	if (op->opaque->stdout_gsource) {
	/* Make sure we have read everything from the buffer.
	* Depending on the priority mainloop gives the fd, operation_finished
	* could occur before all the reads are done. Force the read now.*/
	crm_trace("%s dispatching stdout", prefix);
	dispatch_stdout(op);
	crm_trace("%s: %p", op->id, op->stdout_data);
	mainloop_del_fd(op->opaque->stdout_gsource);
	op->opaque->stdout_gsource = NULL;
	}

	if (signo) {
	if (mainloop_child_timeout(p)) {
	crm_warn("%s - timed out after %dms", prefix, op->timeout);
	op->status = PCMK_LRM_OP_TIMEOUT;
	op->rc = PCMK_OCF_TIMEOUT;

	+ } else if (op->cancel) {
	+ /* If an in-flight recurring operation was killed because it was
	+ * cancelled, don't treat that as a failure.
	+ */
	+ crm_info("%s - terminated with signal %d", prefix, signo);
	+ op->status = PCMK_LRM_OP_CANCELLED;
	+ op->rc = PCMK_OCF_OK;
	+
	} else {
	- do_crm_log_unlikely((op->cancel) ? LOG_INFO : LOG_WARNING,
	- "%s - terminated with signal %d", prefix, signo);
	+ crm_warn("%s - terminated with signal %d", prefix, signo);
	op->status = PCMK_LRM_OP_ERROR;
	op->rc = PCMK_OCF_SIGNAL;
	}

	} else {
	op->rc = exitcode;
	crm_debug("%s - exited with rc=%d", prefix, exitcode);
	}

	free(prefix);
	prefix = crm_strdup_printf("%s:%d:stderr", op->id, op->pid);
	crm_log_output(LOG_NOTICE, prefix, op->stderr_data);

	free(prefix);
	prefix = crm_strdup_printf("%s:%d:stdout", op->id, op->pid);
	crm_log_output(LOG_DEBUG, prefix, op->stdout_data);

	free(prefix);
	operation_finalize(op);
	}

	/*!
	* \internal
	* \brief Set operation rc and status per errno from stat(), fork() or execvp()
	*
	* \param[in,out] op Operation to set rc and status for
	* \param[in] error Value of errno after system call
	*
	* \return void
	*/
	static void
	services_handle_exec_error(svc_action_t * op, int error)
	{
	int rc_not_installed, rc_insufficient_priv, rc_exec_error;

	/* Mimic the return codes for each standard as that's what we'll convert back from in get_uniform_rc() */
	if (safe_str_eq(op->standard, PCMK_RESOURCE_CLASS_LSB)
	&& safe_str_eq(op->action, "status")) {

	rc_not_installed = PCMK_LSB_STATUS_NOT_INSTALLED;
	rc_insufficient_priv = PCMK_LSB_STATUS_INSUFFICIENT_PRIV;
	rc_exec_error = PCMK_LSB_STATUS_UNKNOWN;

	#if SUPPORT_NAGIOS
	} else if (safe_str_eq(op->standard, PCMK_RESOURCE_CLASS_NAGIOS)) {
	rc_not_installed = NAGIOS_NOT_INSTALLED;
	rc_insufficient_priv = NAGIOS_INSUFFICIENT_PRIV;
	rc_exec_error = PCMK_OCF_EXEC_ERROR;
	#endif

	} else {
	rc_not_installed = PCMK_OCF_NOT_INSTALLED;
	rc_insufficient_priv = PCMK_OCF_INSUFFICIENT_PRIV;
	rc_exec_error = PCMK_OCF_EXEC_ERROR;
	}

	switch (error) { /* see execve(2), stat(2) and fork(2) */
	case ENOENT: /* No such file or directory */
	case EISDIR: /* Is a directory */
	case ENOTDIR: /* Path component is not a directory */
	case EINVAL: /* Invalid executable format */
	case ENOEXEC: /* Invalid executable format */
	op->rc = rc_not_installed;
	op->status = PCMK_LRM_OP_NOT_INSTALLED;
	break;
	case EACCES: /* permission denied (various errors) */
	case EPERM: /* permission denied (various errors) */
	op->rc = rc_insufficient_priv;
	op->status = PCMK_LRM_OP_ERROR;
	break;
	default:
	op->rc = rc_exec_error;
	op->status = PCMK_LRM_OP_ERROR;
	}
	}

	static void
	action_launch_child(svc_action_t *op)
	{
	int lpc;

	/* SIGPIPE is ignored (which is different from signal blocking) by the gnutls library.
	* Depending on the libqb version in use, libqb may set SIGPIPE to be ignored as well.
	* We do not want this to be inherited by the child process. By resetting this the signal
	* to the default behavior, we avoid some potential odd problems that occur during OCF
	* scripts when SIGPIPE is ignored by the environment. */
	signal(SIGPIPE, SIG_DFL);

	#if defined(HAVE_SCHED_SETSCHEDULER)
	if (sched_getscheduler(0) != SCHED_OTHER) {
	struct sched_param sp;

	memset(&sp, 0, sizeof(sp));
	sp.sched_priority = 0;

	if (sched_setscheduler(0, SCHED_OTHER, &sp) == -1) {
	crm_perror(LOG_ERR, "Could not reset scheduling policy to SCHED_OTHER for %s", op->id);
	}
	}
	#endif
	if (setpriority(PRIO_PROCESS, 0, 0) == -1) {
	crm_perror(LOG_ERR, "Could not reset process priority to 0 for %s", op->id);
	}

	/* Man: The call setpgrp() is equivalent to setpgid(0,0)
	* _and_ compiles on BSD variants too
	* need to investigate if it works the same too.
	*/
	setpgid(0, 0);

	// Close all file descriptors except stdin/stdout/stderr
	for (lpc = getdtablesize() - 1; lpc > STDERR_FILENO; lpc--) {
	close(lpc);
	}

	#if SUPPORT_CIBSECRETS
	if (replace_secret_params(op->rsc, op->params) < 0) {
	/* replacing secrets failed! */
	if (safe_str_eq(op->action,"stop")) {
	/* don't fail on stop! */
	crm_info("proceeding with the stop operation for %s", op->rsc);

	} else {
	crm_err("failed to get secrets for %s, "
	"considering resource not configured", op->rsc);
	_exit(PCMK_OCF_NOT_CONFIGURED);
	}
	}
	#endif

	add_action_env_vars(op);

	/* Become the desired user */
	if (op->opaque->uid && (geteuid() == 0)) {

	// If requested, set effective group
	if (op->opaque->gid && (setgid(op->opaque->gid) < 0)) {
	crm_perror(LOG_ERR, "Could not set child group to %d", op->opaque->gid);
	_exit(PCMK_OCF_NOT_CONFIGURED);
	}

	// Erase supplementary group list
	// (We could do initgroups() if we kept a copy of the username)
	if (setgroups(0, NULL) < 0) {
	crm_perror(LOG_ERR, "Could not set child groups");
	_exit(PCMK_OCF_NOT_CONFIGURED);
	}

	// Set effective user
	if (setuid(op->opaque->uid) < 0) {
	crm_perror(LOG_ERR, "setting user to %d", op->opaque->uid);
	_exit(PCMK_OCF_NOT_CONFIGURED);
	}
	}

	/* execute the RA */
	execvp(op->opaque->exec, op->opaque->args);

	/* Most cases should have been already handled by stat() */
	services_handle_exec_error(op, errno);

	_exit(op->rc);
	}

	#ifndef HAVE_SYS_SIGNALFD_H
	static int sigchld_pipe[2] = { -1, -1 };

	static void
	sigchld_handler()
	{
	if ((sigchld_pipe[1] >= 0) && (write(sigchld_pipe[1], "", 1) == -1)) {
	crm_perror(LOG_TRACE, "Could not poke SIGCHLD self-pipe");
	}
	}
	#endif

	static void
	action_synced_wait(svc_action_t * op, sigset_t *mask)
	{
	int status = 0;
	int timeout = op->timeout;
	int sfd = -1;
	time_t start = -1;
	struct pollfd fds[3];
	int wait_rc = 0;

	#ifdef HAVE_SYS_SIGNALFD_H
	sfd = signalfd(-1, mask, SFD_NONBLOCK);
	if (sfd < 0) {
	crm_perror(LOG_ERR, "signalfd() failed");
	}
	#else
	sfd = sigchld_pipe[0];
	#endif

	fds[0].fd = op->opaque->stdout_fd;
	fds[0].events = POLLIN;
	fds[0].revents = 0;

	fds[1].fd = op->opaque->stderr_fd;
	fds[1].events = POLLIN;
	fds[1].revents = 0;

	fds[2].fd = sfd;
	fds[2].events = POLLIN;
	fds[2].revents = 0;

	crm_trace("Waiting for %d", op->pid);
	start = time(NULL);
	do {
	int poll_rc = poll(fds, 3, timeout);

	if (poll_rc > 0) {
	if (fds[0].revents & POLLIN) {
	svc_read_output(op->opaque->stdout_fd, op, FALSE);
	}

	if (fds[1].revents & POLLIN) {
	svc_read_output(op->opaque->stderr_fd, op, TRUE);
	}

	if (fds[2].revents & POLLIN) {
	#ifdef HAVE_SYS_SIGNALFD_H
	struct signalfd_siginfo fdsi;
	ssize_t s;

	s = read(sfd, &fdsi, sizeof(struct signalfd_siginfo));
	if (s != sizeof(struct signalfd_siginfo)) {
	crm_perror(LOG_ERR, "Read from signal fd %d failed", sfd);

	} else if (fdsi.ssi_signo == SIGCHLD) {
	#else
	if (1) {
	/* Clear out the sigchld pipe. */
	char ch;
	while (read(sfd, &ch, 1) == 1) /omit/;
	#endif
	wait_rc = waitpid(op->pid, &status, WNOHANG);

	if (wait_rc > 0) {
	break;

	} else if (wait_rc < 0){
	if (errno == ECHILD) {
	/* Here, don't dare to kill and bail out... */
	break;

	} else {
	/* ...otherwise pretend process still runs. */
	wait_rc = 0;
	}
	crm_perror(LOG_ERR, "waitpid() for %d failed", op->pid);
	}
	}
	}

	} else if (poll_rc == 0) {
	timeout = 0;
	break;

	} else if (poll_rc < 0) {
	if (errno != EINTR) {
	crm_perror(LOG_ERR, "poll() failed");
	break;
	}
	}

	timeout = op->timeout - (time(NULL) - start) * 1000;

	} while ((op->timeout < 0 \|\| timeout > 0));

	crm_trace("Child done: %d", op->pid);
	if (wait_rc <= 0) {
	op->rc = PCMK_OCF_UNKNOWN_ERROR;

	if (op->timeout > 0 && timeout <= 0) {
	op->status = PCMK_LRM_OP_TIMEOUT;
	crm_warn("%s:%d - timed out after %dms", op->id, op->pid, op->timeout);

	} else {
	op->status = PCMK_LRM_OP_ERROR;
	}

	/* If only child hasn't been successfully waited for, yet.
	This is to limit killing wrong target a bit more. */
	if (wait_rc == 0 && waitpid(op->pid, &status, WNOHANG) == 0) {
	if (kill(op->pid, SIGKILL)) {
	crm_err("kill(%d, KILL) failed: %d", op->pid, errno);
	}
	/* Safe to skip WNOHANG here as we sent non-ignorable signal. */
	while (waitpid(op->pid, &status, 0) == (pid_t) -1 && errno == EINTR) /omit/;
	}

	} else if (WIFEXITED(status)) {
	op->status = PCMK_LRM_OP_DONE;
	op->rc = WEXITSTATUS(status);
	crm_info("Managed %s process %d exited with rc=%d", op->id, op->pid, op->rc);

	} else if (WIFSIGNALED(status)) {
	int signo = WTERMSIG(status);

	op->status = PCMK_LRM_OP_ERROR;
	crm_err("Managed %s process %d exited with signal=%d", op->id, op->pid, signo);
	}
	#ifdef WCOREDUMP
	if (WCOREDUMP(status)) {
	crm_err("Managed %s process %d dumped core", op->id, op->pid);
	}
	#endif

	svc_read_output(op->opaque->stdout_fd, op, FALSE);
	svc_read_output(op->opaque->stderr_fd, op, TRUE);

	close(op->opaque->stdout_fd);
	close(op->opaque->stderr_fd);

	#ifdef HAVE_SYS_SIGNALFD_H
	close(sfd);
	#endif
	}

	/* For an asynchronous 'op', returns FALSE if 'op' should be free'd by the caller */
	/* For a synchronous 'op', returns FALSE if 'op' fails */
	gboolean
	services_os_action_execute(svc_action_t * op)
	{
	int stdout_fd[2];
	int stderr_fd[2];
	int rc;
	struct stat st;
	sigset_t *pmask;

	#ifdef HAVE_SYS_SIGNALFD_H
	sigset_t mask;
	sigset_t old_mask;
	#define sigchld_cleanup() do { \
	if (sigismember(&old_mask, SIGCHLD) == 0) { \
	if (sigprocmask(SIG_UNBLOCK, &mask, NULL) < 0) { \
	crm_perror(LOG_ERR, "sigprocmask() failed to unblock sigchld"); \
	} \
	} \
	} while (0)
	#else
	struct sigaction sa;
	struct sigaction old_sa;
	#define sigchld_cleanup() do { \
	if (sigaction(SIGCHLD, &old_sa, NULL) < 0) { \
	crm_perror(LOG_ERR, "sigaction() failed to remove sigchld handler"); \
	} \
	close(sigchld_pipe[0]); \
	close(sigchld_pipe[1]); \
	sigchld_pipe[0] = sigchld_pipe[1] = -1; \
	} while(0)
	#endif

	/* Fail fast */
	if(stat(op->opaque->exec, &st) != 0) {
	rc = errno;
	crm_warn("Cannot execute '%s': %s (%d)", op->opaque->exec, pcmk_strerror(rc), rc);
	services_handle_exec_error(op, rc);
	if (!op->synchronous) {
	return operation_finalize(op);
	}
	return FALSE;
	}

	if (pipe(stdout_fd) < 0) {
	rc = errno;

	crm_err("pipe(stdout_fd) failed. '%s': %s (%d)", op->opaque->exec, pcmk_strerror(rc), rc);

	services_handle_exec_error(op, rc);
	if (!op->synchronous) {
	return operation_finalize(op);
	}
	return FALSE;
	}

	if (pipe(stderr_fd) < 0) {
	rc = errno;

	close(stdout_fd[0]);
	close(stdout_fd[1]);

	crm_err("pipe(stderr_fd) failed. '%s': %s (%d)", op->opaque->exec, pcmk_strerror(rc), rc);

	services_handle_exec_error(op, rc);
	if (!op->synchronous) {
	return operation_finalize(op);
	}
	return FALSE;
	}

	if (op->synchronous) {
	#ifdef HAVE_SYS_SIGNALFD_H
	sigemptyset(&mask);
	sigaddset(&mask, SIGCHLD);
	sigemptyset(&old_mask);

	if (sigprocmask(SIG_BLOCK, &mask, &old_mask) < 0) {
	crm_perror(LOG_ERR, "sigprocmask() failed to block sigchld");
	}

	pmask = &mask;
	#else
	if(pipe(sigchld_pipe) == -1) {
	crm_perror(LOG_ERR, "pipe() failed");
	}

	rc = crm_set_nonblocking(sigchld_pipe[0]);
	if (rc < 0) {
	crm_warn("Could not set pipe input non-blocking: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	}
	rc = crm_set_nonblocking(sigchld_pipe[1]);
	if (rc < 0) {
	crm_warn("Could not set pipe output non-blocking: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	}

	sa.sa_handler = sigchld_handler;
	sa.sa_flags = 0;
	sigemptyset(&sa.sa_mask);
	if (sigaction(SIGCHLD, &sa, &old_sa) < 0) {
	crm_perror(LOG_ERR, "sigaction() failed to set sigchld handler");
	}

	pmask = NULL;
	#endif
	}

	op->pid = fork();
	switch (op->pid) {
	case -1:
	rc = errno;

	close(stdout_fd[0]);
	close(stdout_fd[1]);
	close(stderr_fd[0]);
	close(stderr_fd[1]);

	crm_err("Could not execute '%s': %s (%d)", op->opaque->exec, pcmk_strerror(rc), rc);
	services_handle_exec_error(op, rc);
	if (!op->synchronous) {
	return operation_finalize(op);
	}

	sigchld_cleanup();
	return FALSE;

	case 0: /* Child */
	close(stdout_fd[0]);
	close(stderr_fd[0]);
	if (STDOUT_FILENO != stdout_fd[1]) {
	if (dup2(stdout_fd[1], STDOUT_FILENO) != STDOUT_FILENO) {
	crm_err("dup2() failed (stdout)");
	}
	close(stdout_fd[1]);
	}
	if (STDERR_FILENO != stderr_fd[1]) {
	if (dup2(stderr_fd[1], STDERR_FILENO) != STDERR_FILENO) {
	crm_err("dup2() failed (stderr)");
	}
	close(stderr_fd[1]);
	}

	if (op->synchronous) {
	sigchld_cleanup();
	}

	action_launch_child(op);
	CRM_ASSERT(0); /* action_launch_child is effectively noreturn */
	}

	/* Only the parent reaches here */
	close(stdout_fd[1]);
	close(stderr_fd[1]);

	op->opaque->stdout_fd = stdout_fd[0];
	rc = crm_set_nonblocking(op->opaque->stdout_fd);
	if (rc < 0) {
	crm_warn("Could not set child output non-blocking: %s "
	CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	}

	op->opaque->stderr_fd = stderr_fd[0];
	rc = crm_set_nonblocking(op->opaque->stderr_fd);
	if (rc < 0) {
	crm_warn("Could not set child error output non-blocking: %s "
	CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	}

	if (op->synchronous) {
	action_synced_wait(op, pmask);
	sigchld_cleanup();
	} else {

	crm_trace("Async waiting for %d - %s", op->pid, op->opaque->exec);
	mainloop_child_add_with_flags(op->pid,
	op->timeout,
	op->id,
	op,
	(op->flags & SVC_ACTION_LEAVE_GROUP) ? mainloop_leave_pid_group : 0,
	operation_finished);


	op->opaque->stdout_gsource = mainloop_add_fd(op->id,
	G_PRIORITY_LOW,
	op->opaque->stdout_fd, op, &stdout_callbacks);

	op->opaque->stderr_gsource = mainloop_add_fd(op->id,
	G_PRIORITY_LOW,
	op->opaque->stderr_fd, op, &stderr_callbacks);

	services_add_inflight_op(op);
	}

	return TRUE;
	}

	GList *
	services_os_get_directory_list(const char *root, gboolean files, gboolean executable)
	{
	GList *list = NULL;
	struct dirent **namelist;
	int entries = 0, lpc = 0;
	char buffer[PATH_MAX];

	entries = scandir(root, &namelist, NULL, alphasort);
	if (entries <= 0) {
	return list;
	}

	for (lpc = 0; lpc < entries; lpc++) {
	struct stat sb;

	if ('.' == namelist[lpc]->d_name[0]) {
	free(namelist[lpc]);
	continue;
	}

	snprintf(buffer, sizeof(buffer), "%s/%s", root, namelist[lpc]->d_name);

	if (stat(buffer, &sb)) {
	continue;
	}

	if (S_ISDIR(sb.st_mode)) {
	if (files) {
	free(namelist[lpc]);
	continue;
	}

	} else if (S_ISREG(sb.st_mode)) {
	if (files == FALSE) {
	free(namelist[lpc]);
	continue;

	} else if (executable
	&& (sb.st_mode & S_IXUSR) == 0
	&& (sb.st_mode & S_IXGRP) == 0 && (sb.st_mode & S_IXOTH) == 0) {
	free(namelist[lpc]);
	continue;
	}
	}

	list = g_list_append(list, strdup(namelist[lpc]->d_name));

	free(namelist[lpc]);
	}

	free(namelist);
	return list;
	}

	GList *
	resources_os_list_ocf_providers(void)
	{
	return get_directory_list(OCF_ROOT_DIR "/resource.d", FALSE, TRUE);
	}

	GList *
	resources_os_list_ocf_agents(const char *provider)
	{
	GList *gIter = NULL;
	GList *result = NULL;
	GList *providers = NULL;

	if (provider) {
	char buffer[500];

	snprintf(buffer, sizeof(buffer), "%s/resource.d/%s", OCF_ROOT_DIR, provider);
	return get_directory_list(buffer, TRUE, TRUE);
	}

	providers = resources_os_list_ocf_providers();
	for (gIter = providers; gIter != NULL; gIter = gIter->next) {
	GList *tmp1 = result;
	GList *tmp2 = resources_os_list_ocf_agents(gIter->data);

	if (tmp2) {
	result = g_list_concat(tmp1, tmp2);
	}
	}
	g_list_free_full(providers, free);
	return result;
	}

	#if SUPPORT_NAGIOS
	GList *
	resources_os_list_nagios_agents(void)
	{
	GList *plugin_list = NULL;
	GList *result = NULL;
	GList *gIter = NULL;

	plugin_list = get_directory_list(NAGIOS_PLUGIN_DIR, TRUE, TRUE);

	/* Make sure both the plugin and its metadata exist */
	for (gIter = plugin_list; gIter != NULL; gIter = gIter->next) {
	const char *plugin = gIter->data;
	char *metadata = crm_strdup_printf(NAGIOS_METADATA_DIR "/%s.xml", plugin);
	struct stat st;

	if (stat(metadata, &st) == 0) {
	result = g_list_append(result, strdup(plugin));
	}

	free(metadata);
	}
	g_list_free_full(plugin_list, free);
	return result;
	}
	#endif
	diff --git a/tools/crm_attribute.c b/tools/crm_attribute.c
	index f0bfdd214d..107dad0160 100644
	--- a/tools/crm_attribute.c
	+++ b/tools/crm_attribute.c
	@@ -1,338 +1,339 @@
	/*
	* Copyright 2004-2018 Andrew Beekhof <andrew@beekhof.net>
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>
	#include <time.h>

	#include <sys/param.h>
	#include <sys/types.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/util.h>
	#include <crm/cluster.h>

	#include <crm/cib.h>
	#include <crm/attrd.h>
	#include <sys/utsname.h>

	gboolean BE_QUIET = FALSE;
	char command = 'G';

	const char *dest_uname = NULL;
	char *dest_node = NULL;
	char *set_name = NULL;
	char *attr_id = NULL;
	char *attr_name = NULL;
	char *attr_pattern = NULL;
	const char *type = NULL;
	const char *rsc_id = NULL;
	const char *attr_value = NULL;
	const char *attr_default = NULL;
	const char *set_type = NULL;

	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	/* Top-level Options */
	{"help", 0, 0, '?', "\tThis text"},
	{"version", 0, 0, '$', "\tVersion information" },
	{"verbose", 0, 0, 'V', "\tIncrease debug output"},
	{"quiet", 0, 0, 'q', "\tPrint only the value on stdout\n"},

	{"name", 1, 0, 'n', "Name of the attribute/option to operate on"},
	{"pattern", 1, 0, 'P', "Pattern matching names of attributes (only with -v/-D and -l reboot)"},

	{"-spacer-", 0, 0, '-', "\nCommands:"},
	{"query", 0, 0, 'G', "\tQuery the current value of the attribute/option"},
	{"update", 1, 0, 'v', "Update the value of the attribute/option"},
	{"delete", 0, 0, 'D', "\tDelete the attribute/option"},

	{"-spacer-", 0, 0, '-', "\nAdditional Options:"},
	{"node", 1, 0, 'N', "Set an attribute for the named node (instead of a cluster option). See also: -l"},
	{"type", 1, 0, 't', "Which part of the configuration to update/delete/query the option in"},
	{"-spacer-", 0, 0, '-', "\t\t\tValid values: crm_config, rsc_defaults, op_defaults, tickets"},
	{"lifetime", 1, 0, 'l', "Lifetime of the node attribute"},
	{"-spacer-", 0, 0, '-', "\t\t\tValid values: reboot, forever"},
	{"utilization", 0, 0, 'z', "Set an utilization attribute for the node."},
	{"set-name", 1, 0, 's', "(Advanced) The attribute set in which to place the value"},
	{"id", 1, 0, 'i', "\t(Advanced) The ID used to identify the attribute"},
	{"default", 1, 0, 'd', "(Advanced) The default value to display if none is found in the configuration"},

	{"inhibit-policy-engine", 0, 0, '!', NULL, 1},

	/* legacy */
	{"quiet", 0, 0, 'Q', NULL, 1},
	{"node-uname", 1, 0, 'U', NULL, 1},
	{"get-value", 0, 0, 'G', NULL, 1},
	{"delete-attr", 0, 0, 'D', NULL, 1},
	{"attr-value", 1, 0, 'v', NULL, 1},
	{"attr-name", 1, 0, 'n', NULL, 1},
	{"attr-id", 1, 0, 'i', NULL, 1},

	{"-spacer-", 1, 0, '-', "\nExamples:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', "Add a new node attribute called 'location' with the value of 'office' for host 'myhost':", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_attribute --node myhost --name location --update office", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Query the value of the 'location' node attribute for host 'myhost':", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_attribute --node myhost --name location --query", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Change the value of the 'location' node attribute for host 'myhost':", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_attribute --node myhost --name location --update backoffice", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Delete the 'location' node attribute for host 'myhost':", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_attribute --node myhost --name location --delete", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Query the value of the cluster-delay cluster option:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_attribute --type crm_config --name cluster-delay --query", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Query the value of the cluster-delay cluster option. Only print the value:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_attribute --type crm_config --name cluster-delay --query --quiet", pcmk_option_example},

	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	int
	main(int argc, char **argv)
	{
	cib_t *the_cib = NULL;
	int rc = pcmk_ok;

	int cib_opts = cib_sync_call;
	int argerr = 0;
	int flag;

	int option_index = 0;
	int is_remote_node = 0;

	crm_log_cli_init("crm_attribute");
	crm_set_options(NULL, "<command> -n <attribute> [options]", long_options,
	"Manage node's attributes and cluster options."
	"\n\nAllows node attributes and cluster options to be queried, modified and deleted.\n");

	if (argc < 2) {
	crm_help('?', CRM_EX_USAGE);
	}

	while (1) {
	flag = crm_get_option(argc, argv, &option_index);
	if (flag == -1)
	break;

	switch (flag) {
	case 'V':
	crm_bump_log_level(argc, argv);
	break;
	case '$':
	case '?':
	crm_help(flag, CRM_EX_OK);
	break;
	case 'G':
	command = flag;
	attr_value = optarg;
	break;
	case 'D':
	case 'v':
	command = flag;
	attr_value = optarg;
	crm_log_args(argc, argv);
	break;
	case 'q':
	case 'Q':
	BE_QUIET = TRUE;
	break;
	case 'U':
	case 'N':
	dest_uname = strdup(optarg);
	break;
	case 's':
	set_name = strdup(optarg);
	break;
	case 'l':
	case 't':
	type = optarg;
	break;
	case 'z':
	type = XML_CIB_TAG_NODES;
	set_type = XML_TAG_UTILIZATION;
	break;
	case 'n':
	attr_name = strdup(optarg);
	break;
	case 'P':
	attr_pattern = strdup(optarg);
	break;
	case 'i':
	attr_id = strdup(optarg);
	break;
	case 'r':
	rsc_id = optarg;
	break;
	case 'd':
	attr_default = optarg;
	break;
	case '!':
	crm_warn("Inhibiting notifications for this update");
	cib_opts \|= cib_inhibit_notify;
	break;
	default:
	printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
	++argerr;
	break;
	}
	}

	if (optind < argc) {
	printf("non-option ARGV-elements: ");
	while (optind < argc)
	printf("%s ", argv[optind++]);
	printf("\n");
	}

	if (optind > argc) {
	++argerr;
	}

	if (argerr) {
	crm_help('?', CRM_EX_USAGE);
	}

	the_cib = cib_new();
	rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command);

	if (rc != pcmk_ok) {
	fprintf(stderr, "Error connecting to the CIB manager: %s\n",
	pcmk_strerror(rc));
	return crm_exit(crm_errno2exit(rc));
	}

	if (type == NULL && dest_uname != NULL) {
	type = "forever";
	}

	if (safe_str_eq(type, "reboot")) {
	type = XML_CIB_TAG_STATUS;

	} else if (safe_str_eq(type, "forever")) {
	type = XML_CIB_TAG_NODES;
	}

	if (type == NULL && dest_uname == NULL) {
	/* we're updating cluster options - don't populate dest_node */
	type = XML_CIB_TAG_CRMCONFIG;

	} else if (safe_str_eq(type, XML_CIB_TAG_CRMCONFIG)) {
	} else if (safe_str_neq(type, XML_CIB_TAG_TICKETS)) {
	/* If we are being called from a resource agent via the cluster,
	* the correct local node name will be passed as an environment
	* variable. Otherwise, we have to ask the cluster.
	*/
	dest_uname = attrd_get_target(dest_uname);
	if (dest_uname == NULL) {
	dest_uname = get_local_node_name();
	}

	rc = query_node_uuid(the_cib, dest_uname, &dest_node, &is_remote_node);
	if (pcmk_ok != rc) {
	fprintf(stderr, "Could not map name=%s to a UUID\n", dest_uname);
	the_cib->cmds->signoff(the_cib);
	cib_delete(the_cib);
	return crm_exit(crm_errno2exit(rc));
	}
	}

	if ((command == 'D') && (attr_name == NULL) && (attr_pattern == NULL)) {
	fprintf(stderr, "Error: must specify attribute name or pattern to delete\n");
	return crm_exit(CRM_EX_USAGE);
	}

	if (attr_pattern) {
	if (((command != 'v') && (command != 'D'))
	\|\| safe_str_neq(type, XML_CIB_TAG_STATUS)) {

	fprintf(stderr, "Error: pattern can only be used with till-reboot update or delete\n");
	return crm_exit(CRM_EX_USAGE);
	}
	command = 'u';
	free(attr_name);
	attr_name = attr_pattern;
	}

	if (((command == 'v') \|\| (command == 'D') \|\| (command == 'u'))
	&& safe_str_eq(type, XML_CIB_TAG_STATUS)
	&& pcmk_ok == attrd_update_delegate(NULL, command, dest_uname, attr_name,
	attr_value, type, set_name, NULL, NULL,
	is_remote_node?attrd_opt_remote:attrd_opt_none)) {
	crm_info("Update %s=%s sent via pacemaker-attrd",
	attr_name, ((command == 'D')? "<none>" : attr_value));

	} else if (command == 'D') {
	rc = delete_attr_delegate(the_cib, cib_opts, type, dest_node, set_type, set_name,
	attr_id, attr_name, attr_value, TRUE, NULL);

	if (rc == -ENXIO) {
	/* Nothing to delete...
	* which means it's not there...
	* which is what the admin wanted
	*/
	rc = pcmk_ok;
	}

	} else if (command == 'v') {
	CRM_LOG_ASSERT(type != NULL);
	CRM_LOG_ASSERT(attr_name != NULL);
	CRM_LOG_ASSERT(attr_value != NULL);

	rc = update_attr_delegate(the_cib, cib_opts, type, dest_node, set_type, set_name,
	attr_id, attr_name, attr_value, TRUE, NULL, is_remote_node ? "remote" : NULL);

	} else { /* query */

	char *read_value = NULL;

	rc = read_attr_delegate(the_cib, type, dest_node, set_type, set_name,
	attr_id, attr_name, &read_value, TRUE, NULL);

	if (rc == -ENXIO && attr_default) {
	read_value = strdup(attr_default);
	rc = pcmk_ok;
	}

	crm_info("Read %s=%s %s%s",
	attr_name, crm_str(read_value), set_name ? "in " : "", set_name ? set_name : "");

	- if (rc == -EINVAL) {
	+ if (rc == -ENOTUNIQ) {
	+ // Multiple matches (already displayed) are not error for queries
	rc = pcmk_ok;

	} else if (BE_QUIET == FALSE) {
	fprintf(stdout, "%s%s %s%s %s%s value=%s\n",
	type ? "scope=" : "", type ? type : "",
	attr_id ? "id=" : "", attr_id ? attr_id : "",
	attr_name ? "name=" : "", attr_name ? attr_name : "",
	read_value ? read_value : "(null)");

	} else if (read_value != NULL) {
	fprintf(stdout, "%s\n", read_value);
	}
	free(read_value);
	}

	- if (rc == -EINVAL) {
	+ if (rc == -ENOTUNIQ) {
	printf("Please choose from one of the matches above and supply the 'id' with --attr-id\n");

	} else if (rc != pcmk_ok) {
	fprintf(stderr, "Error performing operation: %s\n", pcmk_strerror(rc));
	}

	the_cib->cmds->signoff(the_cib);
	cib_delete(the_cib);
	return crm_exit(crm_errno2exit(rc));
	}
	diff --git a/tools/crm_failcount.in b/tools/crm_failcount.in
	index c1c61b4ade..85a6761b49 100755
	--- a/tools/crm_failcount.in
	+++ b/tools/crm_failcount.in
	@@ -1,273 +1,276 @@
	#!@BASH_PATH@

	USAGE_TEXT="Usage: crm_failcount <command> [<options>]
	Common options:
	--help Display this text, then exit
	--version Display version information, then exit
	-V, --verbose Specify multiple times to increase debug output
	-q, --quiet Print only the value (if querying)

	Commands:
	-G, --query Query the current value of the resource's fail count
	-D, --delete Delete resource's recorded failures

	Additional Options:
	-r, --resource=value Name of the resource to use (required)
	-n, --operation=value Name of operation to use (instead of all operations)
	-I, --interval=value If operation is specified, its interval
	-N, --node=value Use failcount on named node (instead of local node)"


	HELP_TEXT="crm_failcount - Query or delete resource fail counts

	$USAGE_TEXT"

	# These constants must track crm_exit_t values
	CRM_EX_OK=0
	CRM_EX_ERROR=1
	CRM_EX_USAGE=64
	CRM_EX_NOSUCH=105

	exit_usage() {
	if [ $# -gt 0 ]; then
	echo "error: $@" >&2
	fi
	echo
	echo "$USAGE_TEXT"
	exit $CRM_EX_USAGE
	}

	warn() {
	echo "warning: $@" >&2
	}

	interval_re() {
	echo "^[[:blank:]]([0-9]+)[[:blank:]](${1})[[:blank:]]*$"
	}

	# This function should follow crm_get_interval() as closely as possible
	parse_interval() {
	INT_S="$1"

	INT_8601RE="^P(([0-9]+)Y)?(([0-9]+)M)?(([0-9]+)D)?T?(([0-9]+)H)?(([0-9]+)M)?(([0-9]+)S)?$"

	- if [[ $INT_S =~ $(interval_re "s\|sec\|") ]]; then
	+ if [[ $INT_S =~ $(interval_re "") ]]; then
	+ echo $(( ${BASH_REMATCH[1]} * 1000 ))
	+
	+ elif [[ $INT_S =~ $(interval_re "s\|sec") ]]; then
	echo $(( ${BASH_REMATCH[1]} * 1000 ))

	elif [[ $INT_S =~ $(interval_re "ms\|msec") ]]; then
	echo "${BASH_REMATCH[1]}"

	elif [[ $INT_S =~ $(interval_re "m\|min") ]]; then
	echo $(( ${BASH_REMATCH[1]} * 60000 ))

	elif [[ $INT_S =~ $(interval_re "h\|hr") ]]; then
	echo $(( ${BASH_REMATCH[1]} * 3600000 ))

	elif [[ $INT_S =~ $(interval_re "us\|usec") ]]; then
	echo $(( ${BASH_REMATCH[1]} / 1000 ))

	elif [[ $INT_S =~ ^P([0-9]+)W$ ]]; then
	echo $(( ${BASH_REMATCH[1]} * 604800000 ))

	elif [[ $INT_S =~ $INT_8601RE ]]; then
	echo $(( ( ${BASH_REMATCH[2]:-0} * 31536000000 ) \
	+ ( ${BASH_REMATCH[4]:-0} * 2592000000 ) \
	+ ( ${BASH_REMATCH[6]:-0} * 86400000 ) \
	+ ( ${BASH_REMATCH[8]:-0} * 3600000 ) \
	+ ( ${BASH_REMATCH[10]:-0} * 60000 ) \
	+ ( ${BASH_REMATCH[12]:-0} * 1000 ) ))

	else
	warn "Unrecognized interval, using 0"
	echo "0"
	fi
	}

	query_single_attr() {
	QSR_TARGET="$1"
	QSR_ATTR="$2"

	crm_attribute $VERBOSE --quiet --query -t status -d 0 \
	-N "$QSR_TARGET" -n "$QSR_ATTR"
	}

	query_attr_sum() {
	QAS_TARGET="$1"
	QAS_PREFIX="$2"

	# Build xpath to match all transient node attributes with prefix
	QAS_XPATH="/cib/status/node_state[@uname='${QAS_TARGET}']"
	QAS_XPATH="${QAS_XPATH}/transient_attributes/instance_attributes"
	QAS_XPATH="${QAS_XPATH}/nvpair[starts-with(@name,'$QAS_PREFIX')]"

	# Query attributes that match xpath
	# @TODO We ignore stderr because we don't want "no results" to look
	# like an error, but that also makes $VERBOSE pointless.
	QAS_ALL=$(cibadmin --query --sync-call --local \
	--xpath="$QAS_XPATH" 2>/dev/null)
	QAS_EX=$?

	# "No results" is not an error
	if [ $QAS_EX -ne $CRM_EX_OK ] && [ $QAS_EX -ne $CRM_EX_NOSUCH ]; then
	echo "error: could not query CIB for fail counts" >&2
	exit $QAS_EX
	fi

	# Extract the attribute values (one per line) from the output
	QAS_VALUE=$(echo "$QAS_ALL" \| sed -n -e \
	's/.<nvpair.value="$[0-9][0-9]$".>.*/\1/p')

	# Sum the values
	QAS_SUM=0
	for i in 0 $QAS_VALUE; do
	QAS_SUM=$(($QAS_SUM + $i))
	done
	echo $QAS_SUM
	}

	query_failcount() {
	QF_TARGET="$1"
	QF_RESOURCE="$2"
	QF_OPERATION="$3"
	QF_INTERVAL="$4"

	QF_ATTR_RSC="fail-count-${QF_RESOURCE}"

	if [ -n "$QF_OPERATION" ]; then
	QF_ATTR_DISPLAY="${QF_ATTR_RSC}#${QF_OPERATION}_${QF_INTERVAL}"
	QF_COUNT=$(query_single_attr "$QF_TARGET" "$QF_ATTR_DISPLAY")
	else
	QF_ATTR_DISPLAY="$QF_ATTR_RSC"
	QF_COUNT=$(query_attr_sum "$QF_TARGET" "${QF_ATTR_RSC}#")
	fi

	# @COMPAT attributes set < 1.1.17:
	# If we didn't find any per-operation failcount,
	# check whether there is a legacy per-resource failcount.
	if [ "$QF_COUNT" = "0" ]; then
	QF_COUNT=$(query_single_attr "$QF_TARGET" "$QF_ATTR_RSC")
	if [ "$QF_COUNT" != "0" ]; then
	QF_ATTR_DISPLAY="$QF_ATTR_RSC"
	fi
	fi

	# Echo result (comparable to crm_attribute, for backward compatibility)
	if [ -n "$QUIET" ]; then
	echo $QF_COUNT
	else
	echo "scope=status name=$QF_ATTR_DISPLAY value=$QF_COUNT"
	fi
	}

	clear_failcount() {
	CF_TARGET="$1"
	CF_RESOURCE="$2"
	CF_OPERATION="$3"
	CF_INTERVAL="$4"

	if [ -n "$CF_OPERATION" ]; then
	CF_OPERATION="-n $CF_OPERATION -I ${CF_INTERVAL}ms"
	fi
	crm_resource $QUIET $VERBOSE --cleanup \
	-N "$CF_TARGET" -r "$CF_RESOURCE" $CF_OPERATION
	}

	QUIET=""
	VERBOSE=""

	command=""
	resource=""
	operation=""
	interval="0"
	target=$(crm_node -n 2>/dev/null)

	SHORTOPTS="qDGQVN:U:v:i:l:r:n:I:"

	LONGOPTS_COMMON="help,version,verbose,quiet"
	LONGOPTS_COMMANDS="query,delete"
	LONGOPTS_OTHER="resource:,node:,operation:,interval:"
	LONGOPTS_COMPAT="delete-attr,get-value,resource-id:,uname:,lifetime:,attr-value:,attr-id:"

	LONGOPTS="$LONGOPTS_COMMON,$LONGOPTS_COMMANDS,$LONGOPTS_OTHER,$LONGOPTS_COMPAT"

	TEMP=$(@GETOPT_PATH@ -o $SHORTOPTS --long $LONGOPTS -n crm_failcount -- "$@")
	if [ $? -ne 0 ]; then
	exit_usage
	fi
	eval set -- "$TEMP" # Quotes around $TEMP are essential

	while true ; do
	case "$1" in
	--help)
	echo "$HELP_TEXT"
	exit $CRM_EX_OK
	;;
	--version)
	crm_attribute --version
	exit $?
	;;
	-q\|-Q\|--quiet)
	QUIET="--quiet"
	shift
	;;
	-V\|--verbose)
	VERBOSE="$VERBOSE $1"
	shift
	;;
	-G\|--query\|--get-value)
	command="--query"
	shift
	;;
	-D\|--delete\|--delete-attr)
	command="--delete"
	shift
	;;
	-r\|--resource\|--resource-id)
	resource="$2"
	shift 2
	;;
	-n\|--operation)
	operation="$2"
	shift 2
	;;
	-I\|--interval)
	interval="$2"
	shift 2
	;;
	-N\|--node\|-U\|--uname)
	target="$2"
	shift 2
	;;
	-v\|--attr-value)
	if [ "$2" = "0" ]; then
	command="--delete"
	else
	warn "ignoring deprecated option '$1' with nonzero value"
	fi
	shift 2
	;;
	-i\|--attr-id\|-l\|--lifetime)
	warn "ignoring deprecated option '$1'"
	shift 2
	;;
	--)
	shift
	break
	;;
	*)
	exit_usage "unknown option '$1'"
	;;
	esac
	done

	[ -n "$command" ] \|\| exit_usage "must specify a command"
	[ -n "$resource" ] \|\| exit_usage "resource name required"
	[ -n "$target" ] \|\| exit_usage "node name required"

	interval=$(parse_interval $interval)

	if [ "$command" = "--query" ]; then
	query_failcount "$target" "$resource" "$operation" "$interval"
	else
	clear_failcount "$target" "$resource" "$operation" "$interval"
	fi
	diff --git a/tools/crm_resource.c b/tools/crm_resource.c
	index 9818692b38..25e05980d3 100644
	--- a/tools/crm_resource.c
	+++ b/tools/crm_resource.c
	@@ -1,1203 +1,1200 @@
	/*
	* Copyright 2004-2018 Andrew Beekhof <andrew@beekhof.net>
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_resource.h>

	#include <sys/param.h>

	#include <crm/crm.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>
	#include <time.h>

	bool BE_QUIET = FALSE;
	bool scope_master = FALSE;
	int cib_options = cib_sync_call;

	GMainLoop *mainloop = NULL;

	#define message_timeout_ms 60*1000

	static gboolean
	resource_ipc_timeout(gpointer data)
	{
	fprintf(stderr, "No messages received in %d seconds.. aborting\n",
	(int)message_timeout_ms / 1000);
	crm_err("No messages received in %d seconds", (int)message_timeout_ms / 1000);
	return crm_exit(CRM_EX_TIMEOUT);
	}

	static void
	resource_ipc_connection_destroy(gpointer user_data)
	{
	crm_info("Connection to controller was terminated");
	crm_exit(CRM_EX_DISCONNECT);
	}

	static void
	start_mainloop(void)
	{
	if (crmd_replies_needed == 0) {
	return;
	}

	mainloop = g_main_loop_new(NULL, FALSE);
	fprintf(stderr, "Waiting for %d replies from the controller",
	crmd_replies_needed);
	crm_debug("Waiting for %d replies from the controller",
	crmd_replies_needed);

	g_timeout_add(message_timeout_ms, resource_ipc_timeout, NULL);
	g_main_loop_run(mainloop);
	}

	static int
	resource_ipc_callback(const char *buffer, ssize_t length, gpointer userdata)
	{
	xmlNode *msg = string2xml(buffer);

	fprintf(stderr, ".");
	crm_log_xml_trace(msg, "[inbound]");

	crmd_replies_needed--;
	if ((crmd_replies_needed == 0) && mainloop
	&& g_main_loop_is_running(mainloop)) {

	fprintf(stderr, " OK\n");
	crm_debug("Got all the replies we expected");
	return crm_exit(CRM_EX_OK);
	}

	free_xml(msg);
	return 0;
	}

	struct ipc_client_callbacks crm_callbacks = {
	.dispatch = resource_ipc_callback,
	.destroy = resource_ipc_connection_destroy,
	};


	/* short option letters still available: eEJkKXyYZ */

	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	/* Top-level Options */
	{
	"help", no_argument, NULL, '?',
	"\t\tDisplay this text and exit"
	},
	{
	"version", no_argument, NULL, '$',
	"\t\tDisplay version information and exit"
	},
	{
	"verbose", no_argument, NULL, 'V',
	"\t\tIncrease debug output (may be specified multiple times)"
	},
	{
	"quiet", no_argument, NULL, 'Q',
	"\t\tBe less descriptive in results"
	},
	{
	"resource", required_argument, NULL, 'r',
	"\tResource ID"
	},

	{ "-spacer-", no_argument, NULL, '-', "\nQueries:" },
	{
	"list", no_argument, NULL, 'L',
	"\t\tList all cluster resources with status"},
	{
	"list-raw", no_argument, NULL, 'l',
	"\t\tList IDs of all instantiated resources (individual members rather than groups etc.)"
	},
	{
	"list-cts", no_argument, NULL, 'c',
	NULL, pcmk_option_hidden
	},
	{
	"list-operations", no_argument, NULL, 'O',
	"\tList active resource operations, optionally filtered by --resource and/or --node"
	},
	{
	"list-all-operations", no_argument, NULL, 'o',
	"List all resource operations, optionally filtered by --resource and/or --node"
	},
	{
	"list-standards", no_argument, NULL, 0,
	"\tList supported standards"
	},
	{
	"list-ocf-providers", no_argument, NULL, 0,
	"List all available OCF providers"
	},
	{
	"list-agents", required_argument, NULL, 0,
	"List all agents available for the named standard and/or provider."
	},
	{
	"list-ocf-alternatives", required_argument, NULL, 0,
	"List all available providers for the named OCF agent"
	},
	{
	"show-metadata", required_argument, NULL, 0,
	"Show the metadata for the named class:provider:agent"
	},
	{
	"query-xml", no_argument, NULL, 'q',
	"\tShow XML configuration of resource (after any template expansion)"
	},
	{
	"query-xml-raw", no_argument, NULL, 'w',
	"\tShow XML configuration of resource (before any template expansion)"
	},
	{
	"get-parameter", required_argument, NULL, 'g',
	"Display named parameter for resource.\n"
	"\t\t\t\tUse instance attribute unless --meta or --utilization is specified"
	},
	{
	"get-property", required_argument, NULL, 'G',
	"Display named property of resource ('class', 'type', or 'provider') (requires --resource)",
	pcmk_option_hidden
	},
	{
	"locate", no_argument, NULL, 'W',
	"\t\tShow node(s) currently running resource"
	},
	{
	"stack", no_argument, NULL, 'A',
	"\t\tDisplay the prerequisites and dependents of a resource"
	},
	{
	"constraints", no_argument, NULL, 'a',
	"\tDisplay the (co)location constraints that apply to a resource"
	},
	{
	"why", no_argument, NULL, 'Y',
	"\t\tShow why resources are not running, optionally filtered by --resource and/or --node"
	},

	{ "-spacer-", no_argument, NULL, '-', "\nCommands:" },
	{
	"validate", no_argument, NULL, 0,
	"\t\tCall the validate-all action of the local given resource"
	},
	{
	"cleanup", no_argument, NULL, 'C',
	"\t\tIf resource has any past failures, clear its history and fail count.\n"
	"\t\t\t\tOptionally filtered by --resource, --node, --operation, and --interval (otherwise all).\n"
	"\t\t\t\t--operation and --interval apply to fail counts, but entire history is always cleared,\n"
	"\t\t\t\tto allow current state to be rechecked.\n"
	},
	{
	"refresh", no_argument, NULL, 'R',
	"\t\tDelete resource's history (including failures) so its current state is rechecked.\n"
	"\t\t\t\tOptionally filtered by --resource and --node (otherwise all).\n"
	"\t\t\t\tUnless --force is specified, resource's group or clone (if any) will also be refreshed."
	},
	{
	"set-parameter", required_argument, NULL, 'p',
	"Set named parameter for resource (requires -v).\n"
	"\t\t\t\tUse instance attribute unless --meta or --utilization is specified."
	},
	{
	"delete-parameter", required_argument, NULL, 'd',
	"Delete named parameter for resource.\n"
	"\t\t\t\tUse instance attribute unless --meta or --utilization is specified."
	},
	{
	"set-property", required_argument, NULL, 'S',
	"Set named property of resource ('class', 'type', or 'provider') (requires -r, -t, -v)",
	pcmk_option_hidden
	},

	{ "-spacer-", no_argument, NULL, '-', "\nResource location:" },
	{
	"move", no_argument, NULL, 'M',
	"\t\tCreate a constraint to move resource. If --node is specified, the constraint\n"
	"\t\t\t\twill be to move to that node, otherwise it will be to ban the current node.\n"
	"\t\t\t\tUnless --force is specified, this will return an error if the resource is\n"
	"\t\t\t\talready running on the specified node. If --force is specified, this will\n"
	"\t\t\t\talways ban the current node. Optional: --lifetime, --master.\n"
	"\t\t\t\tNOTE: This may prevent the resource from running on its previous location\n"
	"\t\t\t\tuntil the implicit constraint expires or is removed with --clear."
	},
	{
	"ban", no_argument, NULL, 'B',
	"\t\tCreate a constraint to keep resource off a node. Optional: --node, --lifetime, --master.\n"
	"\t\t\t\tNOTE: This will prevent the resource from running on the affected node\n"
	"\t\t\t\tuntil the implicit constraint expires or is removed with --clear.\n"
	"\t\t\t\tIf --node is not specified, it defaults to the node currently running the resource\n"
	"\t\t\t\tfor primitives and groups, or the master for promotable clones with promoted-max=1\n"
	"\t\t\t\t(all other situations result in an error as there is no sane default).\n"
	},
	{
	"clear", no_argument, NULL, 'U',
	"\t\tRemove all constraints created by the --ban and/or --move commands.\n"
	"\t\t\t\tRequires: --resource. Optional: --node, --master.\n"
	"\t\t\t\tIf --node is not specified, all constraints created by --ban and --move\n"
	"\t\t\t\twill be removed for the named resource. If --node and --force are specified,\n"
	"\t\t\t\tany constraint created by --move will be cleared, even if it is not for the specified node."
	},
	{
	"lifetime", required_argument, NULL, 'u',
	"\tLifespan (as ISO 8601 duration) of created constraints (with -B, -M)\n"
	"\t\t\t\t(see https://en.wikipedia.org/wiki/ISO_8601#Durations)"
	},
	{
	"master", no_argument, NULL, 0,
	"\t\tLimit scope of command to the Master role (with -B, -M, -U).\n"
	"\t\t\t\tFor -B and -M, the previous master may remain active in the Slave role."
	},

	{ "-spacer-", no_argument, NULL, '-', "\nAdvanced Commands:" },
	{
	"delete", no_argument, NULL, 'D',
	"\t\t(Advanced) Delete a resource from the CIB. Required: -t"
	},
	{
	"fail", no_argument, NULL, 'F',
	"\t\t(Advanced) Tell the cluster this resource has failed"
	},
	{
	"restart", no_argument, NULL, 0,
	"\t\t(Advanced) Tell the cluster to restart this resource and anything that depends on it"
	},
	{
	"wait", no_argument, NULL, 0,
	"\t\t(Advanced) Wait until the cluster settles into a stable state"
	},
	{
	"force-demote", no_argument, NULL, 0,
	"\t(Advanced) Bypass the cluster and demote a resource on the local node.\n"
	"\t\t\t\tUnless --force is specified, this will refuse to do so if the cluster\n"
	"\t\t\t\tbelieves the resource is a clone instance already running on the local node."
	},
	{
	"force-stop", no_argument, NULL, 0,
	"\t(Advanced) Bypass the cluster and stop a resource on the local node."
	},
	{
	"force-start", no_argument, NULL, 0,
	"\t(Advanced) Bypass the cluster and start a resource on the local node.\n"
	"\t\t\t\tUnless --force is specified, this will refuse to do so if the cluster\n"
	"\t\t\t\tbelieves the resource is a clone instance already running on the local node."
	},
	{
	"force-promote", no_argument, NULL, 0,
	"\t(Advanced) Bypass the cluster and promote a resource on the local node.\n"
	"\t\t\t\tUnless --force is specified, this will refuse to do so if the cluster\n"
	"\t\t\t\tbelieves the resource is a clone instance already running on the local node."
	},
	{
	"force-check", no_argument, NULL, 0,
	"\t(Advanced) Bypass the cluster and check the state of a resource on the local node."
	},

	{ "-spacer-", no_argument, NULL, '-', "\nAdditional Options:" },
	{
	"node", required_argument, NULL, 'N',
	"\tNode name"
	},
	{
	"recursive", no_argument, NULL, 0,
	"\tFollow colocation chains when using --set-parameter"
	},
	{
	"resource-type", required_argument, NULL, 't',
	"Resource XML element (primitive, group, etc.) (with -D)"
	},
	{
	"parameter-value", required_argument, NULL, 'v',
	"Value to use with -p"
	},
	{
	"meta", no_argument, NULL, 'm',
	"\t\tUse resource meta-attribute instead of instance attribute (with -p, -g, -d)"
	},
	{
	"utilization", no_argument, NULL, 'z',
	"\tUse resource utilization attribute instead of instance attribute (with -p, -g, -d)"
	},
	{
	"operation", required_argument, NULL, 'n',
	"\tOperation to clear instead of all (with -C -r)"
	},
	{
	"interval", required_argument, NULL, 'I',
	"\tInterval of operation to clear (default 0) (with -C -r -n)"
	},
	{
	"set-name", required_argument, NULL, 's',
	"\t(Advanced) XML ID of attributes element to use (with -p, -d)"
	},
	{
	"nvpair", required_argument, NULL, 'i',
	"\t(Advanced) XML ID of nvpair element to use (with -p, -d)"
	},
	{
	"timeout", required_argument, NULL, 'T',
	"\t(Advanced) Abort if command does not finish in this time (with --restart, --wait, --force-*)"
	},
	{
	"force", no_argument, NULL, 'f',
	"\t\tIf making CIB changes, do so regardless of quorum.\n"
	"\t\t\t\tSee help for individual commands for additional behavior.\n"
	},
	{
	"xml-file", required_argument, NULL, 'x',
	NULL, pcmk_option_hidden
	},

	/* legacy options */
	{"host-uname", required_argument, NULL, 'H', NULL, pcmk_option_hidden},

	{"-spacer-", 1, NULL, '-', "\nExamples:", pcmk_option_paragraph},
	{"-spacer-", 1, NULL, '-', "List the available OCF agents:", pcmk_option_paragraph},
	{"-spacer-", 1, NULL, '-', " crm_resource --list-agents ocf", pcmk_option_example},
	{"-spacer-", 1, NULL, '-', "List the available OCF agents from the linux-ha project:", pcmk_option_paragraph},
	{"-spacer-", 1, NULL, '-', " crm_resource --list-agents ocf:heartbeat", pcmk_option_example},
	{"-spacer-", 1, NULL, '-', "Move 'myResource' to a specific node:", pcmk_option_paragraph},
	{"-spacer-", 1, NULL, '-', " crm_resource --resource myResource --move --node altNode", pcmk_option_example},
	{"-spacer-", 1, NULL, '-', "Allow (but not force) 'myResource' to move back to its original location:", pcmk_option_paragraph},
	{"-spacer-", 1, NULL, '-', " crm_resource --resource myResource --clear", pcmk_option_example},
	{"-spacer-", 1, NULL, '-', "Stop 'myResource' (and anything that depends on it):", pcmk_option_paragraph},
	{"-spacer-", 1, NULL, '-', " crm_resource --resource myResource --set-parameter target-role --meta --parameter-value Stopped", pcmk_option_example},
	{"-spacer-", 1, NULL, '-', "Tell the cluster not to manage 'myResource':", pcmk_option_paragraph},
	{"-spacer-", 1, NULL, '-', "The cluster will not attempt to start or stop the resource under any circumstances."},
	{"-spacer-", 1, NULL, '-', "Useful when performing maintenance tasks on a resource.", pcmk_option_paragraph},
	{"-spacer-", 1, NULL, '-', " crm_resource --resource myResource --set-parameter is-managed --meta --parameter-value false", pcmk_option_example},
	{"-spacer-", 1, NULL, '-', "Erase the operation history of 'myResource' on 'aNode':", pcmk_option_paragraph},
	{"-spacer-", 1, NULL, '-', "The cluster will 'forget' the existing resource state (including any errors) and attempt to recover the resource."},
	{"-spacer-", 1, NULL, '-', "Useful when a resource had failed permanently and has been repaired by an administrator.", pcmk_option_paragraph},
	{"-spacer-", 1, NULL, '-', " crm_resource --resource myResource --cleanup --node aNode", pcmk_option_example},

	{0, 0, 0, 0}
	};
	/* INDENT-ON */


	int
	main(int argc, char **argv)
	{
	char rsc_cmd = 'L';

	const char *rsc_id = NULL;
	const char *host_uname = NULL;
	const char *prop_name = NULL;
	const char *prop_value = NULL;
	const char *rsc_type = NULL;
	const char *prop_id = NULL;
	const char *prop_set = NULL;
	const char *rsc_long_cmd = NULL;
	const char *longname = NULL;
	const char *operation = NULL;
	const char *interval_spec = NULL;
	const char *cib_file = getenv("CIB_file");
	GHashTable *override_params = NULL;

	char *xml_file = NULL;
	crm_ipc_t *crmd_channel = NULL;
	pe_working_set_t data_set = { 0, };
	cib_t *cib_conn = NULL;
	resource_t *rsc = NULL;
	bool recursive = FALSE;
	char *our_pid = NULL;

	bool require_resource = TRUE; /* whether command requires that resource be specified */
	bool require_dataset = TRUE; /* whether command requires populated dataset instance */
	bool require_crmd = FALSE; // whether command requires controller connection

	int rc = pcmk_ok;
	int is_ocf_rc = 0;
	int option_index = 0;
	int timeout_ms = 0;
	int argerr = 0;
	int flag;
	int find_flags = 0; // Flags to use when searching for resource
	crm_exit_t exit_code = CRM_EX_OK;

	crm_log_cli_init("crm_resource");
	crm_set_options(NULL, "(query\|command) [options]", long_options,
	"Perform tasks related to cluster resources.\nAllows resources to be queried (definition and location), modified, and moved around the cluster.\n");

	while (1) {
	flag = crm_get_option_long(argc, argv, &option_index, &longname);
	if (flag == -1)
	break;

	switch (flag) {
	case 0: /* long options with no short equivalent */
	if (safe_str_eq("master", longname)) {
	scope_master = TRUE;

	} else if(safe_str_eq(longname, "recursive")) {
	recursive = TRUE;

	} else if (safe_str_eq("wait", longname)) {
	rsc_cmd = flag;
	rsc_long_cmd = longname;
	require_resource = FALSE;
	require_dataset = FALSE;

	} else if (
	safe_str_eq("validate", longname)
	\|\| safe_str_eq("restart", longname)
	\|\| safe_str_eq("force-demote", longname)
	\|\| safe_str_eq("force-stop", longname)
	\|\| safe_str_eq("force-start", longname)
	\|\| safe_str_eq("force-promote", longname)
	\|\| safe_str_eq("force-check", longname)) {
	rsc_cmd = flag;
	rsc_long_cmd = longname;
	find_flags = pe_find_renamed\|pe_find_anon;
	crm_log_args(argc, argv);

	} else if (safe_str_eq("list-ocf-providers", longname)
	\|\| safe_str_eq("list-ocf-alternatives", longname)
	\|\| safe_str_eq("list-standards", longname)) {
	const char *text = NULL;
	lrmd_list_t *list = NULL;
	lrmd_list_t *iter = NULL;
	lrmd_t *lrmd_conn = lrmd_api_new();

	if (safe_str_eq("list-ocf-providers", longname)
	\|\| safe_str_eq("list-ocf-alternatives", longname)) {
	rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, optarg, &list);
	text = "OCF providers";

	} else if (safe_str_eq("list-standards", longname)) {
	rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);
	text = "standards";
	}

	if (rc > 0) {
	for (iter = list; iter != NULL; iter = iter->next) {
	printf("%s\n", iter->val);
	}
	lrmd_list_freeall(list);

	} else if (optarg) {
	fprintf(stderr, "No %s found for %s\n", text, optarg);
	exit_code = CRM_EX_NOSUCH;

	} else {
	fprintf(stderr, "No %s found\n", text);
	exit_code = CRM_EX_NOSUCH;
	}

	lrmd_api_delete(lrmd_conn);
	crm_exit(exit_code);

	} else if (safe_str_eq("show-metadata", longname)) {
	char *standard = NULL;
	char *provider = NULL;
	char *type = NULL;
	char *metadata = NULL;
	lrmd_t *lrmd_conn = lrmd_api_new();

	rc = crm_parse_agent_spec(optarg, &standard, &provider, &type);
	if (rc == pcmk_ok) {
	rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard,
	provider, type,
	&metadata, 0);
	} else {
	fprintf(stderr,
	"'%s' is not a valid agent specification\n",
	optarg);
	rc = -ENXIO;
	}

	if (metadata) {
	printf("%s\n", metadata);
	} else {
	fprintf(stderr, "Metadata query for %s failed: %s\n",
	optarg, pcmk_strerror(rc));
	exit_code = crm_errno2exit(rc);
	}
	lrmd_api_delete(lrmd_conn);
	crm_exit(exit_code);

	} else if (safe_str_eq("list-agents", longname)) {
	lrmd_list_t *list = NULL;
	lrmd_list_t *iter = NULL;
	char *provider = strchr (optarg, ':');
	lrmd_t *lrmd_conn = lrmd_api_new();

	if (provider) {
	*provider++ = 0;
	}
	rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, optarg, provider);

	if (rc > 0) {
	for (iter = list; iter != NULL; iter = iter->next) {
	printf("%s\n", iter->val);
	}
	lrmd_list_freeall(list);
	} else {
	fprintf(stderr, "No agents found for standard=%s, provider=%s\n",
	optarg, (provider? provider : "*"));
	exit_code = CRM_EX_NOSUCH;
	}
	lrmd_api_delete(lrmd_conn);
	crm_exit(exit_code);

	} else {
	crm_err("Unhandled long option: %s", longname);
	}
	break;
	case 'V':
	resource_verbose++;
	crm_bump_log_level(argc, argv);
	break;
	case '$':
	case '?':
	crm_help(flag, CRM_EX_OK);
	break;
	case 'x':
	xml_file = strdup(optarg);
	break;
	case 'Q':
	BE_QUIET = TRUE;
	break;
	case 'm':
	attr_set_type = XML_TAG_META_SETS;
	break;
	case 'z':
	attr_set_type = XML_TAG_UTILIZATION;
	break;
	case 'u':
	move_lifetime = strdup(optarg);
	break;
	case 'f':
	do_force = TRUE;
	crm_log_args(argc, argv);
	break;
	case 'i':
	prop_id = optarg;
	break;
	case 's':
	prop_set = optarg;
	break;
	case 'r':
	rsc_id = optarg;
	break;
	case 'v':
	prop_value = optarg;
	break;
	case 't':
	rsc_type = optarg;
	break;
	case 'T':
	timeout_ms = crm_get_msec(optarg);
	break;

	case 'C':
	case 'R':
	crm_log_args(argc, argv);
	require_resource = FALSE;
	if (cib_file == NULL) {
	require_crmd = TRUE;
	}
	rsc_cmd = flag;
	find_flags = pe_find_renamed\|pe_find_anon;
	break;

	case 'n':
	operation = optarg;
	break;

	case 'I':
	interval_spec = optarg;
	break;

	case 'D':
	require_dataset = FALSE;
	crm_log_args(argc, argv);
	rsc_cmd = flag;
	find_flags = pe_find_renamed\|pe_find_any;
	break;

	case 'F':
	require_crmd = TRUE;
	crm_log_args(argc, argv);
	rsc_cmd = flag;
	break;

	case 'U':
	case 'B':
	case 'M':
	crm_log_args(argc, argv);
	rsc_cmd = flag;
	find_flags = pe_find_renamed\|pe_find_anon;
	break;

	case 'c':
	case 'L':
	case 'l':
	case 'O':
	case 'o':
	require_resource = FALSE;
	rsc_cmd = flag;
	break;

	case 'Y':
	require_resource = FALSE;
	rsc_cmd = flag;
	find_flags = pe_find_renamed\|pe_find_anon;
	break;

	case 'q':
	case 'w':
	rsc_cmd = flag;
	find_flags = pe_find_renamed\|pe_find_any;
	break;

	case 'W':
	case 'A':
	case 'a':
	rsc_cmd = flag;
	find_flags = pe_find_renamed\|pe_find_anon;
	break;

	case 'S':
	require_dataset = FALSE;
	crm_log_args(argc, argv);
	prop_name = optarg;
	rsc_cmd = flag;
	find_flags = pe_find_renamed\|pe_find_any;
	break;

	case 'p':
	case 'd':
	crm_log_args(argc, argv);
	prop_name = optarg;
	rsc_cmd = flag;
	find_flags = pe_find_renamed\|pe_find_any;
	break;

	case 'G':
	case 'g':
	prop_name = optarg;
	rsc_cmd = flag;
	find_flags = pe_find_renamed\|pe_find_any;
	break;

	case 'H':
	case 'N':
	crm_trace("Option %c => %s", flag, optarg);
	host_uname = optarg;
	break;

	default:
	CMD_ERR("Argument code 0%o (%c) is not (?yet?) supported", flag, flag);
	++argerr;
	break;
	}
	}

	// Catch the case where the user didn't specify a command
	if (rsc_cmd == 'L') {
	require_resource = FALSE;
	}

	if (optind < argc
	&& argv[optind] != NULL
	&& rsc_cmd == 0
	&& rsc_long_cmd) {

	override_params = crm_str_table_new();
	while (optind < argc && argv[optind] != NULL) {
	char *name = calloc(1, strlen(argv[optind]));
	char *value = calloc(1, strlen(argv[optind]));
	int rc = sscanf(argv[optind], "%[^=]=%s", name, value);

	if(rc == 2) {
	g_hash_table_replace(override_params, name, value);

	} else {
	CMD_ERR("Error parsing '%s' as a name=value pair for --%s", argv[optind], rsc_long_cmd);
	free(value);
	free(name);
	argerr++;
	}
	optind++;
	}

	} else if (optind < argc && argv[optind] != NULL && rsc_cmd == 0) {
	CMD_ERR("non-option ARGV-elements: ");
	while (optind < argc && argv[optind] != NULL) {
	CMD_ERR("[%d of %d] %s ", optind, argc, argv[optind]);
	optind++;
	argerr++;
	}
	}

	if (optind > argc) {
	++argerr;
	}

	if (argerr) {
	CMD_ERR("Invalid option(s) supplied, use --help for valid usage");
	crm_exit(CRM_EX_USAGE);
	}

	our_pid = crm_getpid_s();

	if (do_force) {
	crm_debug("Forcing...");
	cib_options \|= cib_quorum_override;
	}

	data_set.input = NULL; /* make clean-up easier */

	if (require_resource && !rsc_id) {
	CMD_ERR("Must supply a resource id with -r");
	rc = -ENXIO;
	goto bail;
	}

	if (find_flags && rsc_id) {
	require_dataset = TRUE;
	}

	/* Establish a connection to the CIB manager */
	cib_conn = cib_new();
	rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
	if (rc != pcmk_ok) {
	CMD_ERR("Error connecting to the CIB manager: %s", pcmk_strerror(rc));
	goto bail;
	}

	/* Populate working set from XML file if specified or CIB query otherwise */
	if (require_dataset) {
	xmlNode *cib_xml_copy = NULL;

	if (xml_file != NULL) {
	cib_xml_copy = filename2xml(xml_file);

	} else {
	rc = cib_conn->cmds->query(cib_conn, NULL, &cib_xml_copy, cib_scope_local \| cib_sync_call);
	}

	if(rc != pcmk_ok) {
	goto bail;
	}

	/* Populate the working set instance */
	set_working_set_defaults(&data_set);
	rc = update_working_set_xml(&data_set, &cib_xml_copy);
	if (rc != pcmk_ok) {
	goto bail;
	}
	cluster_status(&data_set);
	}

	// If command requires that resource exist if specified, find it
	if (find_flags && rsc_id) {
	rsc = pe_find_resource_with_flags(data_set.resources, rsc_id,
	find_flags);
	if (rsc == NULL) {
	CMD_ERR("Resource '%s' not found", rsc_id);
	rc = -ENXIO;
	goto bail;
	}
	}

	// Establish a connection to the controller if needed
	if (require_crmd) {
	xmlNode *xml = NULL;
	mainloop_io_t *source =
	mainloop_add_ipc_client(CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks);
	crmd_channel = mainloop_get_ipc_client(source);

	if (crmd_channel == NULL) {
	CMD_ERR("Error connecting to the controller");
	rc = -ENOTCONN;
	goto bail;
	}

	xml = create_hello_message(our_pid, crm_system_name, "0", "1");
	crm_ipc_send(crmd_channel, xml, 0, 0, NULL);
	free_xml(xml);
	}

	/* Handle rsc_cmd appropriately */
	if (rsc_cmd == 'L') {
	rc = pcmk_ok;
	cli_resource_print_list(&data_set, FALSE);

	} else if (rsc_cmd == 'l') {
	int found = 0;
	GListPtr lpc = NULL;

	rc = pcmk_ok;
	for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
	rsc = (resource_t *) lpc->data;

	found++;
	cli_resource_print_raw(rsc);
	}

	if (found == 0) {
	printf("NO resources configured\n");
	rc = -ENXIO;
	}

	} else if (rsc_cmd == 0 && rsc_long_cmd && safe_str_eq(rsc_long_cmd, "restart")) {
	rc = cli_resource_restart(rsc, host_uname, timeout_ms, cib_conn);

	} else if (rsc_cmd == 0 && rsc_long_cmd && safe_str_eq(rsc_long_cmd, "wait")) {
	rc = wait_till_stable(timeout_ms, cib_conn);

	} else if (rsc_cmd == 0 && rsc_long_cmd) {
	// validate, force-(stop\|start\|demote\|promote\|check)
	rc = cli_resource_execute(rsc, rsc_id, rsc_long_cmd, override_params,
	timeout_ms, cib_conn, &data_set);
	if (rc >= 0) {
	is_ocf_rc = 1;
	}

	} else if (rsc_cmd == 'A' \|\| rsc_cmd == 'a') {
	GListPtr lpc = NULL;
	xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set.input);

	unpack_constraints(cib_constraints, &data_set);

	// Constraints apply to group/clone, not member/instance
	rsc = uber_parent(rsc);

	for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
	resource_t r = (resource_t ) lpc->data;

	clear_bit(r->flags, pe_rsc_allocating);
	}

	cli_resource_print_colocation(rsc, TRUE, rsc_cmd == 'A', 1);

	fprintf(stdout, "* %s\n", rsc->id);
	cli_resource_print_location(rsc, NULL);

	for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
	resource_t r = (resource_t ) lpc->data;

	clear_bit(r->flags, pe_rsc_allocating);
	}

	cli_resource_print_colocation(rsc, FALSE, rsc_cmd == 'A', 1);

	} else if (rsc_cmd == 'c') {
	GListPtr lpc = NULL;

	rc = pcmk_ok;
	for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
	rsc = (resource_t *) lpc->data;
	cli_resource_print_cts(rsc);
	}
	cli_resource_print_cts_constraints(&data_set);

	} else if (rsc_cmd == 'F') {
	rc = cli_resource_fail(crmd_channel, host_uname, rsc_id, &data_set);
	if (rc == pcmk_ok) {
	start_mainloop();
	}

	} else if (rsc_cmd == 'O') {
	rc = cli_resource_print_operations(rsc_id, host_uname, TRUE, &data_set);

	} else if (rsc_cmd == 'o') {
	rc = cli_resource_print_operations(rsc_id, host_uname, FALSE, &data_set);

	} else if (rsc_cmd == 'W') {
	rc = cli_resource_search(rsc, rsc_id, &data_set);
	if (rc >= 0) {
	rc = pcmk_ok;
	}

	} else if (rsc_cmd == 'q') {
	rc = cli_resource_print(rsc, &data_set, TRUE);

	} else if (rsc_cmd == 'w') {
	rc = cli_resource_print(rsc, &data_set, FALSE);

	} else if (rsc_cmd == 'Y') {
	node_t *dest = NULL;

	if (host_uname) {
	dest = pe_find_node(data_set.nodes, host_uname);
	if (dest == NULL) {
	- CMD_ERR("Unknown node: %s", host_uname);
	- rc = -ENXIO;
	+ rc = -pcmk_err_node_unknown;
	goto bail;
	}
	}
	cli_resource_why(cib_conn, data_set.resources, rsc, dest);
	rc = pcmk_ok;

	} else if (rsc_cmd == 'U') {
	node_t *dest = NULL;

	if (host_uname) {
	dest = pe_find_node(data_set.nodes, host_uname);
	if (dest == NULL) {
	- CMD_ERR("Unknown node: %s", host_uname);
	- rc = -ENXIO;
	+ rc = -pcmk_err_node_unknown;
	goto bail;
	}
	rc = cli_resource_clear(rsc_id, dest->details->uname, NULL, cib_conn);

	} else {
	rc = cli_resource_clear(rsc_id, NULL, data_set.nodes, cib_conn);
	}

	} else if (rsc_cmd == 'M' && host_uname) {
	rc = cli_resource_move(rsc, rsc_id, host_uname, cib_conn, &data_set);

	} else if (rsc_cmd == 'B' && host_uname) {
	node_t *dest = pe_find_node(data_set.nodes, host_uname);

	if (dest == NULL) {
	- CMD_ERR("Error performing operation: node '%s' is unknown", host_uname);
	- rc = -ENXIO;
	+ rc = -pcmk_err_node_unknown;
	goto bail;
	}
	rc = cli_resource_ban(rsc_id, dest->details->uname, NULL, cib_conn);

	} else if (rsc_cmd == 'B' \|\| rsc_cmd == 'M') {
	if (g_list_length(rsc->running_on) == 1) {
	node_t *current = rsc->running_on->data;
	rc = cli_resource_ban(rsc_id, current->details->uname, NULL, cib_conn);

	} else if (is_set(rsc->flags, pe_rsc_promotable)) {
	int count = 0;
	GListPtr iter = NULL;
	node_t *current = NULL;

	for(iter = rsc->children; iter; iter = iter->next) {
	resource_t child = (resource_t )iter->data;
	enum rsc_role_e child_role = child->fns->state(child, TRUE);

	if(child_role == RSC_ROLE_MASTER) {
	count++;
	current = child->running_on->data;
	}
	}

	if(count == 1 && current) {
	rc = cli_resource_ban(rsc_id, current->details->uname, NULL, cib_conn);

	} else {
	rc = -EINVAL;
	exit_code = CRM_EX_USAGE;
	CMD_ERR("Resource '%s' not moved: active in %d locations (promoted in %d).", rsc_id, g_list_length(rsc->running_on), count);
	CMD_ERR("You can prevent '%s' from running on a specific location with: --ban --node <name>", rsc_id);
	CMD_ERR("You can prevent '%s' from being promoted at a specific location with:"
	" --ban --master --node <name>", rsc_id);
	}

	} else {
	rc = -EINVAL;
	exit_code = CRM_EX_USAGE;
	CMD_ERR("Resource '%s' not moved: active in %d locations.", rsc_id, g_list_length(rsc->running_on));
	CMD_ERR("You can prevent '%s' from running on a specific location with: --ban --node <name>", rsc_id);
	}

	} else if (rsc_cmd == 'G') {
	rc = cli_resource_print_property(rsc, prop_name, &data_set);

	} else if (rsc_cmd == 'S') {
	xmlNode *msg_data = NULL;

	if ((rsc_type == NULL) \|\| !strlen(rsc_type)) {
	CMD_ERR("Must specify -t with resource type");
	rc = -ENXIO;
	goto bail;

	} else if ((prop_value == NULL) \|\| !strlen(prop_value)) {
	CMD_ERR("Must supply -v with new value");
	rc = -EINVAL;
	goto bail;
	}

	CRM_LOG_ASSERT(prop_name != NULL);

	msg_data = create_xml_node(NULL, rsc_type);
	crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);
	crm_xml_add(msg_data, prop_name, prop_value);

	rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
	free_xml(msg_data);

	} else if (rsc_cmd == 'g') {
	rc = cli_resource_print_attribute(rsc, prop_name, &data_set);

	} else if (rsc_cmd == 'p') {
	if (prop_value == NULL \|\| strlen(prop_value) == 0) {
	CMD_ERR("You need to supply a value with the -v option");
	rc = -EINVAL;
	goto bail;
	}

	/* coverity[var_deref_model] False positive */
	rc = cli_resource_update_attribute(rsc, rsc_id, prop_set, prop_id,
	prop_name, prop_value, recursive,
	cib_conn, &data_set);

	} else if (rsc_cmd == 'd') {
	/* coverity[var_deref_model] False positive */
	rc = cli_resource_delete_attribute(rsc, rsc_id, prop_set, prop_id,
	prop_name, cib_conn, &data_set);

	} else if ((rsc_cmd == 'C') && rsc) {
	if (do_force == FALSE) {
	rsc = uber_parent(rsc);
	}
	crmd_replies_needed = 0;

	crm_debug("Erasing failures of %s (%s requested) on %s",
	rsc->id, rsc_id, (host_uname? host_uname: "all nodes"));
	rc = cli_resource_delete(crmd_channel, host_uname, rsc,
	operation, interval_spec, TRUE, &data_set);

	if ((rc == pcmk_ok) && !BE_QUIET) {
	// Show any reasons why resource might stay stopped
	cli_resource_check(cib_conn, rsc);
	}

	if (rc == pcmk_ok) {
	start_mainloop();
	}

	} else if (rsc_cmd == 'C') {
	rc = cli_cleanup_all(crmd_channel, host_uname, operation, interval_spec,
	&data_set);

	} else if ((rsc_cmd == 'R') && rsc) {
	if (do_force == FALSE) {
	rsc = uber_parent(rsc);
	}
	crmd_replies_needed = 0;

	crm_debug("Re-checking the state of %s (%s requested) on %s",
	rsc->id, rsc_id, (host_uname? host_uname: "all nodes"));
	rc = cli_resource_delete(crmd_channel, host_uname, rsc,
	NULL, 0, FALSE, &data_set);

	if ((rc == pcmk_ok) && !BE_QUIET) {
	// Show any reasons why resource might stay stopped
	cli_resource_check(cib_conn, rsc);
	}

	if (rc == pcmk_ok) {
	start_mainloop();
	}

	} else if (rsc_cmd == 'R') {
	const char *router_node = host_uname;
	xmlNode *msg_data = NULL;
	xmlNode *cmd = NULL;
	int attr_options = attrd_opt_none;

	if (host_uname) {
	node_t *node = pe_find_node(data_set.nodes, host_uname);

	if (node && is_remote_node(node)) {
	if (node->details->remote_rsc == NULL \|\| node->details->remote_rsc->running_on == NULL) {
	CMD_ERR("No cluster connection to Pacemaker Remote node %s detected",
	host_uname);
	rc = -ENXIO;
	goto bail;
	}
	node = node->details->remote_rsc->running_on->data;
	router_node = node->details->uname;
	attr_options \|= attrd_opt_remote;
	}
	}

	if (crmd_channel == NULL) {
	printf("Dry run: skipping clean-up of %s due to CIB_file\n",
	host_uname? host_uname : "all nodes");
	rc = pcmk_ok;
	goto bail;
	}

	msg_data = create_xml_node(NULL, "crm-resource-reprobe-op");
	crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, host_uname);
	if (safe_str_neq(router_node, host_uname)) {
	crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
	}

	cmd = create_request(CRM_OP_REPROBE, msg_data, router_node,
	CRM_SYSTEM_CRMD, crm_system_name, our_pid);
	free_xml(msg_data);

	crm_debug("Re-checking the state of all resources on %s", host_uname?host_uname:"all nodes");

	rc = attrd_clear_delegate(NULL, host_uname, NULL, NULL, NULL, NULL,
	attr_options);

	if (crm_ipc_send(crmd_channel, cmd, 0, 0, NULL) > 0) {
	start_mainloop();
	}

	free_xml(cmd);

	} else if (rsc_cmd == 'D') {
	xmlNode *msg_data = NULL;

	if (rsc_type == NULL) {
	CMD_ERR("You need to specify a resource type with -t");
	rc = -ENXIO;
	goto bail;
	}

	msg_data = create_xml_node(NULL, rsc_type);
	crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);

	rc = cib_conn->cmds->remove(cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
	free_xml(msg_data);

	} else {
	CMD_ERR("Unknown command: %c", rsc_cmd);
	}

	bail:

	free(our_pid);

	if (data_set.input != NULL) {
	cleanup_alloc_calculations(&data_set);
	}
	if (cib_conn != NULL) {
	cib_conn->cmds->signoff(cib_conn);
	cib_delete(cib_conn);
	}

	if (is_ocf_rc) {
	exit_code = rc;

	} else if (rc != pcmk_ok) {
	CMD_ERR("Error performing operation: %s", pcmk_strerror(rc));
	if (rc == -pcmk_err_no_quorum) {
	CMD_ERR("To ignore quorum, use --force");
	}
	if (exit_code == CRM_EX_OK) {
	exit_code = crm_errno2exit(rc);
	}
	}

	return crm_exit(exit_code);
	}
	diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
	index 61430c3d8f..297b6f055b 100644
	--- a/tools/crm_resource_runtime.c
	+++ b/tools/crm_resource_runtime.c
	@@ -1,1982 +1,1978 @@
	/*
	* Copyright 2004-2018 Andrew Beekhof <andrew@beekhof.net>
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_resource.h>

	int resource_verbose = 0;
	bool do_force = FALSE;
	int crmd_replies_needed = 1; /* The welcome message */

	const char *attr_set_type = XML_TAG_ATTR_SETS;

	static int
	do_find_resource(const char rsc, resource_t the_rsc, pe_working_set_t * data_set)
	{
	int found = 0;
	GListPtr lpc = NULL;

	for (lpc = the_rsc->running_on; lpc != NULL; lpc = lpc->next) {
	node_t node = (node_t ) lpc->data;

	if (BE_QUIET) {
	fprintf(stdout, "%s\n", node->details->uname);
	} else {
	const char *state = "";

	if (!pe_rsc_is_clone(the_rsc) && the_rsc->fns->state(the_rsc, TRUE) == RSC_ROLE_MASTER) {
	state = "Master";
	}
	fprintf(stdout, "resource %s is running on: %s %s\n", rsc, node->details->uname, state);
	}

	found++;
	}

	if (BE_QUIET == FALSE && found == 0) {
	fprintf(stderr, "resource %s is NOT running\n", rsc);
	}

	return found;
	}

	int
	cli_resource_search(resource_t rsc, const char requested_name,
	pe_working_set_t *data_set)
	{
	int found = 0;
	resource_t *parent = uber_parent(rsc);

	if (pe_rsc_is_clone(rsc)) {
	for (GListPtr iter = rsc->children; iter != NULL; iter = iter->next) {
	found += do_find_resource(requested_name, iter->data, data_set);
	}

	/* The anonymous clone children's common ID is supplied */
	} else if (pe_rsc_is_clone(parent)
	&& is_not_set(rsc->flags, pe_rsc_unique)
	&& rsc->clone_name
	&& safe_str_eq(requested_name, rsc->clone_name)
	&& safe_str_neq(requested_name, rsc->id)) {

	for (GListPtr iter = parent->children; iter; iter = iter->next) {
	found += do_find_resource(requested_name, iter->data, data_set);
	}

	} else {
	found += do_find_resource(requested_name, rsc, data_set);
	}

	return found;
	}

	#define XPATH_MAX 1024

	static int
	find_resource_attr(cib_t * the_cib, const char attr, const char rsc, const char *set_type,
	const char set_name, const char attr_id, const char attr_name, char *value)
	{
	int offset = 0;
	int rc = pcmk_ok;
	xmlNode *xml_search = NULL;
	char *xpath_string = NULL;

	if(value) {
	*value = NULL;
	}

	if(the_cib == NULL) {
	return -ENOTCONN;
	}

	xpath_string = calloc(1, XPATH_MAX);
	offset +=
	snprintf(xpath_string + offset, XPATH_MAX - offset, "%s", get_object_path("resources"));

	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "//*[@id=\"%s\"]", rsc);

	if (set_type) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "/%s", set_type);
	if (set_name) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "[@id=\"%s\"]", set_name);
	}
	}

	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "//nvpair[");
	if (attr_id) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "@id=\"%s\"", attr_id);
	}

	if (attr_name) {
	if (attr_id) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, " and ");
	}
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "@name=\"%s\"", attr_name);
	}
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "]");
	CRM_LOG_ASSERT(offset > 0);

	rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);

	if (rc != pcmk_ok) {
	goto bail;
	}

	crm_log_xml_debug(xml_search, "Match");
	if (xml_has_children(xml_search)) {
	xmlNode *child = NULL;

	rc = -EINVAL;
	printf("Multiple attributes match name=%s\n", attr_name);

	for (child = __xml_first_child(xml_search); child != NULL; child = __xml_next(child)) {
	printf(" Value: %s \t(id=%s)\n",
	crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
	}

	} else if(value) {
	const char *tmp = crm_element_value(xml_search, attr);

	if (tmp) {
	*value = strdup(tmp);
	}
	}

	bail:
	free(xpath_string);
	free_xml(xml_search);
	return rc;
	}

	static resource_t *
	find_matching_attr_resource(resource_t * rsc, const char * rsc_id, const char * attr_set, const char * attr_id,
	const char * attr_name, cib_t * cib, const char * cmd)
	{
	int rc = pcmk_ok;
	char *lookup_id = NULL;
	char *local_attr_id = NULL;

	if(do_force == TRUE) {
	return rsc;

	} else if(rsc->parent) {
	switch(rsc->parent->variant) {
	case pe_group:
	if (BE_QUIET == FALSE) {
	printf("Performing %s of '%s' for '%s' will not apply to its peers in '%s'\n", cmd, attr_name, rsc_id, rsc->parent->id);
	}
	break;

	case pe_clone:
	rc = find_resource_attr(cib, XML_ATTR_ID, rsc_id, attr_set_type, attr_set, attr_id, attr_name, &local_attr_id);
	free(local_attr_id);

	if(rc != pcmk_ok) {
	rsc = rsc->parent;
	if (BE_QUIET == FALSE) {
	printf("Performing %s of '%s' on '%s', the parent of '%s'\n", cmd, attr_name, rsc->id, rsc_id);
	}
	}
	break;
	default:
	break;
	}

	} else if (rsc->parent && BE_QUIET == FALSE) {
	printf("Forcing %s of '%s' for '%s' instead of '%s'\n", cmd, attr_name, rsc_id, rsc->parent->id);

	} else if(rsc->parent == NULL && rsc->children) {
	resource_t *child = rsc->children->data;

	if(child->variant == pe_native) {
	lookup_id = clone_strip(child->id); /* Could be a cloned group! */
	rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name, &local_attr_id);

	if(rc == pcmk_ok) {
	rsc = child;
	if (BE_QUIET == FALSE) {
	printf("A value for '%s' already exists in child '%s', performing %s on that instead of '%s'\n", attr_name, lookup_id, cmd, rsc_id);
	}
	}

	free(local_attr_id);
	free(lookup_id);
	}
	}

	return rsc;
	}

	int
	cli_resource_update_attribute(resource_t rsc, const char requested_name,
	const char attr_set, const char attr_id,
	const char attr_name, const char attr_value,
	bool recursive, cib_t *cib,
	pe_working_set_t *data_set)
	{
	int rc = pcmk_ok;
	static bool need_init = TRUE;

	char *lookup_id = NULL;
	char *local_attr_id = NULL;
	char *local_attr_set = NULL;

	xmlNode *xml_top = NULL;
	xmlNode *xml_obj = NULL;

	if(attr_id == NULL
	&& do_force == FALSE
	&& pcmk_ok != find_resource_attr(
	cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL, NULL, NULL, attr_name, NULL)) {
	printf("\n");
	}

	if (safe_str_eq(attr_set_type, XML_TAG_ATTR_SETS)) {
	if (do_force == FALSE) {
	rc = find_resource_attr(cib, XML_ATTR_ID, uber_parent(rsc)->id,
	XML_TAG_META_SETS, attr_set, attr_id,
	attr_name, &local_attr_id);
	if (rc == pcmk_ok && BE_QUIET == FALSE) {
	printf("WARNING: There is already a meta attribute for '%s' called '%s' (id=%s)\n",
	uber_parent(rsc)->id, attr_name, local_attr_id);
	printf(" Delete '%s' first or use --force to override\n", local_attr_id);
	}
	free(local_attr_id);
	if (rc == pcmk_ok) {
	return -ENOTUNIQ;
	}
	}

	} else {
	rsc = find_matching_attr_resource(rsc, requested_name, attr_set,
	attr_id, attr_name, cib, "update");
	}

	lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
	rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name,
	&local_attr_id);

	if (rc == pcmk_ok) {
	crm_debug("Found a match for name=%s: id=%s", attr_name, local_attr_id);
	attr_id = local_attr_id;

	} else if (rc != -ENXIO) {
	free(lookup_id);
	free(local_attr_id);
	return rc;

	} else {
	const char *tag = crm_element_name(rsc->xml);

	if (attr_set == NULL) {
	local_attr_set = crm_concat(lookup_id, attr_set_type, '-');
	attr_set = local_attr_set;
	}
	if (attr_id == NULL) {
	local_attr_id = crm_concat(attr_set, attr_name, '-');
	attr_id = local_attr_id;
	}

	xml_top = create_xml_node(NULL, tag);
	crm_xml_add(xml_top, XML_ATTR_ID, lookup_id);

	xml_obj = create_xml_node(xml_top, attr_set_type);
	crm_xml_add(xml_obj, XML_ATTR_ID, attr_set);
	}

	xml_obj = crm_create_nvpair_xml(xml_obj, attr_id, attr_name, attr_value);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	crm_log_xml_debug(xml_top, "Update");

	rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top, cib_options);
	if (rc == pcmk_ok && BE_QUIET == FALSE) {
	printf("Set '%s' option: id=%s%s%s%s%s=%s\n", lookup_id, local_attr_id,
	attr_set ? " set=" : "", attr_set ? attr_set : "",
	attr_name ? " name=" : "", attr_name ? attr_name : "", attr_value);
	}

	free_xml(xml_top);

	free(lookup_id);
	free(local_attr_id);
	free(local_attr_set);

	if(recursive && safe_str_eq(attr_set_type, XML_TAG_META_SETS)) {
	GListPtr lpc = NULL;

	if(need_init) {
	xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);

	need_init = FALSE;
	unpack_constraints(cib_constraints, data_set);

	for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
	resource_t r = (resource_t ) lpc->data;

	clear_bit(r->flags, pe_rsc_allocating);
	}
	}

	crm_debug("Looking for dependencies %p", rsc->rsc_cons_lhs);
	set_bit(rsc->flags, pe_rsc_allocating);
	for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
	rsc_colocation_t cons = (rsc_colocation_t ) lpc->data;
	resource_t *peer = cons->rsc_lh;

	crm_debug("Checking %s %d", cons->id, cons->score);
	if (cons->score > 0 && is_not_set(peer->flags, pe_rsc_allocating)) {
	/* Don't get into colocation loops */
	crm_debug("Setting %s=%s for dependent resource %s", attr_name, attr_value, peer->id);
	cli_resource_update_attribute(peer, peer->id, NULL, NULL,
	attr_name, attr_value, recursive,
	cib, data_set);
	}
	}
	}

	return rc;
	}

	int
	cli_resource_delete_attribute(resource_t rsc, const char requested_name,
	const char attr_set, const char attr_id,
	const char attr_name, cib_t cib,
	pe_working_set_t *data_set)
	{
	xmlNode *xml_obj = NULL;

	int rc = pcmk_ok;
	char *lookup_id = NULL;
	char *local_attr_id = NULL;

	if(attr_id == NULL
	&& do_force == FALSE
	&& find_resource_attr(
	cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL, NULL, NULL, attr_name, NULL) != pcmk_ok) {
	printf("\n");
	}

	if(safe_str_eq(attr_set_type, XML_TAG_META_SETS)) {
	rsc = find_matching_attr_resource(rsc, requested_name, attr_set,
	attr_id, attr_name, cib, "delete");
	}

	lookup_id = clone_strip(rsc->id);
	rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name,
	&local_attr_id);

	if (rc == -ENXIO) {
	free(lookup_id);
	return pcmk_ok;

	} else if (rc != pcmk_ok) {
	free(lookup_id);
	return rc;
	}

	if (attr_id == NULL) {
	attr_id = local_attr_id;
	}

	xml_obj = crm_create_nvpair_xml(NULL, attr_id, attr_name, NULL);
	crm_log_xml_debug(xml_obj, "Delete");

	CRM_ASSERT(cib);
	rc = cib->cmds->remove(cib, XML_CIB_TAG_RESOURCES, xml_obj, cib_options);

	if (rc == pcmk_ok && BE_QUIET == FALSE) {
	printf("Deleted '%s' option: id=%s%s%s%s%s\n", lookup_id, local_attr_id,
	attr_set ? " set=" : "", attr_set ? attr_set : "",
	attr_name ? " name=" : "", attr_name ? attr_name : "");
	}

	free(lookup_id);
	free_xml(xml_obj);
	free(local_attr_id);
	return rc;
	}

	static int
	send_lrm_rsc_op(crm_ipc_t * crmd_channel, const char *op,
	const char host_uname, const char rsc_id,
	bool only_failed, pe_working_set_t * data_set)
	{
	char *our_pid = NULL;
	char *key = NULL;
	int rc = -ECOMM;
	xmlNode *cmd = NULL;
	xmlNode *xml_rsc = NULL;
	const char *value = NULL;
	const char *router_node = host_uname;
	xmlNode *params = NULL;
	xmlNode *msg_data = NULL;
	resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);

	if (rsc == NULL) {
	CMD_ERR("Resource %s not found", rsc_id);
	return -ENXIO;

	} else if (rsc->variant != pe_native) {
	CMD_ERR("We can only process primitive resources, not %s", rsc_id);
	return -EINVAL;

	} else if (host_uname == NULL) {
	CMD_ERR("Please supply a node name with --node");
	return -EINVAL;
	} else {
	node_t *node = pe_find_node(data_set->nodes, host_uname);

	if (node && is_remote_node(node)) {
	if (node->details->remote_rsc == NULL \|\| node->details->remote_rsc->running_on == NULL) {
	CMD_ERR("No cluster connection to Pacemaker Remote node %s detected",
	host_uname);
	return -ENXIO;
	}
	node = node->details->remote_rsc->running_on->data;
	router_node = node->details->uname;
	}
	}

	key = generate_transition_key(0, getpid(), 0, "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");

	msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
	crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
	free(key);

	crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, host_uname);
	if (safe_str_neq(router_node, host_uname)) {
	crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
	}

	xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
	if (rsc->clone_name) {
	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->clone_name);
	crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc->id);

	} else {
	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->id);
	}

	value = crm_copy_xml_element(rsc->xml, xml_rsc, XML_ATTR_TYPE);
	if (value == NULL) {
	CMD_ERR("%s has no type! Aborting...", rsc_id);
	return -ENXIO;
	}

	value = crm_copy_xml_element(rsc->xml, xml_rsc, XML_AGENT_ATTR_CLASS);
	if (value == NULL) {
	CMD_ERR("%s has no class! Aborting...", rsc_id);
	return -ENXIO;
	}

	crm_copy_xml_element(rsc->xml, xml_rsc, XML_AGENT_ATTR_PROVIDER);

	params = create_xml_node(msg_data, XML_TAG_ATTRS);
	crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);

	key = crm_meta_name(XML_LRM_ATTR_INTERVAL_MS);
	crm_xml_add(params, key, "60000"); /* 1 minute */
	free(key);

	our_pid = crm_getpid_s();
	cmd = create_request(op, msg_data, router_node, CRM_SYSTEM_CRMD, crm_system_name, our_pid);

	/* crm_log_xml_warn(cmd, "send_lrm_rsc_op"); */
	free_xml(msg_data);

	if (crm_ipc_send(crmd_channel, cmd, 0, 0, NULL) > 0) {
	rc = 0;

	} else {
	crm_debug("Could not send %s op to the controller", op);
	rc = -ENOTCONN;
	}

	free_xml(cmd);
	return rc;
	}

	/*!
	* \internal
	* \brief Get resource name as used in failure-related node attributes
	*
	* \param[in] rsc Resource to check
	*
	* \return Newly allocated string containing resource's fail name
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	rsc_fail_name(resource_t *rsc)
	{
	const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);

	return is_set(rsc->flags, pe_rsc_unique)? strdup(name) : clone_strip(name);
	}

	static int
	clear_rsc_history(crm_ipc_t crmd_channel, const char host_uname,
	const char rsc_id, pe_working_set_t data_set)
	{
	int rc = pcmk_ok;

	/* Erase the resource's entire LRM history in the CIB, even if we're only
	* clearing a single operation's fail count. If we erased only entries for a
	* single operation, we might wind up with a wrong idea of the current
	* resource state, and we might not re-probe the resource.
	*/
	rc = send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_DELETE, host_uname, rsc_id,
	TRUE, data_set);
	if (rc != pcmk_ok) {
	return rc;
	}
	crmd_replies_needed++;

	crm_trace("Processing %d mainloop inputs", crmd_replies_needed);
	while (g_main_context_iteration(NULL, FALSE)) {
	crm_trace("Processed mainloop input, %d still remaining",
	crmd_replies_needed);
	}

	if (crmd_replies_needed < 0) {
	crmd_replies_needed = 0;
	}
	return rc;
	}

	static int
	clear_rsc_failures(crm_ipc_t crmd_channel, const char node_name,
	const char rsc_id, const char operation,
	const char interval_spec, pe_working_set_t data_set)
	{
	int rc = pcmk_ok;
	const char *failed_value = NULL;
	const char *failed_id = NULL;
	const char *interval_ms_s = NULL;
	GHashTable *rscs = NULL;
	GHashTableIter iter;

	/* Create a hash table to use as a set of resources to clean. This lets us
	* clean each resource only once (per node) regardless of how many failed
	* operations it has.
	*/
	rscs = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL);

	// Normalize interval to milliseconds for comparison to history entry
	if (operation) {
	interval_ms_s = crm_strdup_printf("%u",
	crm_parse_interval_spec(interval_spec));
	}

	for (xmlNode *xml_op = __xml_first_child(data_set->failed); xml_op != NULL;
	xml_op = __xml_next(xml_op)) {

	failed_id = crm_element_value(xml_op, XML_LRM_ATTR_RSCID);
	if (failed_id == NULL) {
	// Malformed history entry, should never happen
	continue;
	}

	// No resource specified means all resources match
	if (rsc_id) {
	resource_t *fail_rsc = pe_find_resource_with_flags(data_set->resources,
	failed_id,
	pe_find_renamed\|pe_find_anon);

	if (!fail_rsc \|\| safe_str_neq(rsc_id, fail_rsc->id)) {
	continue;
	}
	}

	// Host name should always have been provided by this point
	failed_value = crm_element_value(xml_op, XML_ATTR_UNAME);
	if (safe_str_neq(node_name, failed_value)) {
	continue;
	}

	// No operation specified means all operations match
	if (operation) {
	failed_value = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	if (safe_str_neq(operation, failed_value)) {
	continue;
	}

	// Interval (if operation was specified) defaults to 0 (not all)
	failed_value = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
	if (safe_str_neq(interval_ms_s, failed_value)) {
	continue;
	}
	}

	g_hash_table_add(rscs, (gpointer) failed_id);
	}

	g_hash_table_iter_init(&iter, rscs);
	while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
	crm_debug("Erasing failures of %s on %s", failed_id, node_name);
	rc = clear_rsc_history(crmd_channel, node_name, failed_id, data_set);
	if (rc != pcmk_ok) {
	return rc;
	}
	}
	g_hash_table_destroy(rscs);
	return rc;
	}

	static int
	clear_rsc_fail_attrs(resource_t rsc, const char operation,
	const char interval_spec, node_t node)
	{
	int rc = pcmk_ok;
	int attr_options = attrd_opt_none;
	char *rsc_name = rsc_fail_name(rsc);

	if (is_remote_node(node)) {
	attr_options \|= attrd_opt_remote;
	}
	rc = attrd_clear_delegate(NULL, node->details->uname, rsc_name, operation,
	interval_spec, NULL, attr_options);
	free(rsc_name);
	return rc;
	}

	int
	cli_resource_delete(crm_ipc_t crmd_channel, const char host_uname,
	resource_t rsc, const char operation,
	const char *interval_spec, bool just_failures,
	pe_working_set_t *data_set)
	{
	int rc = pcmk_ok;
	node_t *node = NULL;

	if (rsc == NULL) {
	return -ENXIO;

	} else if (rsc->children) {
	GListPtr lpc = NULL;

	for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
	resource_t child = (resource_t ) lpc->data;

	rc = cli_resource_delete(crmd_channel, host_uname, child, operation,
	interval_spec, just_failures, data_set);
	if (rc != pcmk_ok) {
	return rc;
	}
	}
	return pcmk_ok;

	} else if (host_uname == NULL) {
	GListPtr lpc = NULL;
	GListPtr nodes = g_hash_table_get_values(rsc->known_on);

	if(nodes == NULL && do_force) {
	nodes = node_list_dup(data_set->nodes, FALSE, FALSE);

	} else if(nodes == NULL && rsc->exclusive_discover) {
	GHashTableIter iter;
	pe_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
	if(node->weight >= 0) {
	nodes = g_list_prepend(nodes, node);
	}
	}

	} else if(nodes == NULL) {
	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	}

	for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node = (node_t *) lpc->data;

	if (node->details->online) {
	rc = cli_resource_delete(crmd_channel, node->details->uname,
	rsc, operation, interval_spec,
	just_failures, data_set);
	}
	if (rc != pcmk_ok) {
	g_list_free(nodes);
	return rc;
	}
	}

	g_list_free(nodes);
	return pcmk_ok;
	}

	node = pe_find_node(data_set->nodes, host_uname);

	if (node == NULL) {
	printf("Unable to clean up %s because node %s not found\n",
	rsc->id, host_uname);
	return -ENODEV;
	}

	if (!node->details->rsc_discovery_enabled) {
	printf("Unable to clean up %s because resource discovery disabled on %s\n",
	rsc->id, host_uname);
	return -EOPNOTSUPP;
	}

	if (crmd_channel == NULL) {
	printf("Dry run: skipping clean-up of %s on %s due to CIB_file\n",
	rsc->id, host_uname);
	return pcmk_ok;
	}

	rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
	if (rc != pcmk_ok) {
	printf("Unable to clean up %s failures on %s: %s\n",
	rsc->id, host_uname, pcmk_strerror(rc));
	return rc;
	}

	if (just_failures) {
	rc = clear_rsc_failures(crmd_channel, host_uname, rsc->id, operation,
	interval_spec, data_set);
	} else {
	rc = clear_rsc_history(crmd_channel, host_uname, rsc->id, data_set);
	}
	if (rc != pcmk_ok) {
	printf("Cleaned %s failures on %s, but unable to clean history: %s\n",
	rsc->id, host_uname, pcmk_strerror(rc));
	} else {
	printf("Cleaned up %s on %s\n", rsc->id, host_uname);
	}
	return rc;
	}

	int
	cli_cleanup_all(crm_ipc_t crmd_channel, const char node_name,
	const char operation, const char interval_spec,
	pe_working_set_t *data_set)
	{
	int rc = pcmk_ok;
	int attr_options = attrd_opt_none;
	const char *display_name = node_name? node_name : "all nodes";

	if (crmd_channel == NULL) {
	printf("Dry run: skipping clean-up of %s due to CIB_file\n",
	display_name);
	return pcmk_ok;
	}
	crmd_replies_needed = 0;

	if (node_name) {
	node_t *node = pe_find_node(data_set->nodes, node_name);

	if (node == NULL) {
	CMD_ERR("Unknown node: %s", node_name);
	return -ENXIO;
	}
	if (is_remote_node(node)) {
	attr_options \|= attrd_opt_remote;
	}
	}

	rc = attrd_clear_delegate(NULL, node_name, NULL, operation, interval_spec,
	NULL, attr_options);
	if (rc != pcmk_ok) {
	printf("Unable to clean up all failures on %s: %s\n",
	display_name, pcmk_strerror(rc));
	return rc;
	}

	if (node_name) {
	rc = clear_rsc_failures(crmd_channel, node_name, NULL,
	operation, interval_spec, data_set);
	if (rc != pcmk_ok) {
	printf("Cleaned all resource failures on %s, but unable to clean history: %s\n",
	node_name, pcmk_strerror(rc));
	return rc;
	}
	} else {
	for (GList *iter = data_set->nodes; iter; iter = iter->next) {
	pe_node_t node = (pe_node_t ) iter->data;

	rc = clear_rsc_failures(crmd_channel, node->details->uname, NULL,
	operation, interval_spec, data_set);
	if (rc != pcmk_ok) {
	printf("Cleaned all resource failures on all nodes, but unable to clean history: %s\n",
	pcmk_strerror(rc));
	return rc;
	}
	}
	}

	printf("Cleaned up all resources on %s\n", display_name);
	return pcmk_ok;
	}

	void
	cli_resource_check(cib_t * cib_conn, resource_t *rsc)
	{
	int need_nl = 0;
	char *role_s = NULL;
	char *managed = NULL;
	resource_t *parent = uber_parent(rsc);

	find_resource_attr(cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
	NULL, NULL, NULL, XML_RSC_ATTR_MANAGED, &managed);

	find_resource_attr(cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
	NULL, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, &role_s);

	if(role_s) {
	enum rsc_role_e role = text2role(role_s);
	if(role == RSC_ROLE_UNKNOWN) {
	// Treated as if unset

	} else if(role == RSC_ROLE_STOPPED) {
	printf("\n * The configuration specifies that '%s' should remain stopped\n", parent->id);
	need_nl++;

	} else if (is_set(parent->flags, pe_rsc_promotable)
	&& (role == RSC_ROLE_SLAVE)) {
	printf("\n * The configuration specifies that '%s' should not be promoted\n", parent->id);
	need_nl++;
	}
	}

	if(managed && crm_is_true(managed) == FALSE) {
	printf("%s * The configuration prevents the cluster from stopping or starting '%s' (unmanaged)\n", need_nl == 0?"\n":"", parent->id);
	need_nl++;
	}

	if(need_nl) {
	printf("\n");
	}
	}

	int
	cli_resource_fail(crm_ipc_t * crmd_channel, const char *host_uname,
	const char rsc_id, pe_working_set_t data_set)
	{
	crm_warn("Failing: %s", rsc_id);
	return send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_FAIL, host_uname, rsc_id, FALSE, data_set);
	}

	static GHashTable *
	generate_resource_params(resource_t * rsc, pe_working_set_t * data_set)
	{
	GHashTable *params = NULL;
	GHashTable *meta = NULL;
	GHashTable *combined = NULL;
	GHashTableIter iter;

	if (!rsc) {
	crm_err("Resource does not exist in config");
	return NULL;
	}

	params = crm_str_table_new();
	meta = crm_str_table_new();
	combined = crm_str_table_new();

	get_rsc_attributes(params, rsc, NULL /* TODO: Pass in local node */ , data_set);
	get_meta_attributes(meta, rsc, NULL /* TODO: Pass in local node */ , data_set);

	if (params) {
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	g_hash_table_insert(combined, strdup(key), strdup(value));
	}
	g_hash_table_destroy(params);
	}

	if (meta) {
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	char *crm_name = crm_meta_name(key);

	g_hash_table_insert(combined, crm_name, strdup(value));
	}
	g_hash_table_destroy(meta);
	}

	return combined;
	}

	static bool resource_is_running_on(resource_t rsc, const char host)
	{
	bool found = TRUE;
	GListPtr hIter = NULL;
	GListPtr hosts = NULL;

	if(rsc == NULL) {
	return FALSE;
	}

	rsc->fns->location(rsc, &hosts, TRUE);
	for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
	pe_node_t node = (pe_node_t ) hIter->data;

	if(strcmp(host, node->details->uname) == 0) {
	crm_trace("Resource %s is running on %s\n", rsc->id, host);
	goto done;
	} else if(strcmp(host, node->details->id) == 0) {
	crm_trace("Resource %s is running on %s\n", rsc->id, host);
	goto done;
	}
	}

	if(host != NULL) {
	crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
	found = FALSE;

	} else if(host == NULL && hosts == NULL) {
	crm_trace("Resource %s is not running\n", rsc->id);
	found = FALSE;
	}

	done:

	g_list_free(hosts);
	return found;
	}

	/*!
	* \internal
	* \brief Create a list of all resources active on host from a given list
	*
	* \param[in] host Name of host to check whether resources are active
	* \param[in] rsc_list List of resources to check
	*
	* \return New list of resources from list that are active on host
	*/
	static GList *
	get_active_resources(const char host, GList rsc_list)
	{
	GList *rIter = NULL;
	GList *active = NULL;

	for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
	resource_t rsc = (resource_t ) rIter->data;

	/* Expand groups to their members, because if we're restarting a member
	* other than the first, we can't otherwise tell which resources are
	* stopping and starting.
	*/
	if (rsc->variant == pe_group) {
	active = g_list_concat(active,
	get_active_resources(host, rsc->children));
	} else if (resource_is_running_on(rsc, host)) {
	active = g_list_append(active, strdup(rsc->id));
	}
	}
	return active;
	}

	static GList*
	subtract_lists(GList from, GList items)
	{
	GList *item = NULL;
	GList *result = g_list_copy(from);

	for (item = items; item != NULL; item = item->next) {
	GList *candidate = NULL;
	for (candidate = from; candidate != NULL; candidate = candidate->next) {
	crm_info("Comparing %s with %s", (const char *) candidate->data,
	(const char *) item->data);
	if(strcmp(candidate->data, item->data) == 0) {
	result = g_list_remove(result, candidate->data);
	break;
	}
	}
	}

	return result;
	}

	static void dump_list(GList items, const char tag)
	{
	int lpc = 0;
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
	lpc++;
	}
	}

	static void display_list(GList items, const char tag)
	{
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	fprintf(stdout, "%s%s\n", tag, (const char *)item->data);
	}
	}

	/*!
	* \internal
	* \brief Upgrade XML to latest schema version and use it as working set input
	*
	* This also updates the working set timestamp to the current time.
	*
	* \param[in] data_set Working set instance to update
	* \param[in] xml XML to use as input
	*
	* \return pcmk_ok on success, -ENOKEY if unable to upgrade XML
	* \note On success, caller is responsible for freeing memory allocated for
	* data_set->now.
	* \todo This follows the example of other callers of cli_config_update()
	* and returns -ENOKEY ("Required key not available") if that fails,
	* but perhaps -pcmk_err_schema_validation would be better in that case.
	*/
	int
	update_working_set_xml(pe_working_set_t data_set, xmlNode *xml)
	{
	if (cli_config_update(xml, NULL, FALSE) == FALSE) {
	return -ENOKEY;
	}
	data_set->input = *xml;
	data_set->now = crm_time_new(NULL);
	return pcmk_ok;
	}

	/*!
	* \internal
	* \brief Update a working set's XML input based on a CIB query
	*
	* \param[in] data_set Data set instance to initialize
	* \param[in] cib Connection to the CIB manager
	*
	* \return pcmk_ok on success, -errno on failure
	* \note On success, caller is responsible for freeing memory allocated for
	* data_set->input and data_set->now.
	*/
	static int
	update_working_set_from_cib(pe_working_set_t * data_set, cib_t *cib)
	{
	xmlNode *cib_xml_copy = NULL;
	int rc;

	rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local \| cib_sync_call);
	if (rc != pcmk_ok) {
	fprintf(stderr, "Could not obtain the current CIB: %s (%d)\n", pcmk_strerror(rc), rc);
	return rc;
	}
	rc = update_working_set_xml(data_set, &cib_xml_copy);
	if (rc != pcmk_ok) {
	fprintf(stderr, "Could not upgrade the current CIB XML\n");
	free_xml(cib_xml_copy);
	return rc;
	}
	return pcmk_ok;
	}

	static int
	update_dataset(cib_t cib, pe_working_set_t data_set, bool simulate)
	{
	char *pid = NULL;
	char *shadow_file = NULL;
	cib_t *shadow_cib = NULL;
	int rc;

	cleanup_alloc_calculations(data_set);
	rc = update_working_set_from_cib(data_set, cib);
	if (rc != pcmk_ok) {
	return rc;
	}

	if(simulate) {
	pid = crm_getpid_s();
	shadow_cib = cib_shadow_new(pid);
	shadow_file = get_shadow_file(pid);

	if (shadow_cib == NULL) {
	fprintf(stderr, "Could not create shadow cib: '%s'\n", pid);
	rc = -ENXIO;
	goto cleanup;
	}

	rc = write_xml_file(data_set->input, shadow_file, FALSE);

	if (rc < 0) {
	fprintf(stderr, "Could not populate shadow cib: %s (%d)\n", pcmk_strerror(rc), rc);
	goto cleanup;
	}

	rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
	if(rc != pcmk_ok) {
	fprintf(stderr, "Could not connect to shadow cib: %s (%d)\n", pcmk_strerror(rc), rc);
	goto cleanup;
	}

	do_calculations(data_set, data_set->input, NULL);
	run_simulation(data_set, shadow_cib, NULL, TRUE);
	rc = update_dataset(shadow_cib, data_set, FALSE);

	} else {
	cluster_status(data_set);
	}

	cleanup:
	/* Do not free data_set->input here, we need rsc->xml to be valid later on */
	cib_delete(shadow_cib);
	free(pid);

	if(shadow_file) {
	unlink(shadow_file);
	free(shadow_file);
	}

	return rc;
	}

	static int
	max_delay_for_resource(pe_working_set_t * data_set, resource_t *rsc)
	{
	int delay = 0;
	int max_delay = 0;

	if(rsc && rsc->children) {
	GList *iter = NULL;

	for(iter = rsc->children; iter; iter = iter->next) {
	resource_t child = (resource_t )iter->data;

	delay = max_delay_for_resource(data_set, child);
	if(delay > max_delay) {
	double seconds = delay / 1000.0;
	crm_trace("Calculated new delay of %.1fs due to %s", seconds, child->id);
	max_delay = delay;
	}
	}

	} else if(rsc) {
	char *key = crm_strdup_printf("%s_%s_0", rsc->id, RSC_STOP);
	action_t *stop = custom_action(rsc, key, RSC_STOP, NULL, TRUE, FALSE, data_set);
	const char *value = g_hash_table_lookup(stop->meta, XML_ATTR_TIMEOUT);

	max_delay = crm_int_helper(value, NULL);
	pe_free_action(stop);
	}


	return max_delay;
	}

	static int
	max_delay_in(pe_working_set_t * data_set, GList *resources)
	{
	int max_delay = 0;
	GList *item = NULL;

	for (item = resources; item != NULL; item = item->next) {
	int delay = 0;
	resource_t rsc = pe_find_resource(data_set->resources, (const char )item->data);

	delay = max_delay_for_resource(data_set, rsc);

	if(delay > max_delay) {
	double seconds = delay / 1000.0;
	crm_trace("Calculated new delay of %.1fs due to %s", seconds, rsc->id);
	max_delay = delay;
	}
	}

	return 5 + (max_delay / 1000);
	}

	#define waiting_for_starts(d, r, h) ((g_list_length(d) > 0) \|\| \
	(resource_is_running_on((r), (h)) == FALSE))

	/*!
	* \internal
	* \brief Restart a resource (on a particular host if requested).
	*
	* \param[in] rsc The resource to restart
	* \param[in] host The host to restart the resource on (or NULL for all)
	* \param[in] timeout_ms Consider failed if actions do not complete in this time
	* (specified in milliseconds, but a two-second
	* granularity is actually used; if 0, a timeout will be
	* calculated based on the resource timeout)
	* \param[in] cib Connection to the CIB manager
	*
	* \return pcmk_ok on success, -errno on failure (exits on certain failures)
	*/
	int
	cli_resource_restart(resource_t * rsc, const char host, int timeout_ms, cib_t cib)
	{
	int rc = 0;
	int lpc = 0;
	int before = 0;
	int step_timeout_s = 0;
	int sleep_interval = 2;
	int timeout = timeout_ms / 1000;

	bool stop_via_ban = FALSE;
	char *rsc_id = NULL;
	char *orig_target_role = NULL;

	GList *list_delta = NULL;
	GList *target_active = NULL;
	GList *current_active = NULL;
	GList *restart_target_active = NULL;

	pe_working_set_t data_set;

	if(resource_is_running_on(rsc, host) == FALSE) {
	const char *id = rsc->clone_name?rsc->clone_name:rsc->id;
	if(host) {
	printf("%s is not running on %s and so cannot be restarted\n", id, host);
	} else {
	printf("%s is not running anywhere and so cannot be restarted\n", id);
	}
	return -ENXIO;
	}

	/* We might set the target-role meta-attribute */
	attr_set_type = XML_TAG_META_SETS;

	rsc_id = strdup(rsc->id);
	if ((pe_rsc_is_clone(rsc) \|\| pe_bundle_replicas(rsc)) && host) {
	stop_via_ban = TRUE;
	}

	/*
	grab full cib
	determine originally active resources
	disable or ban
	poll cib and watch for affected resources to get stopped
	without --timeout, calculate the stop timeout for each step and wait for that
	if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
	if everything stopped, re-enable or un-ban
	poll cib and watch for affected resources to get started
	without --timeout, calculate the start timeout for each step and wait for that
	if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
	report success

	Optimizations:
	- use constraints to determine ordered list of affected resources
	- Allow a --no-deps option (aka. --force-restart)
	*/


	set_working_set_defaults(&data_set);
	rc = update_dataset(cib, &data_set, FALSE);
	if(rc != pcmk_ok) {
	fprintf(stdout, "Could not get new resource list: %s (%d)\n", pcmk_strerror(rc), rc);
	free(rsc_id);
	return rc;
	}

	restart_target_active = get_active_resources(host, data_set.resources);
	current_active = get_active_resources(host, data_set.resources);

	dump_list(current_active, "Origin");

	if (stop_via_ban) {
	/* Stop the clone or bundle instance by banning it from the host */
	BE_QUIET = TRUE;
	rc = cli_resource_ban(rsc_id, host, NULL, cib);

	} else {
	/* Stop the resource by setting target-role to Stopped.
	* Remember any existing target-role so we can restore it later
	* (though it only makes any difference if it's Slave).
	*/
	char *lookup_id = clone_strip(rsc->id);

	find_resource_attr(cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
	NULL, XML_RSC_ATTR_TARGET_ROLE, &orig_target_role);
	free(lookup_id);
	rc = cli_resource_update_attribute(rsc, rsc_id, NULL, NULL,
	XML_RSC_ATTR_TARGET_ROLE,
	RSC_STOPPED, FALSE, cib, &data_set);
	}
	if(rc != pcmk_ok) {
	fprintf(stderr, "Could not set target-role for %s: %s (%d)\n", rsc_id, pcmk_strerror(rc), rc);
	if (current_active) {
	g_list_free_full(current_active, free);
	}
	if (restart_target_active) {
	g_list_free_full(restart_target_active, free);
	}
	free(rsc_id);
	return crm_exit(crm_errno2exit(rc));
	}

	rc = update_dataset(cib, &data_set, TRUE);
	if(rc != pcmk_ok) {
	fprintf(stderr, "Could not determine which resources would be stopped\n");
	goto failure;
	}

	target_active = get_active_resources(host, data_set.resources);
	dump_list(target_active, "Target");

	list_delta = subtract_lists(current_active, target_active);
	fprintf(stdout, "Waiting for %d resources to stop:\n", g_list_length(list_delta));
	display_list(list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while(g_list_length(list_delta) > 0) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = max_delay_in(&data_set, list_delta) / sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for(lpc = 0; lpc < step_timeout_s && g_list_length(list_delta) > 0; lpc++) {
	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}
	rc = update_dataset(cib, &data_set, FALSE);
	if(rc != pcmk_ok) {
	fprintf(stderr, "Could not determine which resources were stopped\n");
	goto failure;
	}

	if (current_active) {
	g_list_free_full(current_active, free);
	}
	current_active = get_active_resources(host, data_set.resources);
	g_list_free(list_delta);
	list_delta = subtract_lists(current_active, target_active);
	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
	if(before == g_list_length(list_delta)) {
	/* aborted during stop phase, print the contents of list_delta */
	fprintf(stderr, "Could not complete shutdown of %s, %d resources remaining\n", rsc_id, g_list_length(list_delta));
	display_list(list_delta, " * ");
	rc = -ETIME;
	goto failure;
	}

	}

	if (stop_via_ban) {
	rc = cli_resource_clear(rsc_id, host, NULL, cib);

	} else if (orig_target_role) {
	rc = cli_resource_update_attribute(rsc, rsc_id, NULL, NULL,
	XML_RSC_ATTR_TARGET_ROLE,
	orig_target_role, FALSE, cib,
	&data_set);
	free(orig_target_role);
	orig_target_role = NULL;
	} else {
	rc = cli_resource_delete_attribute(rsc, rsc_id, NULL, NULL,
	XML_RSC_ATTR_TARGET_ROLE, cib,
	&data_set);
	}

	if(rc != pcmk_ok) {
	fprintf(stderr, "Could not unset target-role for %s: %s (%d)\n", rsc_id, pcmk_strerror(rc), rc);
	free(rsc_id);
	return crm_exit(crm_errno2exit(rc));
	}

	if (target_active) {
	g_list_free_full(target_active, free);
	}
	target_active = restart_target_active;
	if (list_delta) {
	g_list_free(list_delta);
	}
	list_delta = subtract_lists(target_active, current_active);
	fprintf(stdout, "Waiting for %d resources to start again:\n", g_list_length(list_delta));
	display_list(list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (waiting_for_starts(list_delta, rsc, host)) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = max_delay_in(&data_set, list_delta) / sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {

	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}

	rc = update_dataset(cib, &data_set, FALSE);
	if(rc != pcmk_ok) {
	fprintf(stderr, "Could not determine which resources were started\n");
	goto failure;
	}

	if (current_active) {
	g_list_free_full(current_active, free);
	}

	/* It's OK if dependent resources moved to a different node,
	* so we check active resources on all nodes.
	*/
	current_active = get_active_resources(NULL, data_set.resources);
	g_list_free(list_delta);
	list_delta = subtract_lists(target_active, current_active);
	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	if(before == g_list_length(list_delta)) {
	/* aborted during start phase, print the contents of list_delta */
	fprintf(stdout, "Could not complete restart of %s, %d resources remaining\n", rsc_id, g_list_length(list_delta));
	display_list(list_delta, " * ");
	rc = -ETIME;
	goto failure;
	}

	}

	rc = pcmk_ok;
	goto done;

	failure:
	if (stop_via_ban) {
	cli_resource_clear(rsc_id, host, NULL, cib);
	} else if (orig_target_role) {
	cli_resource_update_attribute(rsc, rsc_id, NULL, NULL,
	XML_RSC_ATTR_TARGET_ROLE,
	orig_target_role, FALSE, cib, &data_set);
	free(orig_target_role);
	} else {
	cli_resource_delete_attribute(rsc, rsc_id, NULL, NULL,
	XML_RSC_ATTR_TARGET_ROLE, cib, &data_set);
	}

	done:
	if (list_delta) {
	g_list_free(list_delta);
	}
	if (current_active) {
	g_list_free_full(current_active, free);
	}
	if (target_active && (target_active != restart_target_active)) {
	g_list_free_full(target_active, free);
	}
	if (restart_target_active) {
	g_list_free_full(restart_target_active, free);
	}
	cleanup_alloc_calculations(&data_set);
	free(rsc_id);
	return rc;
	}

	static inline int action_is_pending(action_t *action)
	{
	if(is_set(action->flags, pe_action_optional)) {
	return FALSE;
	} else if(is_set(action->flags, pe_action_runnable) == FALSE) {
	return FALSE;
	} else if(is_set(action->flags, pe_action_pseudo)) {
	return FALSE;
	} else if(safe_str_eq("notify", action->task)) {
	return FALSE;
	}
	return TRUE;
	}

	/*!
	* \internal
	* \brief Return TRUE if any actions in a list are pending
	*
	* \param[in] actions List of actions to check
	*
	* \return TRUE if any actions in the list are pending, FALSE otherwise
	*/
	static bool
	actions_are_pending(GListPtr actions)
	{
	GListPtr action;

	for (action = actions; action != NULL; action = action->next) {
	action_t a = (action_t )action->data;
	if (action_is_pending(a)) {
	crm_notice("Waiting for %s (flags=0x%.8x)", a->uuid, a->flags);
	return TRUE;
	}
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Print pending actions to stderr
	*
	* \param[in] actions List of actions to check
	*
	* \return void
	*/
	static void
	print_pending_actions(GListPtr actions)
	{
	GListPtr action;

	fprintf(stderr, "Pending actions:\n");
	for (action = actions; action != NULL; action = action->next) {
	action_t a = (action_t ) action->data;

	if (action_is_pending(a)) {
	fprintf(stderr, "\tAction %d: %s", a->id, a->uuid);
	if (a->node) {
	fprintf(stderr, "\ton %s", a->node->details->uname);
	}
	fprintf(stderr, "\n");
	}
	}
	}

	/* For --wait, timeout (in seconds) to use if caller doesn't specify one */
	#define WAIT_DEFAULT_TIMEOUT_S (60 * 60)

	/* For --wait, how long to sleep between cluster state checks */
	#define WAIT_SLEEP_S (2)

	/*!
	* \internal
	* \brief Wait until all pending cluster actions are complete
	*
	* This waits until either the CIB's transition graph is idle or a timeout is
	* reached.
	*
	* \param[in] timeout_ms Consider failed if actions do not complete in this time
	* (specified in milliseconds, but one-second granularity
	* is actually used; if 0, a default will be used)
	* \param[in] cib Connection to the CIB manager
	*
	* \return pcmk_ok on success, -errno on failure
	*/
	int
	wait_till_stable(int timeout_ms, cib_t * cib)
	{
	pe_working_set_t data_set;
	int rc = -1;
	int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
	time_t expire_time = time(NULL) + timeout_s;
	time_t time_diff;
	bool printed_version_warning = BE_QUIET; // i.e. don't print if quiet

	set_working_set_defaults(&data_set);
	do {

	/* Abort if timeout is reached */
	time_diff = expire_time - time(NULL);
	if (time_diff > 0) {
	crm_info("Waiting up to %ld seconds for cluster actions to complete", time_diff);
	} else {
	print_pending_actions(data_set.actions);
	cleanup_alloc_calculations(&data_set);
	return -ETIME;
	}
	if (rc == pcmk_ok) { /* this avoids sleep on first loop iteration */
	sleep(WAIT_SLEEP_S);
	}

	/* Get latest transition graph */
	cleanup_alloc_calculations(&data_set);
	rc = update_working_set_from_cib(&data_set, cib);
	if (rc != pcmk_ok) {
	cleanup_alloc_calculations(&data_set);
	return rc;
	}
	do_calculations(&data_set, data_set.input, NULL);

	if (!printed_version_warning) {
	/* If the DC has a different version than the local node, the two
	* could come to different conclusions about what actions need to be
	* done. Warn the user in this case.
	*
	* @TODO A possible long-term solution would be to reimplement the
	* wait as a new controller operation that would be forwarded to the
	* DC. However, that would have potential problems of its own.
	*/
	const char *dc_version = g_hash_table_lookup(data_set.config_hash,
	"dc-version");

	if (safe_str_neq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION)) {
	printf("warning: --wait command may not work properly in mixed-version cluster\n");
	printed_version_warning = TRUE;
	}
	}

	} while (actions_are_pending(data_set.actions));

	return pcmk_ok;
	}

	int
	cli_resource_execute(resource_t rsc, const char requested_name,
	const char rsc_action, GHashTable override_hash,
	int timeout_ms, cib_t * cib, pe_working_set_t *data_set)
	{
	int rc = pcmk_ok;
	svc_action_t *op = NULL;
	const char *rid = NULL;
	const char *rtype = NULL;
	const char *rprov = NULL;
	const char *rclass = NULL;
	const char *action = NULL;
	GHashTable *params = NULL;

	if (safe_str_eq(rsc_action, "validate")) {
	action = "validate-all";

	} else if (safe_str_eq(rsc_action, "force-check")) {
	action = "monitor";

	} else if (safe_str_eq(rsc_action, "force-stop")) {
	action = rsc_action+6;

	} else if (safe_str_eq(rsc_action, "force-start")
	\|\| safe_str_eq(rsc_action, "force-demote")
	\|\| safe_str_eq(rsc_action, "force-promote")) {
	action = rsc_action+6;

	if(pe_rsc_is_clone(rsc)) {
	rc = cli_resource_search(rsc, requested_name, data_set);
	if(rc > 0 && do_force == FALSE) {
	CMD_ERR("It is not safe to %s %s here: the cluster claims it is already active",
	action, rsc->id);
	CMD_ERR("Try setting target-role=stopped first or specifying --force");
	crm_exit(CRM_EX_UNSAFE);
	}
	}
	}

	if(pe_rsc_is_clone(rsc)) {
	/* Grab the first child resource in the hope it's not a group */
	rsc = rsc->children->data;
	}

	if(rsc->variant == pe_group) {
	CMD_ERR("Sorry, --%s doesn't support group resources", rsc_action);
	crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
	}

	rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);

	if (safe_str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH)) {
	CMD_ERR("Sorry, --%s doesn't support %s resources yet", rsc_action, rclass);
	crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
	}

	params = generate_resource_params(rsc, data_set);

	/* add meta_timeout env needed by some resource agents */
	if (timeout_ms == 0) {
	timeout_ms = pe_get_configured_timeout(rsc, action, data_set);
	}
	g_hash_table_insert(params, strdup("CRM_meta_timeout"),
	crm_strdup_printf("%d", timeout_ms));

	/* add crm_feature_set env needed by some resource agents */
	g_hash_table_insert(params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));

	rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;

	op = resources_action_create(rid, rclass, rprov, rtype, action, 0,
	timeout_ms, params, 0);
	if (op == NULL) {
	/* Re-run with stderr enabled so we can display a sane error message */
	crm_enable_stderr(TRUE);
	op = resources_action_create(rid, rclass, rprov, rtype, action, 0,
	timeout_ms, params, 0);

	/* We know op will be NULL, but this makes static analysis happy */
	services_action_free(op);

	return crm_exit(CRM_EX_DATAERR);
	}


	setenv("HA_debug", resource_verbose > 0 ? "1" : "0", 1);
	if(resource_verbose > 1) {
	setenv("OCF_TRACE_RA", "1", 1);
	}

	if (override_hash) {
	GHashTableIter iter;
	char *name = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, override_hash);
	while (g_hash_table_iter_next(&iter, (gpointer ) & name, (gpointer ) & value)) {
	printf("Overriding the cluster configuration for '%s' with '%s' = '%s'\n",
	rsc->id, name, value);
	g_hash_table_replace(op->params, strdup(name), strdup(value));
	}
	}

	if (services_action_sync(op)) {
	int more, lpc, last;
	char *local_copy = NULL;

	if (op->status == PCMK_LRM_OP_DONE) {
	printf("Operation %s for %s (%s:%s:%s) returned: '%s' (%d)\n",
	action, rsc->id, rclass, rprov ? rprov : "", rtype,
	services_ocf_exitcode_str(op->rc), op->rc);
	} else {
	printf("Operation %s for %s (%s:%s:%s) failed: '%s' (%d)\n",
	action, rsc->id, rclass, rprov ? rprov : "", rtype,
	services_lrm_status_str(op->status), op->status);
	}

	/* hide output for validate-all if not in verbose */
	if (resource_verbose == 0 && safe_str_eq(action, "validate-all"))
	goto done;

	if (op->stdout_data) {
	local_copy = strdup(op->stdout_data);
	more = strlen(local_copy);
	last = 0;

	for (lpc = 0; lpc < more; lpc++) {
	if (local_copy[lpc] == '\n' \|\| local_copy[lpc] == 0) {
	local_copy[lpc] = 0;
	printf(" > stdout: %s\n", local_copy + last);
	last = lpc + 1;
	}
	}
	free(local_copy);
	}
	if (op->stderr_data) {
	local_copy = strdup(op->stderr_data);
	more = strlen(local_copy);
	last = 0;

	for (lpc = 0; lpc < more; lpc++) {
	if (local_copy[lpc] == '\n' \|\| local_copy[lpc] == 0) {
	local_copy[lpc] = 0;
	printf(" > stderr: %s\n", local_copy + last);
	last = lpc + 1;
	}
	}
	free(local_copy);
	}
	}
	done:
	rc = op->rc;
	services_action_free(op);
	return rc;
	}

	int
	cli_resource_move(resource_t rsc, const char rsc_id, const char *host_name,
	cib_t cib, pe_working_set_t data_set)
	{
	int rc = pcmk_ok;
	int count = 0;
	node_t *current = NULL;
	node_t *dest = pe_find_node(data_set->nodes, host_name);
	bool cur_is_dest = FALSE;

	if (scope_master && is_set(rsc->flags, pe_rsc_promotable)) {
	resource_t *p = uber_parent(rsc);

	if (is_set(p->flags, pe_rsc_promotable)) {
	CMD_ERR("Using parent '%s' for --move command instead of '%s'.", rsc->id, rsc_id);
	rsc_id = p->id;
	rsc = p;

	} else {
	CMD_ERR("Ignoring '--master' option: %s is not a promotable resource",
	rsc_id);
	scope_master = FALSE;
	}
	}

	if (is_set(rsc->flags, pe_rsc_promotable)) {
	GListPtr iter = NULL;

	for(iter = rsc->children; iter; iter = iter->next) {
	resource_t child = (resource_t )iter->data;
	enum rsc_role_e child_role = child->fns->state(child, TRUE);

	if(child_role == RSC_ROLE_MASTER) {
	rsc = child;
	count++;
	}
	}

	if(scope_master == FALSE && count == 0) {
	count = g_list_length(rsc->running_on);
	}

	} else if (pe_rsc_is_clone(rsc)) {
	count = g_list_length(rsc->running_on);

	} else if (g_list_length(rsc->running_on) > 1) {
	- CMD_ERR("Resource '%s' not moved: active on multiple nodes", rsc_id);
	- return -ENOTUNIQ;
	+ return -pcmk_err_multiple;
	}

	if(dest == NULL) {
	- CMD_ERR("Error performing operation: node '%s' is unknown", host_name);
	- return -ENXIO;
	+ return -pcmk_err_node_unknown;
	}

	if(g_list_length(rsc->running_on) == 1) {
	current = rsc->running_on->data;
	}

	if(current == NULL) {
	/* Nothing to check */

	} else if(scope_master && rsc->fns->state(rsc, TRUE) != RSC_ROLE_MASTER) {
	crm_trace("%s is already active on %s but not in correct state", rsc_id, dest->details->uname);
	} else if (safe_str_eq(current->details->uname, dest->details->uname)) {
	cur_is_dest = TRUE;
	if (do_force) {
	crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
	rsc_id, scope_master?"promoted":"active", dest->details->uname);
	} else {
	- CMD_ERR("Error performing operation: %s is already %s on %s",
	- rsc_id, scope_master?"promoted":"active", dest->details->uname);
	- return -EEXIST;
	+ return -pcmk_err_already;
	}
	}

	/* Clear any previous constraints for 'dest' */
	cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib);

	/* Record an explicit preference for 'dest' */
	rc = cli_resource_prefer(rsc_id, dest->details->uname, cib);

	crm_trace("%s%s now prefers node %s%s",
	rsc->id, scope_master?" (master)":"", dest->details->uname, do_force?"(forced)":"");

	/* only ban the previous location if current location != destination location.
	* it is possible to use -M to enforce a location without regard of where the
	* resource is currently located */
	if(do_force && (cur_is_dest == FALSE)) {
	/* Ban the original location if possible */
	if(current) {
	(void)cli_resource_ban(rsc_id, current->details->uname, NULL, cib);

	} else if(count > 1) {
	CMD_ERR("Resource '%s' is currently %s in %d locations. One may now move one to %s",
	rsc_id, scope_master?"promoted":"active", count, dest->details->uname);
	CMD_ERR("You can prevent '%s' from being %s at a specific location with:"
	" --ban %s--host <name>", rsc_id, scope_master?"promoted":"active", scope_master?"--master ":"");

	} else {
	crm_trace("Not banning %s from its current location: not active", rsc_id);
	}
	}

	return rc;
	}

	static void
	cli_resource_why_without_rsc_and_host(cib_t *cib_conn,GListPtr resources)
	{
	GListPtr lpc = NULL;
	GListPtr hosts = NULL;

	for (lpc = resources; lpc != NULL; lpc = lpc->next) {
	resource_t rsc = (resource_t ) lpc->data;
	rsc->fns->location(rsc, &hosts, TRUE);

	if (hosts == NULL) {
	printf("Resource %s is not running\n", rsc->id);
	} else {
	printf("Resource %s is running\n", rsc->id);
	}

	cli_resource_check(cib_conn, rsc);
	g_list_free(hosts);
	hosts = NULL;
	}

	}

	static void
	cli_resource_why_with_rsc_and_host(cib_t *cib_conn, GListPtr resources,
	resource_t rsc, const char host_uname)
	{
	if (resource_is_running_on(rsc, host_uname)) {
	printf("Resource %s is running on host %s\n",rsc->id,host_uname);
	} else {
	printf("Resource %s is not running on host %s\n", rsc->id, host_uname);
	}
	cli_resource_check(cib_conn, rsc);
	}

	static void
	cli_resource_why_without_rsc_with_host(cib_t cib_conn,GListPtr resources,node_t node)
	{
	const char* host_uname = node->details->uname;
	GListPtr allResources = node->details->allocated_rsc;
	GListPtr activeResources = node->details->running_rsc;
	GListPtr unactiveResources = subtract_lists(allResources,activeResources);
	GListPtr lpc = NULL;

	for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
	resource_t rsc = (resource_t ) lpc->data;
	printf("Resource %s is running on host %s\n",rsc->id,host_uname);
	cli_resource_check(cib_conn,rsc);
	}

	for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
	resource_t rsc = (resource_t ) lpc->data;
	printf("Resource %s is assigned to host %s but not running\n",
	rsc->id, host_uname);
	cli_resource_check(cib_conn,rsc);
	}

	g_list_free(allResources);
	g_list_free(activeResources);
	g_list_free(unactiveResources);
	}

	static void
	cli_resource_why_with_rsc_without_host(cib_t *cib_conn, GListPtr resources,
	resource_t *rsc)
	{
	GListPtr hosts = NULL;

	rsc->fns->location(rsc, &hosts, TRUE);
	printf("Resource %s is %srunning\n", rsc->id, (hosts? "" : "not "));
	cli_resource_check(cib_conn, rsc);
	g_list_free(hosts);
	}

	void cli_resource_why(cib_t cib_conn, GListPtr resources, resource_t rsc,
	node_t *node)
	{
	const char *host_uname = (node == NULL)? NULL : node->details->uname;

	if ((rsc == NULL) && (host_uname == NULL)) {
	cli_resource_why_without_rsc_and_host(cib_conn, resources);

	} else if ((rsc != NULL) && (host_uname != NULL)) {
	cli_resource_why_with_rsc_and_host(cib_conn, resources, rsc,
	host_uname);

	} else if ((rsc == NULL) && (host_uname != NULL)) {
	cli_resource_why_without_rsc_with_host(cib_conn, resources, node);

	} else if ((rsc != NULL) && (host_uname == NULL)) {
	cli_resource_why_with_rsc_without_host(cib_conn, resources, rsc);
	}
	}
	diff --git a/tools/crm_standby.in b/tools/crm_standby.in
	index b8222aede6..220c1e7f94 100755
	--- a/tools/crm_standby.in
	+++ b/tools/crm_standby.in
	@@ -1,150 +1,150 @@
	#!@BASH_PATH@

	USAGE_TEXT="Usage: crm_standby <command> [options]

	Common options:
	--help Display this text, then exit
	--version Display version information, then exit
	-V, --verbose Specify multiple times to increase debug output
	-q, --quiet Print only the standby status (if querying)

	Commands:
	-G, --query Query the current value of standby mode (on/off)
	-v, --update=VALUE Update the value of standby mode (on/off)
	-D, --delete Let standby mode use default value

	Additional Options:
	-N, --node=NODE Operate on the named node instead of the current one
	-l, --lifetime=VALUE Until when should the setting take effect
	(valid values: reboot, forever)
	-i, --id=VALUE (Advanced) XML ID used to identify standby attribute"

	HELP_TEXT="crm_standby - Query, enable, or disable standby mode for a node

	Nodes in standby mode may not host cluster resources.

	$USAGE_TEXT
	"

	exit_usage() {
	if [ $# -gt 0 ]; then
	echo "error: $@" >&2
	fi
	echo
	echo "$USAGE_TEXT"
	exit 1
	}

	op=""
	options=""
	lifetime=0
	target=""

	SHORTOPTS_DEPRECATED="U:Q"
	LONGOPTS_DEPRECATED="uname:,get-value,delete-attr,attr-value:,attr-id:"
	SHORTOPTS="VqGv:DN:l:i:"
	LONGOPTS="help,version,verbose,quiet,query,update:,delete,node:,lifetime:,id:"

	TEMP=$(@GETOPT_PATH@ -o ${SHORTOPTS}${SHORTOPTS_DEPRECATED} \
	--long ${LONGOPTS},${LONGOPTS_DEPRECATED} \
	-n crm_standby -- "$@")
	if [ $? -ne 0 ]; then
	exit_usage
	fi

	eval set -- "$TEMP" # Quotes around $TEMP are essential

	while true ; do
	case "$1" in
	--help)
	echo "$HELP_TEXT"
	exit 0
	;;
	--version)
	crm_attribute --version
	exit 0
	;;
	-q\|--quiet\|-V\|--verbose\|-Q)
	options="$options $1"
	shift
	;;
	-N\|--node\|-U\|--uname)
	target="$2"
	shift
	shift
	;;
	-G\|--query\|--get-value)
	options="$options --query"
	op=g
	shift
	;;
	-v\|--update\|--attr-value)
	options="$options --update $2"
	op=u
	shift
	shift
	;;
	-D\|--delete\|--delete-attr)
	options="$options --delete"
	op=d
	shift
	;;
	-l\|--lifetime)
	options="$options --lifetime $2"
	lifetime=1
	shift
	shift
	;;
	-i\|--id\|--attr-id)
	options="$options --id $2"
	shift
	shift
	;;
	--)
	shift
	break
	;;
	*)
	exit_usage "unknown option '$1'"
	;;
	esac
	done

	# It's important to call cluster commands only after arguments are processed,
	# so --version and --help work without problems even if those commands don't.
	if [ "$target" = "" ]; then
	target=$(crm_node -n)
	fi

	options="-N $target -n standby $options"
	if [ x$op = x ]; then
	options="$options -G"; op=g
	fi

	# If the user didn't explicitly specify a lifetime ...
	if [ $lifetime -eq 0 ]; then
	case $op in
	g)
	# For query, report the forever entry if one exists, otherwise
	# report the reboot entry if one exists, otherwise report off.
	- crm_attribute $options -l forever 2>&1 > /dev/null
	+ crm_attribute $options -l forever >/dev/null 2>&1
	if [ $? -eq 0 ]; then
	options="$options -l forever"
	else
	options="$options -l reboot -d off"
	fi
	;;
	u)
	# For update, default to updating the forever entry.
	options="$options -l forever"
	;;
	d)
	# For delete, default to deleting both forever and reboot entries.
	crm_attribute $options -l forever
	crm_attribute $options -l reboot
	exit 0
	;;
	esac
	fi

	crm_attribute $options

File Metadata

Mime Type: text/x-diff
Expires: Sat, Nov 23, 12:39 PM (21 h, 30 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1018728
Default Alt Text: (561 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions