diff --git a/lib/pengine/container.c b/lib/pengine/container.c
index cf81253d67..6e98e6ffd3 100644
--- a/lib/pengine/container.c
+++ b/lib/pengine/container.c
@@ -1,1087 +1,1097 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <crm/pengine/rules.h>
#include <crm/pengine/status.h>
#include <crm/pengine/internal.h>
#include <unpack.h>
#include <crm/msg_xml.h>
#define VARIANT_CONTAINER 1
#include "./variant.h"
void tuple_free(container_grouping_t *tuple);
static char *
next_ip(const char *last_ip)
{
unsigned int oct1 = 0;
unsigned int oct2 = 0;
unsigned int oct3 = 0;
unsigned int oct4 = 0;
int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
if (rc != 4) {
/*@ TODO check for IPv6 */
return NULL;
} else if (oct3 > 253) {
return NULL;
} else if (oct4 > 253) {
++oct3;
oct4 = 1;
} else {
++oct4;
}
return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
}
static int
allocate_ip(container_variant_data_t *data, container_grouping_t *tuple, char *buffer, int max)
{
if(data->ip_range_start == NULL) {
return 0;
} else if(data->ip_last) {
tuple->ipaddr = next_ip(data->ip_last);
} else {
tuple->ipaddr = strdup(data->ip_range_start);
}
data->ip_last = tuple->ipaddr;
#if 0
return snprintf(buffer, max, " --add-host=%s-%d:%s --link %s-docker-%d:%s-link-%d",
data->prefix, tuple->offset, tuple->ipaddr,
data->prefix, tuple->offset, data->prefix, tuple->offset);
#else
return snprintf(buffer, max, " --add-host=%s-%d:%s",
data->prefix, tuple->offset, tuple->ipaddr);
#endif
}
static xmlNode *
create_resource(const char *name, const char *provider, const char *kind)
{
xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
crm_xml_add(rsc, XML_ATTR_ID, name);
crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, "ocf");
crm_xml_add(rsc, XML_AGENT_ATTR_PROVIDER, provider);
crm_xml_add(rsc, XML_ATTR_TYPE, kind);
return rsc;
}
static void
create_nvp(xmlNode *parent, const char *name, const char *value)
{
xmlNode *xml_nvp = create_xml_node(parent, XML_CIB_TAG_NVPAIR);
crm_xml_set_id(xml_nvp, "%s-%s", ID(parent), name);
crm_xml_add(xml_nvp, XML_NVPAIR_ATTR_NAME, name);
crm_xml_add(xml_nvp, XML_NVPAIR_ATTR_VALUE, value);
}
static void
create_op(xmlNode *parent, const char *prefix, const char *task, const char *interval)
{
xmlNode *xml_op = create_xml_node(parent, "op");
crm_xml_set_id(xml_op, "%s-%s-%s", prefix, task, interval);
crm_xml_add(xml_op, XML_LRM_ATTR_INTERVAL, interval);
crm_xml_add(xml_op, "name", task);
}
/*!
* \internal
* \brief Check whether cluster can manage resource inside container
*
* \param[in] data Container variant data
*
* \return TRUE if networking configuration is acceptable, FALSE otherwise
*
* \note The resource is manageable if an IP range or control port has been
* specified. If a control port is used without an IP range, replicas per
* host must be 1.
*/
static bool
valid_network(container_variant_data_t *data)
{
if(data->ip_range_start) {
return TRUE;
}
if(data->control_port) {
if(data->replicas_per_host > 1) {
pe_err("Specifying the 'control-port' for %s requires 'replicas-per-host=1'", data->prefix);
data->replicas_per_host = 1;
/* @TODO to be sure: clear_bit(rsc->flags, pe_rsc_unique); */
}
return TRUE;
}
return FALSE;
}
static bool
create_ip_resource(
resource_t *parent, container_variant_data_t *data, container_grouping_t *tuple,
pe_working_set_t * data_set)
{
if(data->ip_range_start) {
char *id = NULL;
xmlNode *xml_ip = NULL;
xmlNode *xml_obj = NULL;
id = crm_strdup_printf("%s-ip-%s", data->prefix, tuple->ipaddr);
crm_xml_sanitize_id(id);
xml_ip = create_resource(id, "heartbeat", "IPaddr2");
free(id);
xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, tuple->offset);
create_nvp(xml_obj, "ip", tuple->ipaddr);
if(data->host_network) {
create_nvp(xml_obj, "nic", data->host_network);
}
if(data->host_netmask) {
create_nvp(xml_obj, "cidr_netmask", data->host_netmask);
} else {
create_nvp(xml_obj, "cidr_netmask", "32");
}
xml_obj = create_xml_node(xml_ip, "operations");
create_op(xml_obj, ID(xml_ip), "monitor", "60s");
// TODO: Other ops? Timeouts and intervals from underlying resource?
if (common_unpack(xml_ip, &tuple->ip, parent, data_set) == false) {
return FALSE;
}
parent->children = g_list_append(parent->children, tuple->ip);
}
return TRUE;
}
static bool
create_docker_resource(
resource_t *parent, container_variant_data_t *data, container_grouping_t *tuple,
pe_working_set_t * data_set)
{
int offset = 0, max = 4096;
char *buffer = calloc(1, max+1);
int doffset = 0, dmax = 1024;
char *dbuffer = calloc(1, dmax+1);
char *id = NULL;
xmlNode *xml_docker = NULL;
xmlNode *xml_obj = NULL;
id = crm_strdup_printf("%s-docker-%d", data->prefix, tuple->offset);
crm_xml_sanitize_id(id);
xml_docker = create_resource(id, "heartbeat", "docker");
free(id);
xml_obj = create_xml_node(xml_docker, XML_TAG_ATTR_SETS);
crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, tuple->offset);
create_nvp(xml_obj, "image", data->image);
create_nvp(xml_obj, "allow_pull", "true");
create_nvp(xml_obj, "force_kill", "false");
create_nvp(xml_obj, "reuse", "false");
offset += snprintf(buffer+offset, max-offset, " --restart=no");
/* Set a container hostname only if we have an IP to map it to.
* The user can set -h or --uts=host themselves if they want a nicer
* name for logs, but this makes applications happy who need their
* hostname to match the IP they bind to.
*/
if (data->ip_range_start != NULL) {
offset += snprintf(buffer+offset, max-offset, " -h %s-%d",
data->prefix, tuple->offset);
}
if(data->docker_network) {
// offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s", tuple->ipaddr);
offset += snprintf(buffer+offset, max-offset, " --net=%s", data->docker_network);
}
if(data->control_port) {
offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%s", data->control_port);
} else {
offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
}
for(GListPtr pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
container_mount_t *mount = pIter->data;
if(mount->flags) {
char *source = crm_strdup_printf(
"%s/%s-%d", mount->source, data->prefix, tuple->offset);
if(doffset > 0) {
doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
}
doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
offset += snprintf(buffer+offset, max-offset, " -v %s:%s", source, mount->target);
free(source);
} else {
offset += snprintf(buffer+offset, max-offset, " -v %s:%s", mount->source, mount->target);
}
if(mount->options) {
offset += snprintf(buffer+offset, max-offset, ":%s", mount->options);
}
}
for(GListPtr pIter = data->ports; pIter != NULL; pIter = pIter->next) {
container_port_t *port = pIter->data;
if(tuple->ipaddr) {
offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s",
tuple->ipaddr, port->source, port->target);
} else {
offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target);
}
}
if(data->docker_run_options) {
offset += snprintf(buffer+offset, max-offset, " %s", data->docker_run_options);
}
if(data->docker_host_options) {
offset += snprintf(buffer+offset, max-offset, " %s", data->docker_host_options);
}
create_nvp(xml_obj, "run_opts", buffer);
free(buffer);
create_nvp(xml_obj, "mount_points", dbuffer);
free(dbuffer);
if(tuple->child) {
if(data->docker_run_command) {
create_nvp(xml_obj, "run_cmd", data->docker_run_command);
} else {
create_nvp(xml_obj, "run_cmd", SBIN_DIR"/pacemaker_remoted");
}
/* TODO: Allow users to specify their own?
*
* We just want to know if the container is alive, we'll
* monitor the child independently
*/
create_nvp(xml_obj, "monitor_cmd", "/bin/true");
/* } else if(child && data->untrusted) {
* Support this use-case?
*
* The ability to have resources started/stopped by us, but
* unable to set attributes, etc.
*
* Arguably better to control API access this with ACLs like
* "normal" remote nodes
*
* create_nvp(xml_obj, "run_cmd", "/usr/libexec/pacemaker/lrmd");
* create_nvp(xml_obj, "monitor_cmd", "/usr/libexec/pacemaker/lrmd_internal_ctl -c poke");
*/
} else {
if(data->docker_run_command) {
create_nvp(xml_obj, "run_cmd", data->docker_run_command);
}
/* TODO: Allow users to specify their own?
*
* We don't know what's in the container, so we just want
* to know if it is alive
*/
create_nvp(xml_obj, "monitor_cmd", "/bin/true");
}
xml_obj = create_xml_node(xml_docker, "operations");
create_op(xml_obj, ID(xml_docker), "monitor", "60s");
// TODO: Other ops? Timeouts and intervals from underlying resource?
if (common_unpack(xml_docker, &tuple->docker, parent, data_set) == FALSE) {
return FALSE;
}
parent->children = g_list_append(parent->children, tuple->docker);
return TRUE;
}
/*!
* \brief Ban a node from a resource's (and its children's) allowed nodes list
*
* \param[in,out] rsc Resource to modify
* \param[in] uname Name of node to ban
*/
static void
disallow_node(resource_t *rsc, const char *uname)
{
gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);
if (match) {
((pe_node_t *) match)->weight = -INFINITY;
}
if (rsc->children) {
GListPtr child;
for (child = rsc->children; child != NULL; child = child->next) {
disallow_node((resource_t *) (child->data), uname);
}
}
}
static bool
create_remote_resource(
resource_t *parent, container_variant_data_t *data, container_grouping_t *tuple,
pe_working_set_t * data_set)
{
if (tuple->child && valid_network(data)) {
GHashTableIter gIter;
GListPtr rsc_iter = NULL;
node_t *node = NULL;
xmlNode *xml_obj = NULL;
xmlNode *xml_remote = NULL;
char *id = crm_strdup_printf("%s-%d", data->prefix, tuple->offset);
const char *uname = NULL;
if (remote_id_conflict(id, data_set)) {
free(id);
// The biggest hammer we have
id = crm_strdup_printf("pcmk-internal-%s-remote-%d", tuple->child->id, tuple->offset);
CRM_ASSERT(remote_id_conflict(id, data_set) == FALSE);
}
xml_remote = create_resource(id, "pacemaker", "remote");
/* Abandon our created ID, and pull the copy from the XML, because we
* need something that will get freed during data set cleanup to use as
* the node ID and uname.
*/
free(id);
id = NULL;
uname = ID(xml_remote);
xml_obj = create_xml_node(xml_remote, "operations");
create_op(xml_obj, uname, "monitor", "60s");
xml_obj = create_xml_node(xml_remote, XML_TAG_ATTR_SETS);
crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, tuple->offset);
if(tuple->ipaddr) {
create_nvp(xml_obj, "addr", tuple->ipaddr);
} else {
// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
create_nvp(xml_obj, "addr", "#uname");
}
if(data->control_port) {
create_nvp(xml_obj, "port", data->control_port);
} else {
char *port_s = crm_itoa(DEFAULT_REMOTE_PORT);
create_nvp(xml_obj, "port", port_s);
free(port_s);
}
xml_obj = create_xml_node(xml_remote, XML_TAG_META_SETS);
crm_xml_set_id(xml_obj, "%s-meta-%d", data->prefix, tuple->offset);
create_nvp(xml_obj, XML_OP_ATTR_ALLOW_MIGRATE, "false");
/* This sets tuple->docker as tuple->remote's container, which is
* similar to what happens with guest nodes. This is how the PE knows
* that the bundle node is fenced by recovering docker, and that
* remote should be ordered relative to docker.
*/
create_nvp(xml_obj, XML_RSC_ATTR_CONTAINER, tuple->docker->id);
/* Ensure a node has been created for the guest (it may have already
* been, if it has a permanent node attribute), and ensure its weight is
* -INFINITY so no other resources can run on it.
*/
node = pe_find_node(data_set->nodes, uname);
if (node == NULL) {
node = pe_create_node(uname, uname, "remote", "-INFINITY",
data_set);
} else {
node->weight = -INFINITY;
}
/* unpack_remote_nodes() ensures that each remote node and guest node
* has a pe_node_t entry. Ideally, it would do the same for bundle nodes.
* Unfortunately, a bundle has to be mostly unpacked before it's obvious
* what nodes will be needed, so we do it just above.
*
* Worse, that means that the node may have been utilized while
* unpacking other resources, without our weight correction. The most
* likely place for this to happen is when common_unpack() calls
* resource_location() to set a default score in symmetric clusters.
* This adds a node *copy* to each resource's allowed nodes, and these
* copies will have the wrong weight.
*
* As a hacky workaround, fix those copies here.
*
* @TODO Possible alternative: ensure bundles are unpacked before other
* resources, so the weight is correct before any copies are made.
*/
for (rsc_iter = data_set->resources; rsc_iter; rsc_iter = rsc_iter->next) {
disallow_node((resource_t *) (rsc_iter->data), uname);
}
tuple->node = node_copy(node);
tuple->node->weight = 500;
if (common_unpack(xml_remote, &tuple->remote, parent, data_set) == FALSE) {
return FALSE;
}
g_hash_table_iter_init(&gIter, tuple->remote->allowed_nodes);
while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
if(is_remote_node(node)) {
/* Remote resources can only run on 'normal' cluster node */
node->weight = -INFINITY;
}
}
tuple->node->details->remote_rsc = tuple->remote;
/* #kind is irrelevant to bundles since it is only used in location
* constraint rules, and those don't matter for resources inside
* bundles. But just for clarity, a bundle is closer to "container"
* (guest node) than the "remote" set by pe_create_node().
*/
g_hash_table_insert(tuple->node->details->attrs,
strdup("#kind"), strdup("container"));
/* One effect of this is that setup_container() will add
* tuple->remote to tuple->docker's fillers, which will make
* rsc_contains_remote_node() true for tuple->docker.
*
* tuple->child does NOT get added to tuple->docker's fillers.
* The only noticeable effect if it did would be for its fail count to
* be taken into account when checking tuple->docker's migration
* threshold.
*/
parent->children = g_list_append(parent->children, tuple->remote);
}
return TRUE;
}
static bool
create_container(
resource_t *parent, container_variant_data_t *data, container_grouping_t *tuple,
pe_working_set_t * data_set)
{
if(create_docker_resource(parent, data, tuple, data_set) == FALSE) {
return TRUE;
}
if(create_ip_resource(parent, data, tuple, data_set) == FALSE) {
return TRUE;
}
if(create_remote_resource(parent, data, tuple, data_set) == FALSE) {
return TRUE;
}
if(tuple->child && tuple->ipaddr) {
add_hash_param(tuple->child->meta, "external-ip", tuple->ipaddr);
}
if(tuple->remote) {
/*
* Allow the remote connection resource to be allocated to a
* different node than the one on which the docker container
* is active.
*
* Makes it possible to have remote nodes, running docker
* containers with pacemaker_remoted inside in order to start
* services inside those containers.
*/
set_bit(tuple->remote->flags, pe_rsc_allow_remote_remotes);
}
return FALSE;
}
static void mount_free(container_mount_t *mount)
{
free(mount->source);
free(mount->target);
free(mount->options);
free(mount);
}
static void port_free(container_port_t *port)
{
free(port->source);
free(port->target);
free(port);
}
gboolean
container_unpack(resource_t * rsc, pe_working_set_t * data_set)
{
const char *value = NULL;
xmlNode *xml_obj = NULL;
xmlNode *xml_resource = NULL;
container_variant_data_t *container_data = NULL;
CRM_ASSERT(rsc != NULL);
pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);
container_data = calloc(1, sizeof(container_variant_data_t));
rsc->variant_opaque = container_data;
container_data->prefix = strdup(rsc->id);
xml_obj = first_named_child(rsc->xml, "docker");
if(xml_obj == NULL) {
return FALSE;
}
value = crm_element_value(xml_obj, "masters");
container_data->masters = crm_parse_int(value, "0");
if (container_data->masters < 0) {
pe_err("'masters' for %s must be nonnegative integer, using 0",
rsc->id);
container_data->masters = 0;
}
value = crm_element_value(xml_obj, "replicas");
if ((value == NULL) && (container_data->masters > 0)) {
container_data->replicas = container_data->masters;
} else {
container_data->replicas = crm_parse_int(value, "1");
}
if (container_data->replicas < 1) {
pe_err("'replicas' for %s must be positive integer, using 1", rsc->id);
container_data->replicas = 1;
}
/*
* Communication between containers on the same host via the
* floating IPs only works if docker is started with:
* --userland-proxy=false --ip-masq=false
*/
value = crm_element_value(xml_obj, "replicas-per-host");
container_data->replicas_per_host = crm_parse_int(value, "1");
if (container_data->replicas_per_host < 1) {
pe_err("'replicas-per-host' for %s must be positive integer, using 1",
rsc->id);
container_data->replicas_per_host = 1;
}
if (container_data->replicas_per_host == 1) {
clear_bit(rsc->flags, pe_rsc_unique);
}
container_data->docker_run_command = crm_element_value_copy(xml_obj, "run-command");
container_data->docker_run_options = crm_element_value_copy(xml_obj, "options");
container_data->image = crm_element_value_copy(xml_obj, "image");
container_data->docker_network = crm_element_value_copy(xml_obj, "network");
xml_obj = first_named_child(rsc->xml, "network");
if(xml_obj) {
container_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
container_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
container_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
container_data->control_port = crm_element_value_copy(xml_obj, "control-port");
for (xmlNode *xml_child = __xml_first_child_element(xml_obj); xml_child != NULL;
xml_child = __xml_next_element(xml_child)) {
container_port_t *port = calloc(1, sizeof(container_port_t));
port->source = crm_element_value_copy(xml_child, "port");
if(port->source == NULL) {
port->source = crm_element_value_copy(xml_child, "range");
} else {
port->target = crm_element_value_copy(xml_child, "internal-port");
}
if(port->source != NULL && strlen(port->source) > 0) {
if(port->target == NULL) {
port->target = strdup(port->source);
}
container_data->ports = g_list_append(container_data->ports, port);
} else {
pe_err("Invalid port directive %s", ID(xml_child));
port_free(port);
}
}
}
xml_obj = first_named_child(rsc->xml, "storage");
for (xmlNode *xml_child = __xml_first_child_element(xml_obj); xml_child != NULL;
xml_child = __xml_next_element(xml_child)) {
container_mount_t *mount = calloc(1, sizeof(container_mount_t));
mount->source = crm_element_value_copy(xml_child, "source-dir");
if(mount->source == NULL) {
mount->source = crm_element_value_copy(xml_child, "source-dir-root");
mount->flags = 1;
}
mount->target = crm_element_value_copy(xml_child, "target-dir");
mount->options = crm_element_value_copy(xml_child, "options");
if(mount->source && mount->target) {
container_data->mounts = g_list_append(container_data->mounts, mount);
} else {
pe_err("Invalid mount directive %s", ID(xml_child));
mount_free(mount);
}
}
xml_obj = first_named_child(rsc->xml, "primitive");
if (xml_obj && valid_network(container_data)) {
char *value = NULL;
xmlNode *xml_set = NULL;
if(container_data->masters > 0) {
xml_resource = create_xml_node(NULL, XML_CIB_TAG_MASTER);
} else {
xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);
}
crm_xml_set_id(xml_resource, "%s-%s", container_data->prefix, xml_resource->name);
xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
crm_xml_set_id(xml_set, "%s-%s-meta", container_data->prefix, xml_resource->name);
create_nvp(xml_set, XML_RSC_ATTR_ORDERED, "true");
value = crm_itoa(container_data->replicas);
create_nvp(xml_set, XML_RSC_ATTR_INCARNATION_MAX, value);
free(value);
value = crm_itoa(container_data->replicas_per_host);
create_nvp(xml_set, XML_RSC_ATTR_INCARNATION_NODEMAX, value);
free(value);
if(container_data->replicas_per_host > 1) {
create_nvp(xml_set, XML_RSC_ATTR_UNIQUE, "true");
} else {
create_nvp(xml_set, XML_RSC_ATTR_UNIQUE, "false");
}
if(container_data->masters) {
value = crm_itoa(container_data->masters);
create_nvp(xml_set, XML_RSC_ATTR_MASTER_MAX, value);
free(value);
}
//crm_xml_add(xml_obj, XML_ATTR_ID, container_data->prefix);
add_node_copy(xml_resource, xml_obj);
} else if(xml_obj) {
pe_err("Cannot control %s inside %s without either ip-range-start or control-port",
rsc->id, ID(xml_obj));
return FALSE;
}
if(xml_resource) {
int lpc = 0;
GListPtr childIter = NULL;
resource_t *new_rsc = NULL;
container_mount_t *mount = NULL;
container_port_t *port = NULL;
int offset = 0, max = 1024;
char *buffer = NULL;
if (common_unpack(xml_resource, &new_rsc, rsc, data_set) == FALSE) {
pe_err("Failed unpacking resource %s", ID(rsc->xml));
if (new_rsc != NULL && new_rsc->fns != NULL) {
new_rsc->fns->free(new_rsc);
}
return FALSE;
}
container_data->child = new_rsc;
- container_data->child->orig_xml = xml_obj; // Also the trigger for common_free()
- // to free xml_resource as container_data->child->xml
mount = calloc(1, sizeof(container_mount_t));
mount->source = strdup(DEFAULT_REMOTE_KEY_LOCATION);
mount->target = strdup(DEFAULT_REMOTE_KEY_LOCATION);
mount->options = NULL;
mount->flags = 0;
container_data->mounts = g_list_append(container_data->mounts, mount);
mount = calloc(1, sizeof(container_mount_t));
mount->source = strdup(CRM_LOG_DIR "/bundles");
mount->target = strdup("/var/log");
mount->options = NULL;
mount->flags = 1;
container_data->mounts = g_list_append(container_data->mounts, mount);
port = calloc(1, sizeof(container_port_t));
if(container_data->control_port) {
port->source = strdup(container_data->control_port);
} else {
port->source = crm_itoa(DEFAULT_REMOTE_PORT);
}
port->target = strdup(port->source);
container_data->ports = g_list_append(container_data->ports, port);
buffer = calloc(1, max+1);
for(childIter = container_data->child->children; childIter != NULL; childIter = childIter->next) {
container_grouping_t *tuple = calloc(1, sizeof(container_grouping_t));
tuple->child = childIter->data;
tuple->offset = lpc++;
offset += allocate_ip(container_data, tuple, buffer+offset, max-offset);
container_data->tuples = g_list_append(container_data->tuples, tuple);
}
container_data->docker_host_options = buffer;
} else {
// Just a naked container, no pacemaker-remote
int offset = 0, max = 1024;
char *buffer = calloc(1, max+1);
for(int lpc = 0; lpc < container_data->replicas; lpc++) {
container_grouping_t *tuple = calloc(1, sizeof(container_grouping_t));
tuple->offset = lpc;
offset += allocate_ip(container_data, tuple, buffer+offset, max-offset);
container_data->tuples = g_list_append(container_data->tuples, tuple);
}
container_data->docker_host_options = buffer;
}
for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
container_grouping_t *tuple = (container_grouping_t *)gIter->data;
// TODO: Remove from list if create_container() returns TRUE
create_container(rsc, container_data, tuple, data_set);
}
if(container_data->child) {
rsc->children = g_list_append(rsc->children, container_data->child);
}
return TRUE;
}
static int
tuple_rsc_active(resource_t *rsc, gboolean all)
{
if (rsc) {
gboolean child_active = rsc->fns->active(rsc, all);
if (child_active && !all) {
return TRUE;
} else if (!child_active && all) {
return FALSE;
}
}
return -1;
}
gboolean
container_active(resource_t * rsc, gboolean all)
{
container_variant_data_t *container_data = NULL;
GListPtr iter = NULL;
get_container_variant_data(container_data, rsc);
for (iter = container_data->tuples; iter != NULL; iter = iter->next) {
container_grouping_t *tuple = (container_grouping_t *)(iter->data);
int rsc_active;
rsc_active = tuple_rsc_active(tuple->ip, all);
if (rsc_active >= 0) {
return (gboolean) rsc_active;
}
rsc_active = tuple_rsc_active(tuple->child, all);
if (rsc_active >= 0) {
return (gboolean) rsc_active;
}
rsc_active = tuple_rsc_active(tuple->docker, all);
if (rsc_active >= 0) {
return (gboolean) rsc_active;
}
rsc_active = tuple_rsc_active(tuple->remote, all);
if (rsc_active >= 0) {
return (gboolean) rsc_active;
}
}
/* If "all" is TRUE, we've already checked that no resources were inactive,
* so return TRUE; if "all" is FALSE, we didn't find any active resources,
* so return FALSE.
*/
return all;
}
resource_t *
find_container_child(const char *stem, resource_t * rsc, node_t *node)
{
container_variant_data_t *container_data = NULL;
resource_t *parent = uber_parent(rsc);
CRM_ASSERT(parent->parent);
parent = parent->parent;
get_container_variant_data(container_data, parent);
if (is_not_set(rsc->flags, pe_rsc_unique)) {
for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
container_grouping_t *tuple = (container_grouping_t *)gIter->data;
CRM_ASSERT(tuple);
if(tuple->node->details == node->details) {
rsc = tuple->child;
break;
}
}
}
if (rsc && safe_str_neq(stem, rsc->id)) {
free(rsc->clone_name);
rsc->clone_name = strdup(stem);
}
return rsc;
}
static void
print_rsc_in_list(resource_t *rsc, const char *pre_text, long options,
void *print_data)
{
if (rsc != NULL) {
if (options & pe_print_html) {
status_print("<li>");
}
rsc->fns->print(rsc, pre_text, options, print_data);
if (options & pe_print_html) {
status_print("</li>\n");
}
}
}
static void
container_print_xml(resource_t * rsc, const char *pre_text, long options, void *print_data)
{
container_variant_data_t *container_data = NULL;
char *child_text = NULL;
CRM_CHECK(rsc != NULL, return);
if (pre_text == NULL) {
pre_text = "";
}
child_text = crm_concat(pre_text, " ", ' ');
get_container_variant_data(container_data, rsc);
status_print("%s<bundle ", pre_text);
status_print("id=\"%s\" ", rsc->id);
status_print("type=\"docker\" ");
status_print("image=\"%s\" ", container_data->image);
status_print("unique=\"%s\" ", is_set(rsc->flags, pe_rsc_unique)? "true" : "false");
status_print("managed=\"%s\" ", is_set(rsc->flags, pe_rsc_managed) ? "true" : "false");
status_print("failed=\"%s\" ", is_set(rsc->flags, pe_rsc_failed) ? "true" : "false");
status_print(">\n");
for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
container_grouping_t *tuple = (container_grouping_t *)gIter->data;
CRM_ASSERT(tuple);
status_print("%s <replica id=\"%d\">\n", pre_text, tuple->offset);
print_rsc_in_list(tuple->ip, child_text, options, print_data);
print_rsc_in_list(tuple->child, child_text, options, print_data);
print_rsc_in_list(tuple->docker, child_text, options, print_data);
print_rsc_in_list(tuple->remote, child_text, options, print_data);
status_print("%s </replica>\n", pre_text);
}
status_print("%s</bundle>\n", pre_text);
free(child_text);
}
static void
tuple_print(container_grouping_t * tuple, const char *pre_text, long options, void *print_data)
{
node_t *node = NULL;
resource_t *rsc = tuple->child;
int offset = 0;
char buffer[LINE_MAX];
if(rsc == NULL) {
rsc = tuple->docker;
}
if(tuple->remote) {
offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", rsc_printable_id(tuple->remote));
} else {
offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", rsc_printable_id(tuple->docker));
}
if(tuple->ipaddr) {
offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)", tuple->ipaddr);
}
if(tuple->docker && tuple->docker->running_on != NULL) {
node = tuple->docker->running_on->data;
} else if (tuple->docker == NULL && rsc->running_on != NULL) {
node = rsc->running_on->data;
}
common_print(rsc, pre_text, buffer, node, options, print_data);
}
void
container_print(resource_t * rsc, const char *pre_text, long options, void *print_data)
{
container_variant_data_t *container_data = NULL;
char *child_text = NULL;
CRM_CHECK(rsc != NULL, return);
if (options & pe_print_xml) {
container_print_xml(rsc, pre_text, options, print_data);
return;
}
get_container_variant_data(container_data, rsc);
if (pre_text == NULL) {
pre_text = " ";
}
status_print("%sDocker container%s: %s [%s]%s%s\n",
pre_text, container_data->replicas>1?" set":"", rsc->id, container_data->image,
is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
if (options & pe_print_html) {
status_print("<br />\n<ul>\n");
}
for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
container_grouping_t *tuple = (container_grouping_t *)gIter->data;
CRM_ASSERT(tuple);
if (options & pe_print_html) {
status_print("<li>");
}
if(is_set(options, pe_print_clone_details)) {
child_text = crm_strdup_printf(" %s", pre_text);
if(g_list_length(container_data->tuples) > 1) {
status_print(" %sReplica[%d]\n", pre_text, tuple->offset);
}
if (options & pe_print_html) {
status_print("<br />\n<ul>\n");
}
print_rsc_in_list(tuple->ip, child_text, options, print_data);
print_rsc_in_list(tuple->docker, child_text, options, print_data);
print_rsc_in_list(tuple->remote, child_text, options, print_data);
print_rsc_in_list(tuple->child, child_text, options, print_data);
if (options & pe_print_html) {
status_print("</ul>\n");
}
} else {
child_text = crm_strdup_printf("%s ", pre_text);
tuple_print(tuple, child_text, options, print_data);
}
free(child_text);
if (options & pe_print_html) {
status_print("</li>\n");
}
}
if (options & pe_print_html) {
status_print("</ul>\n");
}
}
void
tuple_free(container_grouping_t *tuple)
{
if(tuple == NULL) {
return;
}
// TODO: Free tuple->node ?
if(tuple->ip) {
tuple->ip->fns->free(tuple->ip);
+ tuple->ip->xml = NULL;
+ free_xml(tuple->ip->xml);
tuple->ip = NULL;
}
if(tuple->child) {
+ free_xml(tuple->child->xml);
+ tuple->child->xml = NULL;
tuple->child->fns->free(tuple->child);
tuple->child = NULL;
}
if(tuple->docker) {
+ free_xml(tuple->docker->xml);
+ tuple->docker->xml = NULL;
tuple->docker->fns->free(tuple->docker);
tuple->docker = NULL;
}
if(tuple->remote) {
+ free_xml(tuple->remote->xml);
+ tuple->remote->xml = NULL;
tuple->remote->fns->free(tuple->remote);
tuple->remote = NULL;
}
free(tuple->ipaddr);
free(tuple);
}
void
container_free(resource_t * rsc)
{
container_variant_data_t *container_data = NULL;
CRM_CHECK(rsc != NULL, return);
get_container_variant_data(container_data, rsc);
pe_rsc_trace(rsc, "Freeing %s", rsc->id);
free(container_data->prefix);
free(container_data->image);
free(container_data->control_port);
free(container_data->host_network);
free(container_data->host_netmask);
free(container_data->ip_range_start);
free(container_data->docker_network);
free(container_data->docker_run_options);
free(container_data->docker_run_command);
free(container_data->docker_host_options);
+ if(container_data->child) {
+ free_xml(container_data->child->xml);
+ }
g_list_free_full(container_data->tuples, (GDestroyNotify)tuple_free);
g_list_free_full(container_data->mounts, (GDestroyNotify)mount_free);
g_list_free_full(container_data->ports, (GDestroyNotify)port_free);
+ g_list_free(rsc->children);
common_free(rsc);
}
enum rsc_role_e
container_resource_state(const resource_t * rsc, gboolean current)
{
enum rsc_role_e container_role = RSC_ROLE_UNKNOWN;
return container_role;
}
diff --git a/pengine/regression.sh b/pengine/regression.sh
index e97b54b21c..b844bb98ab 100755
--- a/pengine/regression.sh
+++ b/pengine/regression.sh
@@ -1,866 +1,869 @@
#!/bin/bash
# Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
#
core=`dirname $0`
. $core/regression.core.sh || exit 1
create_mode="true"
info Generating test outputs for these tests...
# do_test file description
-
info Done.
echo ""
info Performing the following tests from $io_dir
create_mode="false"
echo ""
do_test simple1 "Offline "
do_test simple2 "Start "
do_test simple3 "Start 2 "
do_test simple4 "Start Failed"
do_test simple6 "Stop Start "
do_test simple7 "Shutdown "
#do_test simple8 "Stonith "
#do_test simple9 "Lower version"
#do_test simple10 "Higher version"
do_test simple11 "Priority (ne)"
do_test simple12 "Priority (eq)"
do_test simple8 "Stickiness"
echo ""
do_test group1 "Group "
do_test group2 "Group + Native "
do_test group3 "Group + Group "
do_test group4 "Group + Native (nothing)"
do_test group5 "Group + Native (move) "
do_test group6 "Group + Group (move) "
do_test group7 "Group colocation"
do_test group13 "Group colocation (cant run)"
do_test group8 "Group anti-colocation"
do_test group9 "Group recovery"
do_test group10 "Group partial recovery"
do_test group11 "Group target_role"
do_test group14 "Group stop (graph terminated)"
do_test group15 "-ve group colocation"
do_test bug-1573 "Partial stop of a group with two children"
do_test bug-1718 "Mandatory group ordering - Stop group_FUN"
do_test bug-lf-2613 "Move group on failure"
do_test bug-lf-2619 "Move group on clone failure"
do_test group-fail "Ensure stop order is preserved for partially active groups"
do_test group-unmanaged "No need to restart r115 because r114 is unmanaged"
do_test group-unmanaged-stopped "Make sure r115 is stopped when r114 fails"
do_test group-dependents "Account for the location preferences of things colocated with a group"
echo ""
do_test rsc_dep1 "Must not "
do_test rsc_dep3 "Must "
do_test rsc_dep5 "Must not 3 "
do_test rsc_dep7 "Must 3 "
do_test rsc_dep10 "Must (but cant)"
do_test rsc_dep2 "Must (running) "
do_test rsc_dep8 "Must (running : alt) "
do_test rsc_dep4 "Must (running + move)"
do_test asymmetric "Asymmetric - require explicit location constraints"
echo ""
do_test orphan-0 "Orphan ignore"
do_test orphan-1 "Orphan stop"
do_test orphan-2 "Orphan stop, remove failcount"
echo ""
do_test params-0 "Params: No change"
do_test params-1 "Params: Changed"
do_test params-2 "Params: Resource definition"
do_test params-4 "Params: Reload"
do_test params-5 "Params: Restart based on probe digest"
do_test novell-251689 "Resource definition change + target_role=stopped"
do_test bug-lf-2106 "Restart all anonymous clone instances after config change"
do_test params-6 "Params: Detect reload in previously migrated resource"
do_test nvpair-id-ref "Support id-ref in nvpair with optional name"
do_test not-reschedule-unneeded-monitor "Do not reschedule unneeded monitors while resource definitions have changed"
echo ""
do_test target-0 "Target Role : baseline"
do_test target-1 "Target Role : master"
do_test target-2 "Target Role : invalid"
echo ""
do_test base-score "Set a node's default score for all nodes"
echo ""
do_test date-1 "Dates" -t "2005-020"
do_test date-2 "Date Spec - Pass" -t "2005-020T12:30"
do_test date-3 "Date Spec - Fail" -t "2005-020T11:30"
do_test origin "Timing of recurring operations" -t "2014-05-07 00:28:00"
do_test probe-0 "Probe (anon clone)"
do_test probe-1 "Pending Probe"
do_test probe-2 "Correctly re-probe cloned groups"
do_test probe-3 "Probe (pending node)"
do_test probe-4 "Probe (pending node + stopped resource)"
do_test standby "Standby"
do_test comments "Comments"
echo ""
do_test one-or-more-0 "Everything starts"
do_test one-or-more-1 "Nothing starts because of A"
do_test one-or-more-2 "D can start because of C"
do_test one-or-more-3 "D cannot start because of B and C"
do_test one-or-more-4 "D cannot start because of target-role"
do_test one-or-more-5 "Start A and F even though C and D are stopped"
do_test one-or-more-6 "Leave A running even though B is stopped"
do_test one-or-more-7 "Leave A running even though C is stopped"
do_test bug-5140-require-all-false "Allow basegrp:0 to stop"
do_test clone-require-all-1 "clone B starts node 3 and 4"
do_test clone-require-all-2 "clone B remains stopped everywhere"
do_test clone-require-all-3 "clone B stops everywhere because A stops everywhere"
do_test clone-require-all-4 "clone B remains on node 3 and 4 with only one instance of A remaining."
do_test clone-require-all-5 "clone B starts on node 1 3 and 4"
do_test clone-require-all-6 "clone B remains active after shutting down instances of A"
do_test clone-require-all-7 "clone A and B both start at the same time. all instances of A start before B."
do_test clone-require-all-no-interleave-1 "C starts everywhere after A and B"
do_test clone-require-all-no-interleave-2 "C starts on nodes 1, 2, and 4 with only one active instance of B"
do_test clone-require-all-no-interleave-3 "C remains active when instance of B is stopped on one node and started on another."
do_test one-or-more-unrunnnable-instances "Avoid dependencies on instances that won't ever be started"
echo ""
do_test order1 "Order start 1 "
do_test order2 "Order start 2 "
do_test order3 "Order stop "
do_test order4 "Order (multiple) "
do_test order5 "Order (move) "
do_test order6 "Order (move w/ restart) "
do_test order7 "Order (mandatory) "
do_test order-optional "Order (score=0) "
do_test order-required "Order (score=INFINITY) "
do_test bug-lf-2171 "Prevent group start when clone is stopped"
do_test order-clone "Clone ordering should be able to prevent startup of dependent clones"
do_test order-sets "Ordering for resource sets"
do_test order-serialize "Serialize resources without inhibiting migration"
do_test order-serialize-set "Serialize a set of resources without inhibiting migration"
do_test clone-order-primitive "Order clone start after a primitive"
do_test clone-order-16instances "Verify ordering of 16 cloned resources"
do_test order-optional-keyword "Order (optional keyword)"
do_test order-mandatory "Order (mandatory keyword)"
do_test bug-lf-2493 "Don't imply colocation requirements when applying ordering constraints with clones"
do_test ordered-set-basic-startup "Constraint set with default order settings."
do_test ordered-set-natural "Allow natural set ordering"
do_test order-wrong-kind "Order (error)"
echo ""
do_test coloc-loop "Colocation - loop"
do_test coloc-many-one "Colocation - many-to-one"
do_test coloc-list "Colocation - many-to-one with list"
do_test coloc-group "Colocation - groups"
do_test coloc-slave-anti "Anti-colocation with slave shouldn't prevent master colocation"
do_test coloc-attr "Colocation based on node attributes"
do_test coloc-negative-group "Negative colocation with a group"
do_test coloc-intra-set "Intra-set colocation"
do_test bug-lf-2435 "Colocation sets with a negative score"
do_test coloc-clone-stays-active "Ensure clones don't get stopped/demoted because a dependent must stop"
do_test coloc_fp_logic "Verify floating point calculations in colocation are working"
do_test colo_master_w_native "cl#5070 - Verify promotion order is affected when colocating master to native rsc."
do_test colo_slave_w_native "cl#5070 - Verify promotion order is affected when colocating slave to native rsc."
do_test anti-colocation-order "cl#5187 - Prevent resources in an anti-colocation from even temporarily running on a same node"
do_test anti-colocation-master "Organize order of actions for master resources in anti-colocations"
do_test anti-colocation-slave "Organize order of actions for slave resources in anti-colocations"
do_test enforce-colo1 "Always enforce B with A INFINITY."
do_test complex_enforce_colo "Always enforce B with A INFINITY. (make sure heat-engine stops)"
echo ""
do_test rsc-sets-seq-true "Resource Sets - sequential=false"
do_test rsc-sets-seq-false "Resource Sets - sequential=true"
do_test rsc-sets-clone "Resource Sets - Clone"
do_test rsc-sets-master "Resource Sets - Master"
do_test rsc-sets-clone-1 "Resource Sets - Clone (lf#2404)"
#echo ""
#do_test agent1 "version: lt (empty)"
#do_test agent2 "version: eq "
#do_test agent3 "version: gt "
echo ""
do_test attrs1 "string: eq (and) "
do_test attrs2 "string: lt / gt (and)"
do_test attrs3 "string: ne (or) "
do_test attrs4 "string: exists "
do_test attrs5 "string: not_exists "
do_test attrs6 "is_dc: true "
do_test attrs7 "is_dc: false "
do_test attrs8 "score_attribute "
do_test per-node-attrs "Per node resource parameters"
echo ""
do_test mon-rsc-1 "Schedule Monitor - start"
do_test mon-rsc-2 "Schedule Monitor - move "
do_test mon-rsc-3 "Schedule Monitor - pending start "
do_test mon-rsc-4 "Schedule Monitor - move/pending start"
echo ""
do_test rec-rsc-0 "Resource Recover - no start "
do_test rec-rsc-1 "Resource Recover - start "
do_test rec-rsc-2 "Resource Recover - monitor "
do_test rec-rsc-3 "Resource Recover - stop - ignore"
do_test rec-rsc-4 "Resource Recover - stop - block "
do_test rec-rsc-5 "Resource Recover - stop - fence "
do_test rec-rsc-6 "Resource Recover - multiple - restart"
do_test rec-rsc-7 "Resource Recover - multiple - stop "
do_test rec-rsc-8 "Resource Recover - multiple - block "
do_test rec-rsc-9 "Resource Recover - group/group"
do_test monitor-recovery "on-fail=block + resource recovery detected by recurring monitor"
do_test stop-failure-no-quorum "Stop failure without quorum"
do_test stop-failure-no-fencing "Stop failure without fencing available"
do_test stop-failure-with-fencing "Stop failure with fencing available"
do_test multiple-active-block-group "Support of multiple-active=block for resource groups"
do_test multiple-monitor-one-failed "Consider resource failed if any of the configured monitor operations failed"
echo ""
do_test quorum-1 "No quorum - ignore"
do_test quorum-2 "No quorum - freeze"
do_test quorum-3 "No quorum - stop "
do_test quorum-4 "No quorum - start anyway"
do_test quorum-5 "No quorum - start anyway (group)"
do_test quorum-6 "No quorum - start anyway (clone)"
do_test bug-cl-5212 "No promotion with no-quorum-policy=freeze"
echo ""
do_test rec-node-1 "Node Recover - Startup - no fence"
do_test rec-node-2 "Node Recover - Startup - fence "
do_test rec-node-3 "Node Recover - HA down - no fence"
do_test rec-node-4 "Node Recover - HA down - fence "
do_test rec-node-5 "Node Recover - CRM down - no fence"
do_test rec-node-6 "Node Recover - CRM down - fence "
do_test rec-node-7 "Node Recover - no quorum - ignore "
do_test rec-node-8 "Node Recover - no quorum - freeze "
do_test rec-node-9 "Node Recover - no quorum - stop "
do_test rec-node-10 "Node Recover - no quorum - stop w/fence"
do_test rec-node-11 "Node Recover - CRM down w/ group - fence "
do_test rec-node-12 "Node Recover - nothing active - fence "
do_test rec-node-13 "Node Recover - failed resource + shutdown - fence "
do_test rec-node-15 "Node Recover - unknown lrm section"
do_test rec-node-14 "Serialize all stonith's"
echo ""
do_test multi1 "Multiple Active (stop/start)"
echo ""
do_test migrate-begin "Normal migration"
do_test migrate-success "Completed migration"
do_test migrate-partial-1 "Completed migration, missing stop on source"
do_test migrate-partial-2 "Successful migrate_to only"
do_test migrate-partial-3 "Successful migrate_to only, target down"
do_test migrate-partial-4 "Migrate from the correct host after migrate_to+migrate_from"
do_test bug-5186-partial-migrate "Handle partial migration when src node loses membership"
do_test migrate-fail-2 "Failed migrate_from"
do_test migrate-fail-3 "Failed migrate_from + stop on source"
do_test migrate-fail-4 "Failed migrate_from + stop on target - ideally we wouldn't need to re-stop on target"
do_test migrate-fail-5 "Failed migrate_from + stop on source and target"
do_test migrate-fail-6 "Failed migrate_to"
do_test migrate-fail-7 "Failed migrate_to + stop on source"
do_test migrate-fail-8 "Failed migrate_to + stop on target - ideally we wouldn't need to re-stop on target"
do_test migrate-fail-9 "Failed migrate_to + stop on source and target"
do_test migrate-stop "Migration in a stopping stack"
do_test migrate-start "Migration in a starting stack"
do_test migrate-stop_start "Migration in a restarting stack"
do_test migrate-stop-complex "Migration in a complex stopping stack"
do_test migrate-start-complex "Migration in a complex starting stack"
do_test migrate-stop-start-complex "Migration in a complex moving stack"
do_test migrate-shutdown "Order the post-migration 'stop' before node shutdown"
do_test migrate-1 "Migrate (migrate)"
do_test migrate-2 "Migrate (stable)"
do_test migrate-3 "Migrate (failed migrate_to)"
do_test migrate-4 "Migrate (failed migrate_from)"
do_test novell-252693 "Migration in a stopping stack"
do_test novell-252693-2 "Migration in a starting stack"
do_test novell-252693-3 "Non-Migration in a starting and stopping stack"
do_test bug-1820 "Migration in a group"
do_test bug-1820-1 "Non-migration in a group"
do_test migrate-5 "Primitive migration with a clone"
do_test migrate-fencing "Migration after Fencing"
do_test migrate-both-vms "Migrate two VMs that have no colocation"
do_test 1-a-then-bm-move-b "Advanced migrate logic. A then B. migrate B."
do_test 2-am-then-b-move-a "Advanced migrate logic, A then B, migrate A without stopping B"
do_test 3-am-then-bm-both-migrate "Advanced migrate logic. A then B. migrate both"
do_test 4-am-then-bm-b-not-migratable "Advanced migrate logic, A then B, B not migratable"
do_test 5-am-then-bm-a-not-migratable "Advanced migrate logic. A then B. move both, a not migratable"
do_test 6-migrate-group "Advanced migrate logic, migrate a group"
do_test 7-migrate-group-one-unmigratable "Advanced migrate logic, migrate group mixed with allow-migrate true/false"
do_test 8-am-then-bm-a-migrating-b-stopping "Advanced migrate logic, A then B, A migrating, B stopping"
do_test 9-am-then-bm-b-migrating-a-stopping "Advanced migrate logic, A then B, B migrate, A stopping"
do_test 10-a-then-bm-b-move-a-clone "Advanced migrate logic, A clone then B, migrate B while stopping A"
do_test 11-a-then-bm-b-move-a-clone-starting "Advanced migrate logic, A clone then B, B moving while A is start/stopping"
do_test a-promote-then-b-migrate "A promote then B start. migrate B"
do_test a-demote-then-b-migrate "A demote then B stop. migrate B"
#do_test migrate-versioned "Disable migration for versioned resources"
#echo ""
#do_test complex1 "Complex "
do_test bug-lf-2422 "Dependency on partially active group - stop ocfs:*"
echo ""
do_test clone-anon-probe-1 "Probe the correct (anonymous) clone instance for each node"
do_test clone-anon-probe-2 "Avoid needless re-probing of anonymous clones"
do_test clone-anon-failcount "Merge failcounts for anonymous clones"
do_test inc0 "Incarnation start"
do_test inc1 "Incarnation start order"
do_test inc2 "Incarnation silent restart, stop, move"
do_test inc3 "Inter-incarnation ordering, silent restart, stop, move"
do_test inc4 "Inter-incarnation ordering, silent restart, stop, move (ordered)"
do_test inc5 "Inter-incarnation ordering, silent restart, stop, move (restart 1)"
do_test inc6 "Inter-incarnation ordering, silent restart, stop, move (restart 2)"
do_test inc7 "Clone colocation"
do_test inc8 "Clone anti-colocation"
do_test inc9 "Non-unique clone"
do_test inc10 "Non-unique clone (stop)"
do_test inc11 "Primitive colocation with clones"
do_test inc12 "Clone shutdown"
do_test cloned-group "Make sure only the correct number of cloned groups are started"
do_test cloned-group-stop "Ensure stopping qpidd also stops glance and cinder"
do_test clone-no-shuffle "Don't prioritize allocation of instances that must be moved"
do_test clone-max-zero "Orphan processing with clone-max=0"
do_test clone-anon-dup "Bug LF#2087 - Correctly parse the state of anonymous clones that are active more than once per node"
do_test bug-lf-2160 "Don't shuffle clones due to colocation"
do_test bug-lf-2213 "clone-node-max enforcement for cloned groups"
do_test bug-lf-2153 "Clone ordering constraints"
do_test bug-lf-2361 "Ensure clones observe mandatory ordering constraints if the LHS is unrunnable"
do_test bug-lf-2317 "Avoid needless restart of primitive depending on a clone"
do_test clone-colocate-instance-1 "Colocation with a specific clone instance (negative example)"
do_test clone-colocate-instance-2 "Colocation with a specific clone instance"
do_test clone-order-instance "Ordering with specific clone instances"
do_test bug-lf-2453 "Enforce mandatory clone ordering without colocation"
do_test bug-lf-2508 "Correctly reconstruct the status of anonymous cloned groups"
do_test bug-lf-2544 "Balanced clone placement"
do_test bug-lf-2445 "Redistribute clones with node-max > 1 and stickiness = 0"
do_test bug-lf-2574 "Avoid clone shuffle"
do_test bug-lf-2581 "Avoid group restart due to unrelated clone (re)start"
do_test bug-cl-5168 "Don't shuffle clones"
do_test bug-cl-5170 "Prevent clone from starting with on-fail=block"
do_test clone-fail-block-colocation "Move colocated group when failed clone has on-fail=block"
do_test clone-interleave-1 "Clone-3 cannot start on pcmk-1 due to interleaved ordering (no colocation)"
do_test clone-interleave-2 "Clone-3 must stop on pcmk-1 due to interleaved ordering (no colocation)"
do_test clone-interleave-3 "Clone-3 must be recovered on pcmk-1 due to interleaved ordering (no colocation)"
echo ""
do_test cloned_start_one "order first clone then clone... first clone_min=2"
do_test cloned_start_two "order first clone then clone... first clone_min=2"
do_test cloned_stop_one "order first clone then clone... first clone_min=2"
do_test cloned_stop_two "order first clone then clone... first clone_min=2"
do_test clone_min_interleave_start_one "order first clone then clone... first clone_min=2 and then has interleave=true"
do_test clone_min_interleave_start_two "order first clone then clone... first clone_min=2 and then has interleave=true"
do_test clone_min_interleave_stop_one "order first clone then clone... first clone_min=2 and then has interleave=true"
do_test clone_min_interleave_stop_two "order first clone then clone... first clone_min=2 and then has interleave=true"
do_test clone_min_start_one "order first clone then primitive... first clone_min=2"
do_test clone_min_start_two "order first clone then primitive... first clone_min=2"
do_test clone_min_stop_all "order first clone then primitive... first clone_min=2"
do_test clone_min_stop_one "order first clone then primitive... first clone_min=2"
do_test clone_min_stop_two "order first clone then primitive... first clone_min=2"
echo ""
do_test unfence-startup "Clean unfencing"
do_test unfence-definition "Unfencing when the agent changes"
do_test unfence-parameters "Unfencing when the agent parameters changes"
echo ""
do_test master-0 "Stopped -> Slave"
do_test master-1 "Stopped -> Promote"
do_test master-2 "Stopped -> Promote : notify"
do_test master-3 "Stopped -> Promote : master location"
do_test master-4 "Started -> Promote : master location"
do_test master-5 "Promoted -> Promoted"
do_test master-6 "Promoted -> Promoted (2)"
do_test master-7 "Promoted -> Fenced"
do_test master-8 "Promoted -> Fenced -> Moved"
do_test master-9 "Stopped + Promotable + No quorum"
do_test master-10 "Stopped -> Promotable : notify with monitor"
do_test master-11 "Stopped -> Promote : colocation"
do_test novell-239082 "Demote/Promote ordering"
do_test novell-239087 "Stable master placement"
do_test master-12 "Promotion based solely on rsc_location constraints"
do_test master-13 "Include preferences of colocated resources when placing master"
do_test master-demote "Ordering when actions depends on demoting a slave resource"
do_test master-ordering "Prevent resources from starting that need a master"
do_test bug-1765 "Master-Master Colocation (dont stop the slaves)"
do_test master-group "Promotion of cloned groups"
do_test bug-lf-1852 "Don't shuffle master/slave instances unnecessarily"
do_test master-failed-demote "Don't retry failed demote actions"
do_test master-failed-demote-2 "Don't retry failed demote actions (notify=false)"
do_test master-depend "Ensure resources that depend on the master don't get allocated until the master does"
do_test master-reattach "Re-attach to a running master"
do_test master-allow-start "Don't include master score if it would prevent allocation"
do_test master-colocation "Allow master instances placemaker to be influenced by colocation constraints"
do_test master-pseudo "Make sure promote/demote pseudo actions are created correctly"
do_test master-role "Prevent target-role from promoting more than master-max instances"
do_test bug-lf-2358 "Master-Master anti-colocation"
do_test master-promotion-constraint "Mandatory master colocation constraints"
do_test unmanaged-master "Ensure role is preserved for unmanaged resources"
do_test master-unmanaged-monitor "Start the correct monitor operation for unmanaged masters"
do_test master-demote-2 "Demote does not clear past failure"
do_test master-move "Move master based on failure of colocated group"
do_test master-probed-score "Observe the promotion score of probed resources"
do_test colocation_constraint_stops_master "cl#5054 - Ensure master is demoted when stopped by colocation constraint"
do_test colocation_constraint_stops_slave "cl#5054 - Ensure slave is not demoted when stopped by colocation constraint"
do_test order_constraint_stops_master "cl#5054 - Ensure master is demoted when stopped by order constraint"
do_test order_constraint_stops_slave "cl#5054 - Ensure slave is not demoted when stopped by order constraint"
do_test master_monitor_restart "cl#5072 - Ensure master monitor operation will start after promotion."
do_test bug-rh-880249 "Handle replacement of an m/s resource with a primitive"
do_test bug-5143-ms-shuffle "Prevent master shuffling due to promotion score"
do_test master-demote-block "Block promotion if demote fails with on-fail=block"
do_test master-dependent-ban "Don't stop instances from being active because a dependent is banned from that host"
do_test master-stop "Stop instances due to location constraint with role=Started"
do_test master-partially-demoted-group "Allow partially demoted group to finish demoting"
do_test bug-cl-5213 "Ensure role colocation with -INFINITY is enforced"
do_test bug-cl-5219 "Allow unrelated resources with a common colocation target to remain promoted"
do_test master-asymmetrical-order "Fix the behaviors of multi-state resources with asymmetrical ordering"
do_test master-notify "Master promotion with notifies"
echo ""
do_test history-1 "Correctly parse stateful-1 resource state"
echo ""
do_test managed-0 "Managed (reference)"
do_test managed-1 "Not managed - down "
do_test managed-2 "Not managed - up "
do_test bug-5028 "Shutdown should block if anything depends on an unmanaged resource"
do_test bug-5028-detach "Ensure detach still works"
do_test bug-5028-bottom "Ensure shutdown still blocks if the blocked resource is at the bottom of the stack"
do_test unmanaged-stop-1 "cl#5155 - Block the stop of resources if any depending resource is unmanaged "
do_test unmanaged-stop-2 "cl#5155 - Block the stop of resources if the first resource in a mandatory stop order is unmanaged "
do_test unmanaged-stop-3 "cl#5155 - Block the stop of resources if any depending resource in a group is unmanaged "
do_test unmanaged-stop-4 "cl#5155 - Block the stop of resources if any depending resource in the middle of a group is unmanaged "
do_test unmanaged-block-restart "Block restart of resources if any dependent resource in a group is unmanaged"
echo ""
do_test interleave-0 "Interleave (reference)"
do_test interleave-1 "coloc - not interleaved"
do_test interleave-2 "coloc - interleaved "
do_test interleave-3 "coloc - interleaved (2)"
do_test interleave-pseudo-stop "Interleaved clone during stonith"
do_test interleave-stop "Interleaved clone during stop"
do_test interleave-restart "Interleaved clone during dependency restart"
echo ""
do_test notify-0 "Notify reference"
do_test notify-1 "Notify simple"
do_test notify-2 "Notify simple, confirm"
do_test notify-3 "Notify move, confirm"
do_test novell-239079 "Notification priority"
#do_test notify-2 "Notify - 764"
echo ""
do_test 594 "OSDL #594 - Unrunnable actions scheduled in transition"
do_test 662 "OSDL #662 - Two resources start on one node when incarnation_node_max = 1"
do_test 696 "OSDL #696 - CRM starts stonith RA without monitor"
do_test 726 "OSDL #726 - Attempting to schedule rsc_posic041_monitor_5000 _after_ a stop"
do_test 735 "OSDL #735 - Correctly detect that rsc_hadev1 is stopped on hadev3"
do_test 764 "OSDL #764 - Missing monitor op for DoFencing:child_DoFencing:1"
do_test 797 "OSDL #797 - Assert triggered: task_id_i > max_call_id"
do_test 829 "OSDL #829"
do_test 994 "OSDL #994 - Stopping the last resource in a resource group causes the entire group to be restarted"
do_test 994-2 "OSDL #994 - with a dependent resource"
do_test 1360 "OSDL #1360 - Clone stickiness"
do_test 1484 "OSDL #1484 - on_fail=stop"
do_test 1494 "OSDL #1494 - Clone stability"
do_test unrunnable-1 "Unrunnable"
do_test unrunnable-2 "Unrunnable 2"
do_test stonith-0 "Stonith loop - 1"
do_test stonith-1 "Stonith loop - 2"
do_test stonith-2 "Stonith loop - 3"
do_test stonith-3 "Stonith startup"
do_test stonith-4 "Stonith node state"
do_test bug-1572-1 "Recovery of groups depending on master/slave"
do_test bug-1572-2 "Recovery of groups depending on master/slave when the master is never re-promoted"
do_test bug-1685 "Depends-on-master ordering"
do_test bug-1822 "Don't promote partially active groups"
do_test bug-pm-11 "New resource added to a m/s group"
do_test bug-pm-12 "Recover only the failed portion of a cloned group"
do_test bug-n-387749 "Don't shuffle clone instances"
do_test bug-n-385265 "Don't ignore the failure stickiness of group children - resource_idvscommon should stay stopped"
do_test bug-n-385265-2 "Ensure groups are migrated instead of remaining partially active on the current node"
do_test bug-lf-1920 "Correctly handle probes that find active resources"
do_test bnc-515172 "Location constraint with multiple expressions"
do_test colocate-primitive-with-clone "Optional colocation with a clone"
do_test use-after-free-merge "Use-after-free in native_merge_weights"
do_test bug-lf-2551 "STONITH ordering for stop"
do_test bug-lf-2606 "Stonith implies demote"
do_test bug-lf-2474 "Ensure resource op timeout takes precedence over op_defaults"
do_test bug-suse-707150 "Prevent vm-01 from starting due to colocation/ordering"
do_test bug-5014-A-start-B-start "Verify when A starts B starts using symmetrical=false"
do_test bug-5014-A-stop-B-started "Verify when A stops B does not stop if it has already started using symmetric=false"
do_test bug-5014-A-stopped-B-stopped "Verify when A is stopped and B has not started, B does not start before A using symmetric=false"
do_test bug-5014-CthenAthenB-C-stopped "Verify when C then A is symmetrical=true, A then B is symmetric=false, and C is stopped that nothing starts."
do_test bug-5014-CLONE-A-start-B-start "Verify when A starts B starts using clone resources with symmetric=false"
do_test bug-5014-CLONE-A-stop-B-started "Verify when A stops B does not stop if it has already started using clone resources with symmetric=false."
do_test bug-5014-GROUP-A-start-B-start "Verify when A starts B starts when using group resources with symmetric=false."
do_test bug-5014-GROUP-A-stopped-B-started "Verify when A stops B does not stop if it has already started using group resources with symmetric=false."
do_test bug-5014-GROUP-A-stopped-B-stopped "Verify when A is stopped and B has not started, B does not start before A using group resources with symmetric=false."
do_test bug-5014-ordered-set-symmetrical-false "Verify ordered sets work with symmetrical=false"
do_test bug-5014-ordered-set-symmetrical-true "Verify ordered sets work with symmetrical=true"
do_test bug-5007-masterslave_colocation "Verify use of colocation scores other than INFINITY and -INFINITY work on multi-state resources."
do_test bug-5038 "Prevent restart of anonymous clones when clone-max decreases"
do_test bug-5025-1 "Automatically clean up failcount after resource config change with reload"
do_test bug-5025-2 "Make sure clear failcount action isn't set when config does not change."
do_test bug-5025-3 "Automatically clean up failcount after resource config change with restart"
do_test bug-5025-4 "Clear failcount when last failure is a start op and rsc attributes changed."
do_test failcount "Ensure failcounts are correctly expired"
do_test failcount-block "Ensure failcounts are not expired when on-fail=block is present"
do_test per-op-failcount "Ensure per-operation failcount is handled and not passed to fence agent"
do_test monitor-onfail-restart "bug-5058 - Monitor failure with on-fail set to restart"
do_test monitor-onfail-stop "bug-5058 - Monitor failure wiht on-fail set to stop"
do_test bug-5059 "No need to restart p_stateful1:*"
do_test bug-5069-op-enabled "Test on-fail=ignore with failure when monitor is enabled."
do_test bug-5069-op-disabled "Test on-fail-ignore with failure when monitor is disabled."
do_test obsolete-lrm-resource "cl#5115 - Do not use obsolete lrm_resource sections"
do_test expire-non-blocked-failure "Ignore failure-timeout only if the failed operation has on-fail=block"
do_test asymmetrical-order-move "Respect asymmetrical ordering when trying to move resources"
do_test start-then-stop-with-unfence "Avoid graph loop with start-then-stop constraint plus unfencing"
do_test ignore_stonith_rsc_order1 "cl#5056- Ignore order constraint between stonith and non-stonith rsc."
do_test ignore_stonith_rsc_order2 "cl#5056- Ignore order constraint with group rsc containing mixed stonith and non-stonith."
do_test ignore_stonith_rsc_order3 "cl#5056- Ignore order constraint, stonith clone and mixed group"
do_test ignore_stonith_rsc_order4 "cl#5056- Ignore order constraint, stonith clone and clone with nested mixed group"
do_test honor_stonith_rsc_order1 "cl#5056- Honor order constraint, stonith clone and pure stonith group(single rsc)."
do_test honor_stonith_rsc_order2 "cl#5056- Honor order constraint, stonith clone and pure stonith group(multiple rsc)"
do_test honor_stonith_rsc_order3 "cl#5056- Honor order constraint, stonith clones with nested pure stonith group."
do_test honor_stonith_rsc_order4 "cl#5056- Honor order constraint, between two native stonith rscs."
do_test probe-timeout "cl#5099 - Default probe timeout"
do_test concurrent-fencing "Allow performing fencing operations in parallel"
echo ""
do_test systemhealth1 "System Health () #1"
do_test systemhealth2 "System Health () #2"
do_test systemhealth3 "System Health () #3"
do_test systemhealthn1 "System Health (None) #1"
do_test systemhealthn2 "System Health (None) #2"
do_test systemhealthn3 "System Health (None) #3"
do_test systemhealthm1 "System Health (Migrate On Red) #1"
do_test systemhealthm2 "System Health (Migrate On Red) #2"
do_test systemhealthm3 "System Health (Migrate On Red) #3"
do_test systemhealtho1 "System Health (Only Green) #1"
do_test systemhealtho2 "System Health (Only Green) #2"
do_test systemhealtho3 "System Health (Only Green) #3"
do_test systemhealthp1 "System Health (Progessive) #1"
do_test systemhealthp2 "System Health (Progessive) #2"
do_test systemhealthp3 "System Health (Progessive) #3"
echo ""
do_test utilization "Placement Strategy - utilization"
do_test minimal "Placement Strategy - minimal"
do_test balanced "Placement Strategy - balanced"
echo ""
do_test placement-stickiness "Optimized Placement Strategy - stickiness"
do_test placement-priority "Optimized Placement Strategy - priority"
do_test placement-location "Optimized Placement Strategy - location"
do_test placement-capacity "Optimized Placement Strategy - capacity"
echo ""
do_test utilization-order1 "Utilization Order - Simple"
do_test utilization-order2 "Utilization Order - Complex"
do_test utilization-order3 "Utilization Order - Migrate"
do_test utilization-order4 "Utilization Order - Live Mirgration (bnc#695440)"
do_test utilization-shuffle "Don't displace prmExPostgreSQLDB2 on act2, Start prmExPostgreSQLDB1 on act3"
do_test load-stopped-loop "Avoid transition loop due to load_stopped (cl#5044)"
do_test load-stopped-loop-2 "cl#5235 - Prevent graph loops that can be introduced by load_stopped -> migrate_to ordering"
echo ""
do_test colocated-utilization-primitive-1 "Colocated Utilization - Primitive"
do_test colocated-utilization-primitive-2 "Colocated Utilization - Choose the most capable node"
do_test colocated-utilization-group "Colocated Utilization - Group"
do_test colocated-utilization-clone "Colocated Utilization - Clone"
do_test utilization-check-allowed-nodes "Only check the capacities of the nodes that can run the resource"
echo ""
do_test reprobe-target_rc "Ensure correct target_rc for reprobe of inactive resources"
do_test node-maintenance-1 "cl#5128 - Node maintenance"
do_test node-maintenance-2 "cl#5128 - Node maintenance (coming out of maintenance mode)"
do_test shutdown-maintenance-node "Do not fence a maintenance node if it shuts down cleanly"
do_test rsc-maintenance "Per-resource maintenance"
echo ""
do_test not-installed-agent "The resource agent is missing"
do_test not-installed-tools "Something the resource agent needs is missing"
echo ""
do_test stopped-monitor-00 "Stopped Monitor - initial start"
do_test stopped-monitor-01 "Stopped Monitor - failed started"
do_test stopped-monitor-02 "Stopped Monitor - started multi-up"
do_test stopped-monitor-03 "Stopped Monitor - stop started"
do_test stopped-monitor-04 "Stopped Monitor - failed stop"
do_test stopped-monitor-05 "Stopped Monitor - start unmanaged"
do_test stopped-monitor-06 "Stopped Monitor - unmanaged multi-up"
do_test stopped-monitor-07 "Stopped Monitor - start unmanaged multi-up"
do_test stopped-monitor-08 "Stopped Monitor - migrate"
do_test stopped-monitor-09 "Stopped Monitor - unmanage started"
do_test stopped-monitor-10 "Stopped Monitor - unmanaged started multi-up"
do_test stopped-monitor-11 "Stopped Monitor - stop unmanaged started"
do_test stopped-monitor-12 "Stopped Monitor - unmanaged started multi-up (targer-role="Stopped")"
do_test stopped-monitor-20 "Stopped Monitor - initial stop"
do_test stopped-monitor-21 "Stopped Monitor - stopped single-up"
do_test stopped-monitor-22 "Stopped Monitor - stopped multi-up"
do_test stopped-monitor-23 "Stopped Monitor - start stopped"
do_test stopped-monitor-24 "Stopped Monitor - unmanage stopped"
do_test stopped-monitor-25 "Stopped Monitor - unmanaged stopped multi-up"
do_test stopped-monitor-26 "Stopped Monitor - start unmanaged stopped"
do_test stopped-monitor-27 "Stopped Monitor - unmanaged stopped multi-up (target-role="Started")"
do_test stopped-monitor-30 "Stopped Monitor - new node started"
do_test stopped-monitor-31 "Stopped Monitor - new node stopped"
echo""
do_test ticket-primitive-1 "Ticket - Primitive (loss-policy=stop, initial)"
do_test ticket-primitive-2 "Ticket - Primitive (loss-policy=stop, granted)"
do_test ticket-primitive-3 "Ticket - Primitive (loss-policy-stop, revoked)"
do_test ticket-primitive-4 "Ticket - Primitive (loss-policy=demote, initial)"
do_test ticket-primitive-5 "Ticket - Primitive (loss-policy=demote, granted)"
do_test ticket-primitive-6 "Ticket - Primitive (loss-policy=demote, revoked)"
do_test ticket-primitive-7 "Ticket - Primitive (loss-policy=fence, initial)"
do_test ticket-primitive-8 "Ticket - Primitive (loss-policy=fence, granted)"
do_test ticket-primitive-9 "Ticket - Primitive (loss-policy=fence, revoked)"
do_test ticket-primitive-10 "Ticket - Primitive (loss-policy=freeze, initial)"
do_test ticket-primitive-11 "Ticket - Primitive (loss-policy=freeze, granted)"
do_test ticket-primitive-12 "Ticket - Primitive (loss-policy=freeze, revoked)"
do_test ticket-primitive-13 "Ticket - Primitive (loss-policy=stop, standby, granted)"
do_test ticket-primitive-14 "Ticket - Primitive (loss-policy=stop, granted, standby)"
do_test ticket-primitive-15 "Ticket - Primitive (loss-policy=stop, standby, revoked)"
do_test ticket-primitive-16 "Ticket - Primitive (loss-policy=demote, standby, granted)"
do_test ticket-primitive-17 "Ticket - Primitive (loss-policy=demote, granted, standby)"
do_test ticket-primitive-18 "Ticket - Primitive (loss-policy=demote, standby, revoked)"
do_test ticket-primitive-19 "Ticket - Primitive (loss-policy=fence, standby, granted)"
do_test ticket-primitive-20 "Ticket - Primitive (loss-policy=fence, granted, standby)"
do_test ticket-primitive-21 "Ticket - Primitive (loss-policy=fence, standby, revoked)"
do_test ticket-primitive-22 "Ticket - Primitive (loss-policy=freeze, standby, granted)"
do_test ticket-primitive-23 "Ticket - Primitive (loss-policy=freeze, granted, standby)"
do_test ticket-primitive-24 "Ticket - Primitive (loss-policy=freeze, standby, revoked)"
echo""
do_test ticket-group-1 "Ticket - Group (loss-policy=stop, initial)"
do_test ticket-group-2 "Ticket - Group (loss-policy=stop, granted)"
do_test ticket-group-3 "Ticket - Group (loss-policy-stop, revoked)"
do_test ticket-group-4 "Ticket - Group (loss-policy=demote, initial)"
do_test ticket-group-5 "Ticket - Group (loss-policy=demote, granted)"
do_test ticket-group-6 "Ticket - Group (loss-policy=demote, revoked)"
do_test ticket-group-7 "Ticket - Group (loss-policy=fence, initial)"
do_test ticket-group-8 "Ticket - Group (loss-policy=fence, granted)"
do_test ticket-group-9 "Ticket - Group (loss-policy=fence, revoked)"
do_test ticket-group-10 "Ticket - Group (loss-policy=freeze, initial)"
do_test ticket-group-11 "Ticket - Group (loss-policy=freeze, granted)"
do_test ticket-group-12 "Ticket - Group (loss-policy=freeze, revoked)"
do_test ticket-group-13 "Ticket - Group (loss-policy=stop, standby, granted)"
do_test ticket-group-14 "Ticket - Group (loss-policy=stop, granted, standby)"
do_test ticket-group-15 "Ticket - Group (loss-policy=stop, standby, revoked)"
do_test ticket-group-16 "Ticket - Group (loss-policy=demote, standby, granted)"
do_test ticket-group-17 "Ticket - Group (loss-policy=demote, granted, standby)"
do_test ticket-group-18 "Ticket - Group (loss-policy=demote, standby, revoked)"
do_test ticket-group-19 "Ticket - Group (loss-policy=fence, standby, granted)"
do_test ticket-group-20 "Ticket - Group (loss-policy=fence, granted, standby)"
do_test ticket-group-21 "Ticket - Group (loss-policy=fence, standby, revoked)"
do_test ticket-group-22 "Ticket - Group (loss-policy=freeze, standby, granted)"
do_test ticket-group-23 "Ticket - Group (loss-policy=freeze, granted, standby)"
do_test ticket-group-24 "Ticket - Group (loss-policy=freeze, standby, revoked)"
echo""
do_test ticket-clone-1 "Ticket - Clone (loss-policy=stop, initial)"
do_test ticket-clone-2 "Ticket - Clone (loss-policy=stop, granted)"
do_test ticket-clone-3 "Ticket - Clone (loss-policy-stop, revoked)"
do_test ticket-clone-4 "Ticket - Clone (loss-policy=demote, initial)"
do_test ticket-clone-5 "Ticket - Clone (loss-policy=demote, granted)"
do_test ticket-clone-6 "Ticket - Clone (loss-policy=demote, revoked)"
do_test ticket-clone-7 "Ticket - Clone (loss-policy=fence, initial)"
do_test ticket-clone-8 "Ticket - Clone (loss-policy=fence, granted)"
do_test ticket-clone-9 "Ticket - Clone (loss-policy=fence, revoked)"
do_test ticket-clone-10 "Ticket - Clone (loss-policy=freeze, initial)"
do_test ticket-clone-11 "Ticket - Clone (loss-policy=freeze, granted)"
do_test ticket-clone-12 "Ticket - Clone (loss-policy=freeze, revoked)"
do_test ticket-clone-13 "Ticket - Clone (loss-policy=stop, standby, granted)"
do_test ticket-clone-14 "Ticket - Clone (loss-policy=stop, granted, standby)"
do_test ticket-clone-15 "Ticket - Clone (loss-policy=stop, standby, revoked)"
do_test ticket-clone-16 "Ticket - Clone (loss-policy=demote, standby, granted)"
do_test ticket-clone-17 "Ticket - Clone (loss-policy=demote, granted, standby)"
do_test ticket-clone-18 "Ticket - Clone (loss-policy=demote, standby, revoked)"
do_test ticket-clone-19 "Ticket - Clone (loss-policy=fence, standby, granted)"
do_test ticket-clone-20 "Ticket - Clone (loss-policy=fence, granted, standby)"
do_test ticket-clone-21 "Ticket - Clone (loss-policy=fence, standby, revoked)"
do_test ticket-clone-22 "Ticket - Clone (loss-policy=freeze, standby, granted)"
do_test ticket-clone-23 "Ticket - Clone (loss-policy=freeze, granted, standby)"
do_test ticket-clone-24 "Ticket - Clone (loss-policy=freeze, standby, revoked)"
echo""
do_test ticket-master-1 "Ticket - Master (loss-policy=stop, initial)"
do_test ticket-master-2 "Ticket - Master (loss-policy=stop, granted)"
do_test ticket-master-3 "Ticket - Master (loss-policy-stop, revoked)"
do_test ticket-master-4 "Ticket - Master (loss-policy=demote, initial)"
do_test ticket-master-5 "Ticket - Master (loss-policy=demote, granted)"
do_test ticket-master-6 "Ticket - Master (loss-policy=demote, revoked)"
do_test ticket-master-7 "Ticket - Master (loss-policy=fence, initial)"
do_test ticket-master-8 "Ticket - Master (loss-policy=fence, granted)"
do_test ticket-master-9 "Ticket - Master (loss-policy=fence, revoked)"
do_test ticket-master-10 "Ticket - Master (loss-policy=freeze, initial)"
do_test ticket-master-11 "Ticket - Master (loss-policy=freeze, granted)"
do_test ticket-master-12 "Ticket - Master (loss-policy=freeze, revoked)"
do_test ticket-master-13 "Ticket - Master (loss-policy=stop, standby, granted)"
do_test ticket-master-14 "Ticket - Master (loss-policy=stop, granted, standby)"
do_test ticket-master-15 "Ticket - Master (loss-policy=stop, standby, revoked)"
do_test ticket-master-16 "Ticket - Master (loss-policy=demote, standby, granted)"
do_test ticket-master-17 "Ticket - Master (loss-policy=demote, granted, standby)"
do_test ticket-master-18 "Ticket - Master (loss-policy=demote, standby, revoked)"
do_test ticket-master-19 "Ticket - Master (loss-policy=fence, standby, granted)"
do_test ticket-master-20 "Ticket - Master (loss-policy=fence, granted, standby)"
do_test ticket-master-21 "Ticket - Master (loss-policy=fence, standby, revoked)"
do_test ticket-master-22 "Ticket - Master (loss-policy=freeze, standby, granted)"
do_test ticket-master-23 "Ticket - Master (loss-policy=freeze, granted, standby)"
do_test ticket-master-24 "Ticket - Master (loss-policy=freeze, standby, revoked)"
echo ""
do_test ticket-rsc-sets-1 "Ticket - Resource sets (1 ticket, initial)"
do_test ticket-rsc-sets-2 "Ticket - Resource sets (1 ticket, granted)"
do_test ticket-rsc-sets-3 "Ticket - Resource sets (1 ticket, revoked)"
do_test ticket-rsc-sets-4 "Ticket - Resource sets (2 tickets, initial)"
do_test ticket-rsc-sets-5 "Ticket - Resource sets (2 tickets, granted)"
do_test ticket-rsc-sets-6 "Ticket - Resource sets (2 tickets, granted)"
do_test ticket-rsc-sets-7 "Ticket - Resource sets (2 tickets, revoked)"
do_test ticket-rsc-sets-8 "Ticket - Resource sets (1 ticket, standby, granted)"
do_test ticket-rsc-sets-9 "Ticket - Resource sets (1 ticket, granted, standby)"
do_test ticket-rsc-sets-10 "Ticket - Resource sets (1 ticket, standby, revoked)"
do_test ticket-rsc-sets-11 "Ticket - Resource sets (2 tickets, standby, granted)"
do_test ticket-rsc-sets-12 "Ticket - Resource sets (2 tickets, standby, granted)"
do_test ticket-rsc-sets-13 "Ticket - Resource sets (2 tickets, granted, standby)"
do_test ticket-rsc-sets-14 "Ticket - Resource sets (2 tickets, standby, revoked)"
do_test cluster-specific-params "Cluster-specific instance attributes based on rules"
do_test site-specific-params "Site-specific instance attributes based on rules"
echo ""
do_test template-1 "Template - 1"
do_test template-2 "Template - 2"
do_test template-3 "Template - 3 (merge operations)"
do_test template-coloc-1 "Template - Colocation 1"
do_test template-coloc-2 "Template - Colocation 2"
do_test template-coloc-3 "Template - Colocation 3"
do_test template-order-1 "Template - Order 1"
do_test template-order-2 "Template - Order 2"
do_test template-order-3 "Template - Order 3"
do_test template-ticket "Template - Ticket"
do_test template-rsc-sets-1 "Template - Resource Sets 1"
do_test template-rsc-sets-2 "Template - Resource Sets 2"
do_test template-rsc-sets-3 "Template - Resource Sets 3"
do_test template-rsc-sets-4 "Template - Resource Sets 4"
do_test template-clone-primitive "Cloned primitive from template"
do_test template-clone-group "Cloned group from template"
do_test location-sets-templates "Resource sets and templates - Location"
do_test tags-coloc-order-1 "Tags - Colocation and Order (Simple)"
do_test tags-coloc-order-2 "Tags - Colocation and Order (Resource Sets with Templates)"
do_test tags-location "Tags - Location"
do_test tags-ticket "Tags - Ticket"
echo ""
do_test container-1 "Container - initial"
do_test container-2 "Container - monitor failed"
do_test container-3 "Container - stop failed"
do_test container-4 "Container - reached migration-threshold"
do_test container-group-1 "Container in group - initial"
do_test container-group-2 "Container in group - monitor failed"
do_test container-group-3 "Container in group - stop failed"
do_test container-group-4 "Container in group - reached migration-threshold"
do_test container-is-remote-node "Place resource within container when container is remote-node"
do_test bug-rh-1097457 "Kill user defined container/contents ordering"
do_test bug-cl-5247 "Graph loop when recovering m/s resource in a container"
do_test bundle-order-startup "Bundle startup ordering"
do_test bundle-order-partial-start "Bundle startup ordering when some dependancies are already running"
do_test bundle-order-partial-start-2 "Bundle startup ordering when some dependancies and the container are already running"
do_test bundle-order-stop "Bundle stop ordering"
do_test bundle-order-partial-stop "Bundle startup ordering when some dependancies are already stopped"
+do_test bundle-order-startup-clone "Prevent startup because bundle isn't promoted"
+do_test bundle-order-startup-clone-2 "Bundle startup with clones"
+do_test bundle-order-stop-clone "Stop bundle because clone is stopping"
+
echo ""
do_test whitebox-fail1 "Fail whitebox container rsc."
do_test whitebox-fail2 "Fail whitebox container rsc lrmd connection."
do_test whitebox-fail3 "Failed containers should not run nested on remote nodes."
do_test whitebox-start "Start whitebox container with resources assigned to it"
do_test whitebox-stop "Stop whitebox container with resources assigned to it"
do_test whitebox-move "Move whitebox container with resources assigned to it"
do_test whitebox-asymmetric "Verify connection rsc opts-in based on container resource"
do_test whitebox-ms-ordering "Verify promote/demote can not occur before connection is established"
do_test whitebox-ms-ordering-move "Stop/Start cycle within a moving container"
do_test whitebox-orphaned "Properly shutdown orphaned whitebox container"
do_test whitebox-orphan-ms "Properly tear down orphan ms resources on remote-nodes"
do_test whitebox-unexpectedly-running "Recover container nodes the cluster did not start."
do_test whitebox-migrate1 "Migrate both container and connection resource"
do_test whitebox-imply-stop-on-fence "imply stop action on container node rsc when host node is fenced"
do_test whitebox-nested-group "Verify guest remote-node works nested in a group"
do_test guest-node-host-dies "Verify guest node is recovered if host goes away"
echo ""
do_test remote-startup-probes "Baremetal remote-node startup probes"
do_test remote-startup "Startup a newly discovered remote-nodes with no status."
do_test remote-fence-unclean "Fence unclean baremetal remote-node"
do_test remote-fence-unclean2 "Fence baremetal remote-node after cluster node fails and connection can not be recovered"
do_test remote-move "Move remote-node connection resource"
do_test remote-disable "Disable a baremetal remote-node"
do_test remote-probe-disable "Probe then stop a baremetal remote-node"
do_test remote-orphaned "Properly shutdown orphaned connection resource"
do_test remote-orphaned2 "verify we can handle orphaned remote connections with active resources on the remote"
do_test remote-recover "Recover connection resource after cluster-node fails."
do_test remote-stale-node-entry "Make sure we properly handle leftover remote-node entries in the node section"
do_test remote-partial-migrate "Make sure partial migrations are handled before ops on the remote node."
do_test remote-partial-migrate2 "Make sure partial migration target is prefered for remote connection."
do_test remote-recover-fail "Make sure start failure causes fencing if rsc are active on remote."
do_test remote-start-fail "Make sure a start failure does not result in fencing if no active resources are on remote."
do_test remote-unclean2 "Make monitor failure always results in fencing, even if no rsc are active on remote."
do_test remote-fence-before-reconnect "Fence before clearing recurring monitor failure"
do_test remote-recovery "Recover remote connections before attempting demotion"
do_test remote-recover-connection "Optimistically recovery of only the connection"
do_test remote-recover-all "Fencing when the connection has no home"
do_test remote-recover-no-resources "Fencing when the connection has no home and no active resources"
do_test remote-recover-unknown "Fencing when the connection has no home and the remote has no operation history"
echo ""
do_test resource-discovery "Exercises resource-discovery location constraint option."
do_test rsc-discovery-per-node "Disable resource discovery per node"
echo ""
do_test isolation-start-all "Start docker isolated resources."
do_test isolation-restart-all "Restart docker isolated resources."
do_test isolation-clone "Cloned isolated primitive."
#echo ""
#do_test versioned-resources "Start resources with #ra-version rules"
echo ""
test_results
diff --git a/pengine/test10/bundle-order-startup-clone-2.dot b/pengine/test10/bundle-order-startup-clone-2.dot
new file mode 100644
index 0000000000..af902617b3
--- /dev/null
+++ b/pengine/test10/bundle-order-startup-clone-2.dot
@@ -0,0 +1,335 @@
+digraph "g" {
+"galera-bundle-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-0_start_0 metal-1" -> "galera-bundle-0_monitor_60000 metal-1" [ style = bold]
+"galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold]
+"galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold]
+"galera-bundle-0_start_0 metal-1" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
+"galera-bundle-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-1_start_0 metal-2" -> "galera-bundle-1_monitor_60000 metal-2" [ style = bold]
+"galera-bundle-1_start_0 metal-2" -> "galera:1_monitor_20000 galera-bundle-1" [ style = bold]
+"galera-bundle-1_start_0 metal-2" -> "galera:1_monitor_30000 galera-bundle-1" [ style = bold]
+"galera-bundle-1_start_0 metal-2" -> "galera:1_start_0 galera-bundle-1" [ style = bold]
+"galera-bundle-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-2_start_0 metal-3" -> "galera-bundle-2_monitor_60000 metal-3" [ style = bold]
+"galera-bundle-2_start_0 metal-3" -> "galera:2_monitor_20000 galera-bundle-2" [ style = bold]
+"galera-bundle-2_start_0 metal-3" -> "galera:2_monitor_30000 galera-bundle-2" [ style = bold]
+"galera-bundle-2_start_0 metal-3" -> "galera:2_start_0 galera-bundle-2" [ style = bold]
+"galera-bundle-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
+"galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
+"galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
+"galera-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
+"galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
+"galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
+"galera-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
+"galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
+"galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
+"galera-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle-0_start_0 metal-1" [ style = bold]
+"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle-docker-0_monitor_60000 metal-1" [ style = bold]
+"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle_running_0" [ style = bold]
+"galera-bundle-docker-0_start_0 metal-1" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
+"galera-bundle-docker-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-1_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
+"galera-bundle-docker-1_monitor_0 metal-1" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
+"galera-bundle-docker-1_monitor_0 metal-1" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
+"galera-bundle-docker-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-1_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
+"galera-bundle-docker-1_monitor_0 metal-2" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
+"galera-bundle-docker-1_monitor_0 metal-2" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
+"galera-bundle-docker-1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-1_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
+"galera-bundle-docker-1_monitor_0 metal-3" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
+"galera-bundle-docker-1_monitor_0 metal-3" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
+"galera-bundle-docker-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-1_start_0 metal-2" -> "galera-bundle-1_start_0 metal-2" [ style = bold]
+"galera-bundle-docker-1_start_0 metal-2" -> "galera-bundle-docker-1_monitor_60000 metal-2" [ style = bold]
+"galera-bundle-docker-1_start_0 metal-2" -> "galera-bundle_running_0" [ style = bold]
+"galera-bundle-docker-1_start_0 metal-2" -> "galera:1_start_0 galera-bundle-1" [ style = bold]
+"galera-bundle-docker-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-2_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
+"galera-bundle-docker-2_monitor_0 metal-1" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
+"galera-bundle-docker-2_monitor_0 metal-1" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
+"galera-bundle-docker-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-2_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
+"galera-bundle-docker-2_monitor_0 metal-2" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
+"galera-bundle-docker-2_monitor_0 metal-2" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
+"galera-bundle-docker-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-2_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
+"galera-bundle-docker-2_monitor_0 metal-3" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
+"galera-bundle-docker-2_monitor_0 metal-3" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
+"galera-bundle-docker-2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-2_start_0 metal-3" -> "galera-bundle-2_start_0 metal-3" [ style = bold]
+"galera-bundle-docker-2_start_0 metal-3" -> "galera-bundle-docker-2_monitor_60000 metal-3" [ style = bold]
+"galera-bundle-docker-2_start_0 metal-3" -> "galera-bundle_running_0" [ style = bold]
+"galera-bundle-docker-2_start_0 metal-3" -> "galera:2_start_0 galera-bundle-2" [ style = bold]
+"galera-bundle-docker-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold]
+"galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
+"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold]
+"galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
+"galera-bundle-master_start_0" -> "galera:1_start_0 galera-bundle-1" [ style = bold]
+"galera-bundle-master_start_0" -> "galera:2_start_0 galera-bundle-2" [ style = bold]
+"galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
+"galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
+"galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
+"galera-bundle_start_0" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
+"galera-bundle_start_0" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
+"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold]
+"galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
+"galera:0_monitor_20000 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
+"galera:0_monitor_30000 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
+"galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = bold]
+"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold]
+"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold]
+"galera:0_start_0 galera-bundle-0" -> "galera:1_start_0 galera-bundle-1" [ style = bold]
+"galera:0_start_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
+"galera:1_monitor_20000 galera-bundle-1" [ style=bold color="green" fontcolor="black"]
+"galera:1_monitor_30000 galera-bundle-1" [ style=bold color="green" fontcolor="black"]
+"galera:1_start_0 galera-bundle-1" -> "galera-bundle-master_running_0" [ style = bold]
+"galera:1_start_0 galera-bundle-1" -> "galera:1_monitor_20000 galera-bundle-1" [ style = bold]
+"galera:1_start_0 galera-bundle-1" -> "galera:1_monitor_30000 galera-bundle-1" [ style = bold]
+"galera:1_start_0 galera-bundle-1" -> "galera:2_start_0 galera-bundle-2" [ style = bold]
+"galera:1_start_0 galera-bundle-1" [ style=bold color="green" fontcolor="black"]
+"galera:2_monitor_20000 galera-bundle-2" [ style=bold color="green" fontcolor="black"]
+"galera:2_monitor_30000 galera-bundle-2" [ style=bold color="green" fontcolor="black"]
+"galera:2_start_0 galera-bundle-2" -> "galera-bundle-master_running_0" [ style = bold]
+"galera:2_start_0 galera-bundle-2" -> "galera:2_monitor_20000 galera-bundle-2" [ style = bold]
+"galera:2_start_0 galera-bundle-2" -> "galera:2_monitor_30000 galera-bundle-2" [ style = bold]
+"galera:2_start_0 galera-bundle-2" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-0_monitor_0 metal-1" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
+"haproxy-bundle-docker-0_monitor_0 metal-1" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
+"haproxy-bundle-docker-0_monitor_0 metal-1" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
+"haproxy-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-0_monitor_0 metal-2" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
+"haproxy-bundle-docker-0_monitor_0 metal-2" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
+"haproxy-bundle-docker-0_monitor_0 metal-2" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
+"haproxy-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-0_monitor_0 metal-3" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
+"haproxy-bundle-docker-0_monitor_0 metal-3" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
+"haproxy-bundle-docker-0_monitor_0 metal-3" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
+"haproxy-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-0_start_0 metal-1" -> "haproxy-bundle-docker-0_monitor_60000 metal-1" [ style = bold]
+"haproxy-bundle-docker-0_start_0 metal-1" -> "haproxy-bundle_running_0" [ style = bold]
+"haproxy-bundle-docker-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-1_monitor_0 metal-1" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
+"haproxy-bundle-docker-1_monitor_0 metal-1" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
+"haproxy-bundle-docker-1_monitor_0 metal-1" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
+"haproxy-bundle-docker-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-1_monitor_0 metal-2" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
+"haproxy-bundle-docker-1_monitor_0 metal-2" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
+"haproxy-bundle-docker-1_monitor_0 metal-2" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
+"haproxy-bundle-docker-1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-1_monitor_0 metal-3" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
+"haproxy-bundle-docker-1_monitor_0 metal-3" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
+"haproxy-bundle-docker-1_monitor_0 metal-3" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
+"haproxy-bundle-docker-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-1_start_0 metal-2" -> "haproxy-bundle-docker-1_monitor_60000 metal-2" [ style = bold]
+"haproxy-bundle-docker-1_start_0 metal-2" -> "haproxy-bundle_running_0" [ style = bold]
+"haproxy-bundle-docker-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-2_monitor_0 metal-1" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
+"haproxy-bundle-docker-2_monitor_0 metal-1" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
+"haproxy-bundle-docker-2_monitor_0 metal-1" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
+"haproxy-bundle-docker-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-2_monitor_0 metal-2" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
+"haproxy-bundle-docker-2_monitor_0 metal-2" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
+"haproxy-bundle-docker-2_monitor_0 metal-2" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
+"haproxy-bundle-docker-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-2_monitor_0 metal-3" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
+"haproxy-bundle-docker-2_monitor_0 metal-3" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
+"haproxy-bundle-docker-2_monitor_0 metal-3" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
+"haproxy-bundle-docker-2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-2_start_0 metal-3" -> "haproxy-bundle-docker-2_monitor_60000 metal-3" [ style = bold]
+"haproxy-bundle-docker-2_start_0 metal-3" -> "haproxy-bundle_running_0" [ style = bold]
+"haproxy-bundle-docker-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle_running_0" -> "storage-clone_start_0" [ style = bold]
+"haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
+"haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
+"haproxy-bundle_start_0" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
+"haproxy-bundle_start_0" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
+"haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-0_start_0 metal-1" -> "redis-bundle-0_monitor_60000 metal-1" [ style = bold]
+"redis-bundle-0_start_0 metal-1" -> "redis:0_monitor_20000 redis-bundle-0" [ style = bold]
+"redis-bundle-0_start_0 metal-1" -> "redis:0_promote_0 redis-bundle-0" [ style = bold]
+"redis-bundle-0_start_0 metal-1" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
+"redis-bundle-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-1_start_0 metal-2" -> "redis-bundle-1_monitor_60000 metal-2" [ style = bold]
+"redis-bundle-1_start_0 metal-2" -> "redis:1_monitor_20000 redis-bundle-1" [ style = bold]
+"redis-bundle-1_start_0 metal-2" -> "redis:1_promote_0 redis-bundle-1" [ style = bold]
+"redis-bundle-1_start_0 metal-2" -> "redis:1_start_0 redis-bundle-1" [ style = bold]
+"redis-bundle-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-2_start_0 metal-3" -> "redis-bundle-2_monitor_60000 metal-3" [ style = bold]
+"redis-bundle-2_start_0 metal-3" -> "redis:2_monitor_20000 redis-bundle-2" [ style = bold]
+"redis-bundle-2_start_0 metal-3" -> "redis:2_promote_0 redis-bundle-2" [ style = bold]
+"redis-bundle-2_start_0 metal-3" -> "redis:2_start_0 redis-bundle-2" [ style = bold]
+"redis-bundle-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
+"redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
+"redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
+"redis-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
+"redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
+"redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
+"redis-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
+"redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
+"redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
+"redis-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-0_start_0 metal-1" -> "redis-bundle-0_start_0 metal-1" [ style = bold]
+"redis-bundle-docker-0_start_0 metal-1" -> "redis-bundle-docker-0_monitor_60000 metal-1" [ style = bold]
+"redis-bundle-docker-0_start_0 metal-1" -> "redis-bundle_running_0" [ style = bold]
+"redis-bundle-docker-0_start_0 metal-1" -> "redis:0_promote_0 redis-bundle-0" [ style = bold]
+"redis-bundle-docker-0_start_0 metal-1" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
+"redis-bundle-docker-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-1_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
+"redis-bundle-docker-1_monitor_0 metal-1" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
+"redis-bundle-docker-1_monitor_0 metal-1" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
+"redis-bundle-docker-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-1_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
+"redis-bundle-docker-1_monitor_0 metal-2" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
+"redis-bundle-docker-1_monitor_0 metal-2" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
+"redis-bundle-docker-1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-1_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
+"redis-bundle-docker-1_monitor_0 metal-3" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
+"redis-bundle-docker-1_monitor_0 metal-3" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
+"redis-bundle-docker-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-1_start_0 metal-2" -> "redis-bundle-1_start_0 metal-2" [ style = bold]
+"redis-bundle-docker-1_start_0 metal-2" -> "redis-bundle-docker-1_monitor_60000 metal-2" [ style = bold]
+"redis-bundle-docker-1_start_0 metal-2" -> "redis-bundle_running_0" [ style = bold]
+"redis-bundle-docker-1_start_0 metal-2" -> "redis:1_promote_0 redis-bundle-1" [ style = bold]
+"redis-bundle-docker-1_start_0 metal-2" -> "redis:1_start_0 redis-bundle-1" [ style = bold]
+"redis-bundle-docker-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-2_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
+"redis-bundle-docker-2_monitor_0 metal-1" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
+"redis-bundle-docker-2_monitor_0 metal-1" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
+"redis-bundle-docker-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-2_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
+"redis-bundle-docker-2_monitor_0 metal-2" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
+"redis-bundle-docker-2_monitor_0 metal-2" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
+"redis-bundle-docker-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-2_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
+"redis-bundle-docker-2_monitor_0 metal-3" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
+"redis-bundle-docker-2_monitor_0 metal-3" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
+"redis-bundle-docker-2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-2_start_0 metal-3" -> "redis-bundle-2_start_0 metal-3" [ style = bold]
+"redis-bundle-docker-2_start_0 metal-3" -> "redis-bundle-docker-2_monitor_60000 metal-3" [ style = bold]
+"redis-bundle-docker-2_start_0 metal-3" -> "redis-bundle_running_0" [ style = bold]
+"redis-bundle-docker-2_start_0 metal-3" -> "redis:2_promote_0 redis-bundle-2" [ style = bold]
+"redis-bundle-docker-2_start_0 metal-3" -> "redis:2_start_0 redis-bundle-2" [ style = bold]
+"redis-bundle-docker-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-master_promote_0" -> "redis:0_promote_0 redis-bundle-0" [ style = bold]
+"redis-bundle-master_promote_0" -> "redis:1_promote_0 redis-bundle-1" [ style = bold]
+"redis-bundle-master_promote_0" -> "redis:2_promote_0 redis-bundle-2" [ style = bold]
+"redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_promoted_0" -> "redis-bundle_promoted_0" [ style = bold]
+"redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_running_0" -> "redis-bundle-master_promote_0" [ style = bold]
+"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = bold]
+"redis-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = bold]
+"redis-bundle-master_start_0" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
+"redis-bundle-master_start_0" -> "redis:1_start_0 redis-bundle-1" [ style = bold]
+"redis-bundle-master_start_0" -> "redis:2_start_0 redis-bundle-2" [ style = bold]
+"redis-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold]
+"redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle_promoted_0" -> "storage-clone_start_0" [ style = bold]
+"redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle_running_0" -> "redis-bundle_promote_0" [ style = bold]
+"redis-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle_start_0" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
+"redis-bundle_start_0" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
+"redis-bundle_start_0" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
+"redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = bold]
+"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
+"redis:0_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
+"redis:0_promote_0 redis-bundle-0" -> "redis-bundle-master_promoted_0" [ style = bold]
+"redis:0_promote_0 redis-bundle-0" -> "redis:0_monitor_20000 redis-bundle-0" [ style = bold]
+"redis:0_promote_0 redis-bundle-0" -> "redis:1_promote_0 redis-bundle-1" [ style = bold]
+"redis:0_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
+"redis:0_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = bold]
+"redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_20000 redis-bundle-0" [ style = bold]
+"redis:0_start_0 redis-bundle-0" -> "redis:0_promote_0 redis-bundle-0" [ style = bold]
+"redis:0_start_0 redis-bundle-0" -> "redis:1_start_0 redis-bundle-1" [ style = bold]
+"redis:0_start_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
+"redis:1_monitor_20000 redis-bundle-1" [ style=bold color="green" fontcolor="black"]
+"redis:1_promote_0 redis-bundle-1" -> "redis-bundle-master_promoted_0" [ style = bold]
+"redis:1_promote_0 redis-bundle-1" -> "redis:1_monitor_20000 redis-bundle-1" [ style = bold]
+"redis:1_promote_0 redis-bundle-1" -> "redis:2_promote_0 redis-bundle-2" [ style = bold]
+"redis:1_promote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"]
+"redis:1_start_0 redis-bundle-1" -> "redis-bundle-master_running_0" [ style = bold]
+"redis:1_start_0 redis-bundle-1" -> "redis:1_monitor_20000 redis-bundle-1" [ style = bold]
+"redis:1_start_0 redis-bundle-1" -> "redis:1_promote_0 redis-bundle-1" [ style = bold]
+"redis:1_start_0 redis-bundle-1" -> "redis:2_start_0 redis-bundle-2" [ style = bold]
+"redis:1_start_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"]
+"redis:2_monitor_20000 redis-bundle-2" [ style=bold color="green" fontcolor="black"]
+"redis:2_promote_0 redis-bundle-2" -> "redis-bundle-master_promoted_0" [ style = bold]
+"redis:2_promote_0 redis-bundle-2" -> "redis:2_monitor_20000 redis-bundle-2" [ style = bold]
+"redis:2_promote_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"]
+"redis:2_start_0 redis-bundle-2" -> "redis-bundle-master_running_0" [ style = bold]
+"redis:2_start_0 redis-bundle-2" -> "redis:2_monitor_20000 redis-bundle-2" [ style = bold]
+"redis:2_start_0 redis-bundle-2" -> "redis:2_promote_0 redis-bundle-2" [ style = bold]
+"redis:2_start_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"]
+"storage-clone_confirmed-post_notify_running_0" -> "galera-bundle_start_0" [ style = bold]
+"storage-clone_confirmed-post_notify_running_0" -> "storage:0_monitor_30000 metal-1" [ style = bold]
+"storage-clone_confirmed-post_notify_running_0" -> "storage:1_monitor_30000 metal-2" [ style = bold]
+"storage-clone_confirmed-post_notify_running_0" -> "storage:2_monitor_30000 metal-3" [ style = bold]
+"storage-clone_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"]
+"storage-clone_confirmed-pre_notify_start_0" -> "storage-clone_post_notify_running_0" [ style = bold]
+"storage-clone_confirmed-pre_notify_start_0" -> "storage-clone_start_0" [ style = bold]
+"storage-clone_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"]
+"storage-clone_post_notify_running_0" -> "storage-clone_confirmed-post_notify_running_0" [ style = bold]
+"storage-clone_post_notify_running_0" -> "storage:0_post_notify_start_0 metal-1" [ style = bold]
+"storage-clone_post_notify_running_0" -> "storage:1_post_notify_start_0 metal-2" [ style = bold]
+"storage-clone_post_notify_running_0" -> "storage:2_post_notify_start_0 metal-3" [ style = bold]
+"storage-clone_post_notify_running_0" [ style=bold color="green" fontcolor="orange"]
+"storage-clone_pre_notify_start_0" -> "storage-clone_confirmed-pre_notify_start_0" [ style = bold]
+"storage-clone_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"]
+"storage-clone_running_0" -> "storage-clone_post_notify_running_0" [ style = bold]
+"storage-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"storage-clone_start_0" -> "storage-clone_running_0" [ style = bold]
+"storage-clone_start_0" -> "storage:0_start_0 metal-1" [ style = bold]
+"storage-clone_start_0" -> "storage:1_start_0 metal-2" [ style = bold]
+"storage-clone_start_0" -> "storage:2_start_0 metal-3" [ style = bold]
+"storage-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"storage:0_monitor_0 metal-1" -> "storage-clone_start_0" [ style = bold]
+"storage:0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"storage:0_monitor_30000 metal-1" [ style=bold color="green" fontcolor="black"]
+"storage:0_post_notify_start_0 metal-1" -> "storage-clone_confirmed-post_notify_running_0" [ style = bold]
+"storage:0_post_notify_start_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"storage:0_start_0 metal-1" -> "storage-clone_running_0" [ style = bold]
+"storage:0_start_0 metal-1" -> "storage:0_monitor_30000 metal-1" [ style = bold]
+"storage:0_start_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"storage:1_monitor_0 metal-2" -> "storage-clone_start_0" [ style = bold]
+"storage:1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"storage:1_monitor_30000 metal-2" [ style=bold color="green" fontcolor="black"]
+"storage:1_post_notify_start_0 metal-2" -> "storage-clone_confirmed-post_notify_running_0" [ style = bold]
+"storage:1_post_notify_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"storage:1_start_0 metal-2" -> "storage-clone_running_0" [ style = bold]
+"storage:1_start_0 metal-2" -> "storage:1_monitor_30000 metal-2" [ style = bold]
+"storage:1_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"storage:2_monitor_0 metal-3" -> "storage-clone_start_0" [ style = bold]
+"storage:2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"storage:2_monitor_30000 metal-3" [ style=bold color="green" fontcolor="black"]
+"storage:2_post_notify_start_0 metal-3" -> "storage-clone_confirmed-post_notify_running_0" [ style = bold]
+"storage:2_post_notify_start_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"storage:2_start_0 metal-3" -> "storage-clone_running_0" [ style = bold]
+"storage:2_start_0 metal-3" -> "storage:2_monitor_30000 metal-3" [ style = bold]
+"storage:2_start_0 metal-3" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/pengine/test10/bundle-order-startup-clone-2.exp b/pengine/test10/bundle-order-startup-clone-2.exp
new file mode 100644
index 0000000000..d05eb96b69
--- /dev/null
+++ b/pengine/test10/bundle-order-startup-clone-2.exp
@@ -0,0 +1,1697 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0" priority="1000000">
+ <action_set>
+ <rsc_op id="145" operation="notify" operation_key="storage:0_post_notify_start_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:0 storage:1 storage:2 storage:3" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="storage:0 storage:1 storage:2" CRM_meta_notify_start_uname="metal-1 metal-2 metal-3" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="42" operation="notify" operation_key="storage-clone_post_notify_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="33" operation="monitor" operation_key="storage:0_monitor_30000" on_node="metal-1" on_node_uuid="1">
+ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_name="monitor" CRM_meta_notify="true" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="32" operation="start" operation_key="storage:0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-post_notify_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <rsc_op id="32" operation="start" operation_key="storage:0_start_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:0 storage:1 storage:2 storage:3" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="storage:0 storage:1 storage:2" CRM_meta_notify_start_uname="metal-1 metal-2 metal-3" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="38" operation="start" operation_key="storage-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <rsc_op id="2" operation="monitor" operation_key="storage:0_monitor_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="4" priority="1000000">
+ <action_set>
+ <rsc_op id="146" operation="notify" operation_key="storage:1_post_notify_start_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:0 storage:1 storage:2 storage:3" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="storage:0 storage:1 storage:2" CRM_meta_notify_start_uname="metal-1 metal-2 metal-3" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="42" operation="notify" operation_key="storage-clone_post_notify_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <rsc_op id="35" operation="monitor" operation_key="storage:1_monitor_30000" on_node="metal-2" on_node_uuid="2">
+ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_name="monitor" CRM_meta_notify="true" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="34" operation="start" operation_key="storage:1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-post_notify_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="6">
+ <action_set>
+ <rsc_op id="34" operation="start" operation_key="storage:1_start_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:0 storage:1 storage:2 storage:3" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="storage:0 storage:1 storage:2" CRM_meta_notify_start_uname="metal-1 metal-2 metal-3" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="38" operation="start" operation_key="storage-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <rsc_op id="12" operation="monitor" operation_key="storage:1_monitor_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="8" priority="1000000">
+ <action_set>
+ <rsc_op id="147" operation="notify" operation_key="storage:2_post_notify_start_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:0 storage:1 storage:2 storage:3" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="storage:0 storage:1 storage:2" CRM_meta_notify_start_uname="metal-1 metal-2 metal-3" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="42" operation="notify" operation_key="storage-clone_post_notify_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="9">
+ <action_set>
+ <rsc_op id="37" operation="monitor" operation_key="storage:2_monitor_30000" on_node="metal-3" on_node_uuid="3">
+ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_name="monitor" CRM_meta_notify="true" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="36" operation="start" operation_key="storage:2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-post_notify_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="10">
+ <action_set>
+ <rsc_op id="36" operation="start" operation_key="storage:2_start_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:0 storage:1 storage:2 storage:3" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="storage:0 storage:1 storage:2" CRM_meta_notify_start_uname="metal-1 metal-2 metal-3" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="38" operation="start" operation_key="storage-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="11">
+ <action_set>
+ <rsc_op id="22" operation="monitor" operation_key="storage:2_monitor_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="12" priority="1000000">
+ <action_set>
+ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-post_notify_running_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="42" operation="notify" operation_key="storage-clone_post_notify_running_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="145" operation="notify" operation_key="storage:0_post_notify_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="146" operation="notify" operation_key="storage:1_post_notify_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="147" operation="notify" operation_key="storage:2_post_notify_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="13" priority="1000000">
+ <action_set>
+ <pseudo_event id="42" operation="notify" operation_key="storage-clone_post_notify_running_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="39" operation="running" operation_key="storage-clone_running_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="41" operation="notified" operation_key="storage-clone_confirmed-pre_notify_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="14">
+ <action_set>
+ <pseudo_event id="41" operation="notified" operation_key="storage-clone_confirmed-pre_notify_start_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="40" operation="notify" operation_key="storage-clone_pre_notify_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="15">
+ <action_set>
+ <pseudo_event id="40" operation="notify" operation_key="storage-clone_pre_notify_start_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="16" priority="1000000">
+ <action_set>
+ <pseudo_event id="39" operation="running" operation_key="storage-clone_running_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="32" operation="start" operation_key="storage:0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="34" operation="start" operation_key="storage:1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="36" operation="start" operation_key="storage:2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="38" operation="start" operation_key="storage-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="17">
+ <action_set>
+ <pseudo_event id="38" operation="start" operation_key="storage-clone_start_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="2" operation="monitor" operation_key="storage:0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="12" operation="monitor" operation_key="storage:1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="22" operation="monitor" operation_key="storage:2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="41" operation="notified" operation_key="storage-clone_confirmed-pre_notify_start_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="94" operation="running" operation_key="haproxy-bundle_running_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="131" operation="promoted" operation_key="redis-bundle_promoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="18">
+ <action_set>
+ <rsc_op id="68" operation="monitor" operation_key="galera:0_monitor_30000" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1">
+ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_role="Slave" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="52" operation="start" operation_key="galera-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="66" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="19">
+ <action_set>
+ <rsc_op id="67" operation="monitor" operation_key="galera:0_monitor_20000" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1">
+ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="52" operation="start" operation_key="galera-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="66" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="20">
+ <action_set>
+ <rsc_op id="66" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1">
+ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="50" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="52" operation="start" operation_key="galera-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="75" operation="start" operation_key="galera-bundle-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="21">
+ <action_set>
+ <rsc_op id="51" operation="monitor" operation_key="galera-bundle-docker-0_monitor_60000" on_node="metal-1" on_node_uuid="1">
+ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="50" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="22">
+ <action_set>
+ <rsc_op id="50" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="3" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="4" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="5" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="13" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="14" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="15" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="23" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="24" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="25" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="62" operation="start" operation_key="galera-bundle_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="23">
+ <action_set>
+ <rsc_op id="23" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="24">
+ <action_set>
+ <rsc_op id="13" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="25">
+ <action_set>
+ <rsc_op id="3" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="26">
+ <action_set>
+ <rsc_op id="53" operation="monitor" operation_key="galera-bundle-0_monitor_60000" on_node="metal-1" on_node_uuid="1">
+ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="metal-1" port="3123"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="52" operation="start" operation_key="galera-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="27">
+ <action_set>
+ <rsc_op id="52" operation="start" operation_key="galera-bundle-0_start_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="metal-1" port="3123"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="50" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="28">
+ <action_set>
+ <rsc_op id="71" operation="monitor" operation_key="galera:1_monitor_30000" on_node="galera-bundle-1" on_node_uuid="galera-bundle-1" router_node="metal-2">
+ <primitive id="galera" long-id="galera:1" class="ocf" provider="heartbeat" type="galera"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-1" CRM_meta_on_node_uuid="galera-bundle-1" CRM_meta_role="Slave" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="56" operation="start" operation_key="galera-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="69" operation="start" operation_key="galera:1_start_0" on_node="galera-bundle-1" on_node_uuid="galera-bundle-1" router_node="metal-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="29">
+ <action_set>
+ <rsc_op id="70" operation="monitor" operation_key="galera:1_monitor_20000" on_node="galera-bundle-1" on_node_uuid="galera-bundle-1" router_node="metal-2">
+ <primitive id="galera" long-id="galera:1" class="ocf" provider="heartbeat" type="galera"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-1" CRM_meta_on_node_uuid="galera-bundle-1" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="56" operation="start" operation_key="galera-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="69" operation="start" operation_key="galera:1_start_0" on_node="galera-bundle-1" on_node_uuid="galera-bundle-1" router_node="metal-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="30">
+ <action_set>
+ <rsc_op id="69" operation="start" operation_key="galera:1_start_0" on_node="galera-bundle-1" on_node_uuid="galera-bundle-1" router_node="metal-2">
+ <primitive id="galera" long-id="galera:1" class="ocf" provider="heartbeat" type="galera"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-1" CRM_meta_on_node_uuid="galera-bundle-1" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="54" operation="start" operation_key="galera-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="56" operation="start" operation_key="galera-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="66" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="75" operation="start" operation_key="galera-bundle-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="31">
+ <action_set>
+ <rsc_op id="55" operation="monitor" operation_key="galera-bundle-docker-1_monitor_60000" on_node="metal-2" on_node_uuid="2">
+ <primitive id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-1:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="54" operation="start" operation_key="galera-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="32">
+ <action_set>
+ <rsc_op id="54" operation="start" operation_key="galera-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-1:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="3" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="4" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="5" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="13" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="14" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="15" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="23" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="24" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="25" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="62" operation="start" operation_key="galera-bundle_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="33">
+ <action_set>
+ <rsc_op id="24" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-1:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="34">
+ <action_set>
+ <rsc_op id="14" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-1:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="35">
+ <action_set>
+ <rsc_op id="4" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-1:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="36">
+ <action_set>
+ <rsc_op id="57" operation="monitor" operation_key="galera-bundle-1_monitor_60000" on_node="metal-2" on_node_uuid="2">
+ <primitive id="galera-bundle-1" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="galera-bundle-docker-1" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="metal-2" port="3123"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="56" operation="start" operation_key="galera-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="37">
+ <action_set>
+ <rsc_op id="56" operation="start" operation_key="galera-bundle-1_start_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="galera-bundle-1" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="galera-bundle-docker-1" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="metal-2" port="3123"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="54" operation="start" operation_key="galera-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="38">
+ <action_set>
+ <rsc_op id="74" operation="monitor" operation_key="galera:2_monitor_30000" on_node="galera-bundle-2" on_node_uuid="galera-bundle-2" router_node="metal-3">
+ <primitive id="galera" long-id="galera:2" class="ocf" provider="heartbeat" type="galera"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-2" CRM_meta_on_node_uuid="galera-bundle-2" CRM_meta_role="Slave" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="60" operation="start" operation_key="galera-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="72" operation="start" operation_key="galera:2_start_0" on_node="galera-bundle-2" on_node_uuid="galera-bundle-2" router_node="metal-3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="39">
+ <action_set>
+ <rsc_op id="73" operation="monitor" operation_key="galera:2_monitor_20000" on_node="galera-bundle-2" on_node_uuid="galera-bundle-2" router_node="metal-3">
+ <primitive id="galera" long-id="galera:2" class="ocf" provider="heartbeat" type="galera"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-2" CRM_meta_on_node_uuid="galera-bundle-2" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="60" operation="start" operation_key="galera-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="72" operation="start" operation_key="galera:2_start_0" on_node="galera-bundle-2" on_node_uuid="galera-bundle-2" router_node="metal-3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="40">
+ <action_set>
+ <rsc_op id="72" operation="start" operation_key="galera:2_start_0" on_node="galera-bundle-2" on_node_uuid="galera-bundle-2" router_node="metal-3">
+ <primitive id="galera" long-id="galera:2" class="ocf" provider="heartbeat" type="galera"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-2" CRM_meta_on_node_uuid="galera-bundle-2" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="58" operation="start" operation_key="galera-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="60" operation="start" operation_key="galera-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="69" operation="start" operation_key="galera:1_start_0" on_node="galera-bundle-1" on_node_uuid="galera-bundle-1" router_node="metal-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="75" operation="start" operation_key="galera-bundle-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="41">
+ <action_set>
+ <rsc_op id="59" operation="monitor" operation_key="galera-bundle-docker-2_monitor_60000" on_node="metal-3" on_node_uuid="3">
+ <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-2:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="58" operation="start" operation_key="galera-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="42">
+ <action_set>
+ <rsc_op id="58" operation="start" operation_key="galera-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-2:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="3" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="4" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="5" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="13" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="14" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="15" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="23" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="24" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="25" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="62" operation="start" operation_key="galera-bundle_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="43">
+ <action_set>
+ <rsc_op id="25" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-2:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="44">
+ <action_set>
+ <rsc_op id="15" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-2:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="45">
+ <action_set>
+ <rsc_op id="5" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-2:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="46">
+ <action_set>
+ <rsc_op id="61" operation="monitor" operation_key="galera-bundle-2_monitor_60000" on_node="metal-3" on_node_uuid="3">
+ <primitive id="galera-bundle-2" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="galera-bundle-docker-2" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" addr="metal-3" port="3123"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="60" operation="start" operation_key="galera-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="47">
+ <action_set>
+ <rsc_op id="60" operation="start" operation_key="galera-bundle-2_start_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="galera-bundle-2" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="galera-bundle-docker-2" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" addr="metal-3" port="3123"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="58" operation="start" operation_key="galera-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="48">
+ <action_set>
+ <rsc_op id="88" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_60000" on_node="metal-1" on_node_uuid="1">
+ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="87" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="49">
+ <action_set>
+ <rsc_op id="87" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="6" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="7" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="8" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="16" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="17" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="18" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="26" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="27" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="28" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="93" operation="start" operation_key="haproxy-bundle_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="50">
+ <action_set>
+ <rsc_op id="26" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="51">
+ <action_set>
+ <rsc_op id="16" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="52">
+ <action_set>
+ <rsc_op id="6" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="53">
+ <action_set>
+ <rsc_op id="90" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_60000" on_node="metal-2" on_node_uuid="2">
+ <primitive id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="89" operation="start" operation_key="haproxy-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="54">
+ <action_set>
+ <rsc_op id="89" operation="start" operation_key="haproxy-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="6" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="7" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="8" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="16" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="17" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="18" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="26" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="27" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="28" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="93" operation="start" operation_key="haproxy-bundle_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="55">
+ <action_set>
+ <rsc_op id="27" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="56">
+ <action_set>
+ <rsc_op id="17" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="57">
+ <action_set>
+ <rsc_op id="7" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="58">
+ <action_set>
+ <rsc_op id="92" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_60000" on_node="metal-3" on_node_uuid="3">
+ <primitive id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="91" operation="start" operation_key="haproxy-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="59">
+ <action_set>
+ <rsc_op id="91" operation="start" operation_key="haproxy-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="6" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="7" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="8" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="16" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="17" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="18" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="26" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="27" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="28" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="93" operation="start" operation_key="haproxy-bundle_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="60">
+ <action_set>
+ <rsc_op id="28" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="61">
+ <action_set>
+ <rsc_op id="18" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="62">
+ <action_set>
+ <rsc_op id="8" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="63">
+ <action_set>
+ <rsc_op id="115" operation="monitor" operation_key="redis:0_monitor_20000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_op_target_rc="8" CRM_meta_role="Master" CRM_meta_timeout="60000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="99" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="113" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="114" operation="promote" operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="64">
+ <action_set>
+ <rsc_op id="114" operation="promote" operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="120000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="97" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="99" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="113" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="126" operation="promote" operation_key="redis-bundle-master_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="65">
+ <action_set>
+ <rsc_op id="113" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="200000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="97" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="99" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="122" operation="start" operation_key="redis-bundle-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="66">
+ <action_set>
+ <rsc_op id="98" operation="monitor" operation_key="redis-bundle-docker-0_monitor_60000" on_node="metal-1" on_node_uuid="1">
+ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="97" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="67">
+ <action_set>
+ <rsc_op id="97" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="9" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="10" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="11" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="19" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="20" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="21" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="29" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="30" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="31" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="109" operation="start" operation_key="redis-bundle_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="68">
+ <action_set>
+ <rsc_op id="29" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="69">
+ <action_set>
+ <rsc_op id="19" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="70">
+ <action_set>
+ <rsc_op id="9" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="71">
+ <action_set>
+ <rsc_op id="100" operation="monitor" operation_key="redis-bundle-0_monitor_60000" on_node="metal-1" on_node_uuid="1">
+ <primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="redis-bundle-docker-0" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="metal-1" port="3124"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="99" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="72">
+ <action_set>
+ <rsc_op id="99" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="redis-bundle-docker-0" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="metal-1" port="3124"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="97" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="73">
+ <action_set>
+ <rsc_op id="118" operation="monitor" operation_key="redis:1_monitor_20000" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_op_target_rc="8" CRM_meta_role="Master" CRM_meta_timeout="60000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="103" operation="start" operation_key="redis-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="116" operation="start" operation_key="redis:1_start_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="117" operation="promote" operation_key="redis:1_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="74">
+ <action_set>
+ <rsc_op id="117" operation="promote" operation_key="redis:1_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_timeout="120000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="101" operation="start" operation_key="redis-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="103" operation="start" operation_key="redis-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="114" operation="promote" operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="116" operation="start" operation_key="redis:1_start_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="126" operation="promote" operation_key="redis-bundle-master_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="75">
+ <action_set>
+ <rsc_op id="116" operation="start" operation_key="redis:1_start_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_timeout="200000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="101" operation="start" operation_key="redis-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="103" operation="start" operation_key="redis-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="113" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="122" operation="start" operation_key="redis-bundle-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="76">
+ <action_set>
+ <rsc_op id="102" operation="monitor" operation_key="redis-bundle-docker-1_monitor_60000" on_node="metal-2" on_node_uuid="2">
+ <primitive id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-1:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="101" operation="start" operation_key="redis-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="77">
+ <action_set>
+ <rsc_op id="101" operation="start" operation_key="redis-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-1:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="9" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="10" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="11" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="19" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="20" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="21" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="29" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="30" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="31" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="109" operation="start" operation_key="redis-bundle_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="78">
+ <action_set>
+ <rsc_op id="30" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-1:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="79">
+ <action_set>
+ <rsc_op id="20" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-1:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="80">
+ <action_set>
+ <rsc_op id="10" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-1:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="81">
+ <action_set>
+ <rsc_op id="104" operation="monitor" operation_key="redis-bundle-1_monitor_60000" on_node="metal-2" on_node_uuid="2">
+ <primitive id="redis-bundle-1" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="redis-bundle-docker-1" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="metal-2" port="3124"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="103" operation="start" operation_key="redis-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="82">
+ <action_set>
+ <rsc_op id="103" operation="start" operation_key="redis-bundle-1_start_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="redis-bundle-1" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="redis-bundle-docker-1" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="metal-2" port="3124"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="101" operation="start" operation_key="redis-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="83">
+ <action_set>
+ <rsc_op id="121" operation="monitor" operation_key="redis:2_monitor_20000" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_op_target_rc="8" CRM_meta_role="Master" CRM_meta_timeout="60000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="107" operation="start" operation_key="redis-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="119" operation="start" operation_key="redis:2_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="120" operation="promote" operation_key="redis:2_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="84">
+ <action_set>
+ <rsc_op id="120" operation="promote" operation_key="redis:2_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_timeout="120000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="105" operation="start" operation_key="redis-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="107" operation="start" operation_key="redis-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="117" operation="promote" operation_key="redis:1_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="119" operation="start" operation_key="redis:2_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="126" operation="promote" operation_key="redis-bundle-master_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="85">
+ <action_set>
+ <rsc_op id="119" operation="start" operation_key="redis:2_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_timeout="200000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="105" operation="start" operation_key="redis-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="107" operation="start" operation_key="redis-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="116" operation="start" operation_key="redis:1_start_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="122" operation="start" operation_key="redis-bundle-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="86">
+ <action_set>
+ <rsc_op id="106" operation="monitor" operation_key="redis-bundle-docker-2_monitor_60000" on_node="metal-3" on_node_uuid="3">
+ <primitive id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-2:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="105" operation="start" operation_key="redis-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="87">
+ <action_set>
+ <rsc_op id="105" operation="start" operation_key="redis-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-2:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="9" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="10" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="11" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="19" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="20" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="21" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="29" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="30" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="31" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="109" operation="start" operation_key="redis-bundle_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="88">
+ <action_set>
+ <rsc_op id="31" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-2:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="89">
+ <action_set>
+ <rsc_op id="21" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-2:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="90">
+ <action_set>
+ <rsc_op id="11" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-2:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="91">
+ <action_set>
+ <rsc_op id="108" operation="monitor" operation_key="redis-bundle-2_monitor_60000" on_node="metal-3" on_node_uuid="3">
+ <primitive id="redis-bundle-2" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="redis-bundle-docker-2" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" addr="metal-3" port="3124"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="107" operation="start" operation_key="redis-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="92">
+ <action_set>
+ <rsc_op id="107" operation="start" operation_key="redis-bundle-2_start_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="redis-bundle-2" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="redis-bundle-docker-2" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" addr="metal-3" port="3124"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="105" operation="start" operation_key="redis-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="93" priority="1000000">
+ <action_set>
+ <pseudo_event id="131" operation="promoted" operation_key="redis-bundle_promoted_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="127" operation="promoted" operation_key="redis-bundle-master_promoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="94">
+ <action_set>
+ <pseudo_event id="130" operation="promote" operation_key="redis-bundle_promote_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="110" operation="running" operation_key="redis-bundle_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="95" priority="1000000">
+ <action_set>
+ <pseudo_event id="127" operation="promoted" operation_key="redis-bundle-master_promoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="114" operation="promote" operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="117" operation="promote" operation_key="redis:1_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="120" operation="promote" operation_key="redis:2_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="96">
+ <action_set>
+ <pseudo_event id="126" operation="promote" operation_key="redis-bundle-master_promote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="123" operation="running" operation_key="redis-bundle-master_running_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="130" operation="promote" operation_key="redis-bundle_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="97" priority="1000000">
+ <action_set>
+ <pseudo_event id="123" operation="running" operation_key="redis-bundle-master_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="113" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="116" operation="start" operation_key="redis:1_start_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="119" operation="start" operation_key="redis:2_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="122" operation="start" operation_key="redis-bundle-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="98">
+ <action_set>
+ <pseudo_event id="122" operation="start" operation_key="redis-bundle-master_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="109" operation="start" operation_key="redis-bundle_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="99" priority="1000000">
+ <action_set>
+ <pseudo_event id="110" operation="running" operation_key="redis-bundle_running_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="97" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="101" operation="start" operation_key="redis-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="105" operation="start" operation_key="redis-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="123" operation="running" operation_key="redis-bundle-master_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="100">
+ <action_set>
+ <pseudo_event id="109" operation="start" operation_key="redis-bundle_start_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="101" priority="1000000">
+ <action_set>
+ <pseudo_event id="94" operation="running" operation_key="haproxy-bundle_running_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="87" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="89" operation="start" operation_key="haproxy-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="91" operation="start" operation_key="haproxy-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="102">
+ <action_set>
+ <pseudo_event id="93" operation="start" operation_key="haproxy-bundle_start_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="103" priority="1000000">
+ <action_set>
+ <pseudo_event id="76" operation="running" operation_key="galera-bundle-master_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="66" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="69" operation="start" operation_key="galera:1_start_0" on_node="galera-bundle-1" on_node_uuid="galera-bundle-1" router_node="metal-2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="72" operation="start" operation_key="galera:2_start_0" on_node="galera-bundle-2" on_node_uuid="galera-bundle-2" router_node="metal-3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="75" operation="start" operation_key="galera-bundle-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="104">
+ <action_set>
+ <pseudo_event id="75" operation="start" operation_key="galera-bundle-master_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="62" operation="start" operation_key="galera-bundle_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="105" priority="1000000">
+ <action_set>
+ <pseudo_event id="63" operation="running" operation_key="galera-bundle_running_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="50" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="54" operation="start" operation_key="galera-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="58" operation="start" operation_key="galera-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="76" operation="running" operation_key="galera-bundle-master_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="106">
+ <action_set>
+ <pseudo_event id="62" operation="start" operation_key="galera-bundle_start_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-post_notify_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/bundle-order-startup-clone-2.scores b/pengine/test10/bundle-order-startup-clone-2.scores
new file mode 100644
index 0000000000..493cd87391
--- /dev/null
+++ b/pengine/test10/bundle-order-startup-clone-2.scores
@@ -0,0 +1,584 @@
+Allocation scores:
+clone_color: galera-bundle-master allocation score on galera-bundle-0: 0
+clone_color: galera-bundle-master allocation score on galera-bundle-1: 0
+clone_color: galera-bundle-master allocation score on galera-bundle-2: 0
+clone_color: galera-bundle-master allocation score on metal-1: -INFINITY
+clone_color: galera-bundle-master allocation score on metal-2: -INFINITY
+clone_color: galera-bundle-master allocation score on metal-3: -INFINITY
+clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: galera:0 allocation score on galera-bundle-0: INFINITY
+clone_color: galera:0 allocation score on galera-bundle-1: -INFINITY
+clone_color: galera:0 allocation score on galera-bundle-2: -INFINITY
+clone_color: galera:0 allocation score on metal-1: -INFINITY
+clone_color: galera:0 allocation score on metal-2: -INFINITY
+clone_color: galera:0 allocation score on metal-3: -INFINITY
+clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: galera:1 allocation score on galera-bundle-0: -INFINITY
+clone_color: galera:1 allocation score on galera-bundle-1: INFINITY
+clone_color: galera:1 allocation score on galera-bundle-2: -INFINITY
+clone_color: galera:1 allocation score on metal-1: -INFINITY
+clone_color: galera:1 allocation score on metal-2: -INFINITY
+clone_color: galera:1 allocation score on metal-3: -INFINITY
+clone_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: galera:2 allocation score on galera-bundle-0: -INFINITY
+clone_color: galera:2 allocation score on galera-bundle-1: -INFINITY
+clone_color: galera:2 allocation score on galera-bundle-2: INFINITY
+clone_color: galera:2 allocation score on metal-1: -INFINITY
+clone_color: galera:2 allocation score on metal-2: -INFINITY
+clone_color: galera:2 allocation score on metal-3: -INFINITY
+clone_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
+clone_color: redis-bundle-master allocation score on galera-bundle-1: -INFINITY
+clone_color: redis-bundle-master allocation score on galera-bundle-2: -INFINITY
+clone_color: redis-bundle-master allocation score on metal-1: -INFINITY
+clone_color: redis-bundle-master allocation score on metal-2: -INFINITY
+clone_color: redis-bundle-master allocation score on metal-3: -INFINITY
+clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: redis-bundle-master allocation score on redis-bundle-0: 0
+clone_color: redis-bundle-master allocation score on redis-bundle-1: 0
+clone_color: redis-bundle-master allocation score on redis-bundle-2: 0
+clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY
+clone_color: redis:0 allocation score on galera-bundle-1: -INFINITY
+clone_color: redis:0 allocation score on galera-bundle-2: -INFINITY
+clone_color: redis:0 allocation score on metal-1: -INFINITY
+clone_color: redis:0 allocation score on metal-2: -INFINITY
+clone_color: redis:0 allocation score on metal-3: -INFINITY
+clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: redis:0 allocation score on redis-bundle-0: INFINITY
+clone_color: redis:0 allocation score on redis-bundle-1: -INFINITY
+clone_color: redis:0 allocation score on redis-bundle-2: -INFINITY
+clone_color: redis:1 allocation score on galera-bundle-0: -INFINITY
+clone_color: redis:1 allocation score on galera-bundle-1: -INFINITY
+clone_color: redis:1 allocation score on galera-bundle-2: -INFINITY
+clone_color: redis:1 allocation score on metal-1: -INFINITY
+clone_color: redis:1 allocation score on metal-2: -INFINITY
+clone_color: redis:1 allocation score on metal-3: -INFINITY
+clone_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: redis:1 allocation score on redis-bundle-0: -INFINITY
+clone_color: redis:1 allocation score on redis-bundle-1: INFINITY
+clone_color: redis:1 allocation score on redis-bundle-2: -INFINITY
+clone_color: redis:2 allocation score on galera-bundle-0: -INFINITY
+clone_color: redis:2 allocation score on galera-bundle-1: -INFINITY
+clone_color: redis:2 allocation score on galera-bundle-2: -INFINITY
+clone_color: redis:2 allocation score on metal-1: -INFINITY
+clone_color: redis:2 allocation score on metal-2: -INFINITY
+clone_color: redis:2 allocation score on metal-3: -INFINITY
+clone_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: redis:2 allocation score on redis-bundle-0: -INFINITY
+clone_color: redis:2 allocation score on redis-bundle-1: -INFINITY
+clone_color: redis:2 allocation score on redis-bundle-2: INFINITY
+clone_color: storage-clone allocation score on metal-1: 0
+clone_color: storage-clone allocation score on metal-2: 0
+clone_color: storage-clone allocation score on metal-3: 0
+clone_color: storage-clone allocation score on rabbitmq-bundle-0: 0
+clone_color: storage:0 allocation score on metal-1: 0
+clone_color: storage:0 allocation score on metal-2: 0
+clone_color: storage:0 allocation score on metal-3: 0
+clone_color: storage:0 allocation score on rabbitmq-bundle-0: 0
+clone_color: storage:1 allocation score on metal-1: 0
+clone_color: storage:1 allocation score on metal-2: 0
+clone_color: storage:1 allocation score on metal-3: 0
+clone_color: storage:1 allocation score on rabbitmq-bundle-0: 0
+clone_color: storage:2 allocation score on metal-1: 0
+clone_color: storage:2 allocation score on metal-2: 0
+clone_color: storage:2 allocation score on metal-3: 0
+clone_color: storage:2 allocation score on rabbitmq-bundle-0: 0
+clone_color: storage:3 allocation score on metal-1: 0
+clone_color: storage:3 allocation score on metal-2: 0
+clone_color: storage:3 allocation score on metal-3: 0
+clone_color: storage:3 allocation score on rabbitmq-bundle-0: 0
+container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle allocation score on metal-1: 0
+container_color: galera-bundle allocation score on metal-2: 0
+container_color: galera-bundle allocation score on metal-3: 0
+container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle-0 allocation score on metal-1: 0
+container_color: galera-bundle-0 allocation score on metal-2: 0
+container_color: galera-bundle-0 allocation score on metal-3: 0
+container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-1 allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-1 allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle-1 allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle-1 allocation score on metal-1: 0
+container_color: galera-bundle-1 allocation score on metal-2: 0
+container_color: galera-bundle-1 allocation score on metal-3: 0
+container_color: galera-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-2 allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-2 allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle-2 allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle-2 allocation score on metal-1: 0
+container_color: galera-bundle-2 allocation score on metal-2: 0
+container_color: galera-bundle-2 allocation score on metal-3: 0
+container_color: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle-docker-0 allocation score on metal-1: 0
+container_color: galera-bundle-docker-0 allocation score on metal-2: 0
+container_color: galera-bundle-docker-0 allocation score on metal-3: 0
+container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle-docker-1 allocation score on metal-1: 0
+container_color: galera-bundle-docker-1 allocation score on metal-2: 0
+container_color: galera-bundle-docker-1 allocation score on metal-3: 0
+container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle-docker-2 allocation score on metal-1: 0
+container_color: galera-bundle-docker-2 allocation score on metal-2: 0
+container_color: galera-bundle-docker-2 allocation score on metal-3: 0
+container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-master allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle-master allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle-master allocation score on metal-1: 0
+container_color: galera-bundle-master allocation score on metal-2: 0
+container_color: galera-bundle-master allocation score on metal-3: 0
+container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: 0
+container_color: galera:0 allocation score on galera-bundle-0: -INFINITY
+container_color: galera:0 allocation score on galera-bundle-1: -INFINITY
+container_color: galera:0 allocation score on galera-bundle-2: -INFINITY
+container_color: galera:0 allocation score on metal-1: 0
+container_color: galera:0 allocation score on metal-2: 0
+container_color: galera:0 allocation score on metal-3: 0
+container_color: galera:0 allocation score on rabbitmq-bundle-0: 0
+container_color: galera:1 allocation score on galera-bundle-0: -INFINITY
+container_color: galera:1 allocation score on galera-bundle-1: -INFINITY
+container_color: galera:1 allocation score on galera-bundle-2: -INFINITY
+container_color: galera:1 allocation score on metal-1: 0
+container_color: galera:1 allocation score on metal-2: 0
+container_color: galera:1 allocation score on metal-3: 0
+container_color: galera:1 allocation score on rabbitmq-bundle-0: 0
+container_color: galera:2 allocation score on galera-bundle-0: -INFINITY
+container_color: galera:2 allocation score on galera-bundle-1: -INFINITY
+container_color: galera:2 allocation score on galera-bundle-2: -INFINITY
+container_color: galera:2 allocation score on metal-1: 0
+container_color: galera:2 allocation score on metal-2: 0
+container_color: galera:2 allocation score on metal-3: 0
+container_color: galera:2 allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle allocation score on metal-1: 0
+container_color: haproxy-bundle allocation score on metal-1: 0
+container_color: haproxy-bundle allocation score on metal-1: 0
+container_color: haproxy-bundle allocation score on metal-1: 0
+container_color: haproxy-bundle allocation score on metal-2: 0
+container_color: haproxy-bundle allocation score on metal-2: 0
+container_color: haproxy-bundle allocation score on metal-2: 0
+container_color: haproxy-bundle allocation score on metal-2: 0
+container_color: haproxy-bundle allocation score on metal-3: 0
+container_color: haproxy-bundle allocation score on metal-3: 0
+container_color: haproxy-bundle allocation score on metal-3: 0
+container_color: haproxy-bundle allocation score on metal-3: 0
+container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on metal-1: 0
+container_color: haproxy-bundle-docker-1 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-1 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-1 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-1 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-1: 0
+container_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-2 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-2 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-2 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-2 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: 0
+container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle allocation score on metal-1: 0
+container_color: redis-bundle allocation score on metal-2: 0
+container_color: redis-bundle allocation score on metal-3: 0
+container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle allocation score on redis-bundle-2: -INFINITY
+container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-0 allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle-0 allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle-0 allocation score on metal-1: 0
+container_color: redis-bundle-0 allocation score on metal-2: 0
+container_color: redis-bundle-0 allocation score on metal-3: 0
+container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-0 allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-0 allocation score on redis-bundle-2: -INFINITY
+container_color: redis-bundle-1 allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-1 allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle-1 allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle-1 allocation score on metal-1: 0
+container_color: redis-bundle-1 allocation score on metal-2: 0
+container_color: redis-bundle-1 allocation score on metal-3: 0
+container_color: redis-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle-1 allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-1 allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-1 allocation score on redis-bundle-2: -INFINITY
+container_color: redis-bundle-2 allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-2 allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle-2 allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle-2 allocation score on metal-1: 0
+container_color: redis-bundle-2 allocation score on metal-2: 0
+container_color: redis-bundle-2 allocation score on metal-3: 0
+container_color: redis-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle-2 allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-2 allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-2 allocation score on redis-bundle-2: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on metal-1: 0
+container_color: redis-bundle-docker-0 allocation score on metal-2: 0
+container_color: redis-bundle-docker-0 allocation score on metal-3: 0
+container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on metal-1: 0
+container_color: redis-bundle-docker-1 allocation score on metal-2: 0
+container_color: redis-bundle-docker-1 allocation score on metal-3: 0
+container_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on metal-1: 0
+container_color: redis-bundle-docker-2 allocation score on metal-2: 0
+container_color: redis-bundle-docker-2 allocation score on metal-3: 0
+container_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY
+container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-master allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle-master allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle-master allocation score on metal-1: 0
+container_color: redis-bundle-master allocation score on metal-2: 0
+container_color: redis-bundle-master allocation score on metal-3: 0
+container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: 0
+container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-master allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-master allocation score on redis-bundle-2: -INFINITY
+container_color: redis:0 allocation score on galera-bundle-0: -INFINITY
+container_color: redis:0 allocation score on galera-bundle-1: -INFINITY
+container_color: redis:0 allocation score on galera-bundle-2: -INFINITY
+container_color: redis:0 allocation score on metal-1: 0
+container_color: redis:0 allocation score on metal-2: 0
+container_color: redis:0 allocation score on metal-3: 0
+container_color: redis:0 allocation score on rabbitmq-bundle-0: 0
+container_color: redis:0 allocation score on redis-bundle-0: -INFINITY
+container_color: redis:0 allocation score on redis-bundle-1: -INFINITY
+container_color: redis:0 allocation score on redis-bundle-2: -INFINITY
+container_color: redis:1 allocation score on galera-bundle-0: -INFINITY
+container_color: redis:1 allocation score on galera-bundle-1: -INFINITY
+container_color: redis:1 allocation score on galera-bundle-2: -INFINITY
+container_color: redis:1 allocation score on metal-1: 0
+container_color: redis:1 allocation score on metal-2: 0
+container_color: redis:1 allocation score on metal-3: 0
+container_color: redis:1 allocation score on rabbitmq-bundle-0: 0
+container_color: redis:1 allocation score on redis-bundle-0: -INFINITY
+container_color: redis:1 allocation score on redis-bundle-1: -INFINITY
+container_color: redis:1 allocation score on redis-bundle-2: -INFINITY
+container_color: redis:2 allocation score on galera-bundle-0: -INFINITY
+container_color: redis:2 allocation score on galera-bundle-1: -INFINITY
+container_color: redis:2 allocation score on galera-bundle-2: -INFINITY
+container_color: redis:2 allocation score on metal-1: 0
+container_color: redis:2 allocation score on metal-2: 0
+container_color: redis:2 allocation score on metal-3: 0
+container_color: redis:2 allocation score on rabbitmq-bundle-0: 0
+container_color: redis:2 allocation score on redis-bundle-0: -INFINITY
+container_color: redis:2 allocation score on redis-bundle-1: -INFINITY
+container_color: redis:2 allocation score on redis-bundle-2: -INFINITY
+galera:0 promotion score on galera-bundle-0: -1
+galera:1 promotion score on galera-bundle-1: -1
+galera:2 promotion score on galera-bundle-2: -1
+native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY
+native_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY
+native_color: galera-bundle-0 allocation score on metal-1: 10000
+native_color: galera-bundle-0 allocation score on metal-2: 0
+native_color: galera-bundle-0 allocation score on metal-3: 0
+native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera-bundle-1 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-1 allocation score on galera-bundle-1: -INFINITY
+native_color: galera-bundle-1 allocation score on galera-bundle-2: -INFINITY
+native_color: galera-bundle-1 allocation score on metal-1: 0
+native_color: galera-bundle-1 allocation score on metal-2: 10000
+native_color: galera-bundle-1 allocation score on metal-3: 0
+native_color: galera-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera-bundle-2 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-2 allocation score on galera-bundle-1: -INFINITY
+native_color: galera-bundle-2 allocation score on galera-bundle-2: -INFINITY
+native_color: galera-bundle-2 allocation score on metal-1: 0
+native_color: galera-bundle-2 allocation score on metal-2: 0
+native_color: galera-bundle-2 allocation score on metal-3: 10000
+native_color: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on metal-1: 0
+native_color: galera-bundle-docker-0 allocation score on metal-2: 0
+native_color: galera-bundle-docker-0 allocation score on metal-3: 0
+native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+native_color: galera-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+native_color: galera-bundle-docker-1 allocation score on metal-1: -INFINITY
+native_color: galera-bundle-docker-1 allocation score on metal-2: 0
+native_color: galera-bundle-docker-1 allocation score on metal-3: 0
+native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+native_color: galera-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+native_color: galera-bundle-docker-2 allocation score on metal-1: -INFINITY
+native_color: galera-bundle-docker-2 allocation score on metal-2: -INFINITY
+native_color: galera-bundle-docker-2 allocation score on metal-3: 0
+native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera:0 allocation score on galera-bundle-0: INFINITY
+native_color: galera:0 allocation score on galera-bundle-1: -INFINITY
+native_color: galera:0 allocation score on galera-bundle-2: -INFINITY
+native_color: galera:0 allocation score on metal-1: -INFINITY
+native_color: galera:0 allocation score on metal-2: -INFINITY
+native_color: galera:0 allocation score on metal-3: -INFINITY
+native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera:1 allocation score on galera-bundle-0: -INFINITY
+native_color: galera:1 allocation score on galera-bundle-1: INFINITY
+native_color: galera:1 allocation score on galera-bundle-2: -INFINITY
+native_color: galera:1 allocation score on metal-1: -INFINITY
+native_color: galera:1 allocation score on metal-2: -INFINITY
+native_color: galera:1 allocation score on metal-3: -INFINITY
+native_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera:2 allocation score on galera-bundle-0: -INFINITY
+native_color: galera:2 allocation score on galera-bundle-1: -INFINITY
+native_color: galera:2 allocation score on galera-bundle-2: INFINITY
+native_color: galera:2 allocation score on metal-1: -INFINITY
+native_color: galera:2 allocation score on metal-2: -INFINITY
+native_color: galera:2 allocation score on metal-3: -INFINITY
+native_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+native_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
+native_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
+native_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
+native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+native_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
+native_color: haproxy-bundle-docker-1 allocation score on metal-2: 0
+native_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
+native_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+native_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
+native_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
+native_color: haproxy-bundle-docker-2 allocation score on metal-3: 0
+native_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
+native_color: redis-bundle-0 allocation score on galera-bundle-1: -INFINITY
+native_color: redis-bundle-0 allocation score on galera-bundle-2: -INFINITY
+native_color: redis-bundle-0 allocation score on metal-1: 10000
+native_color: redis-bundle-0 allocation score on metal-2: 0
+native_color: redis-bundle-0 allocation score on metal-3: 0
+native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
+native_color: redis-bundle-0 allocation score on redis-bundle-1: -INFINITY
+native_color: redis-bundle-0 allocation score on redis-bundle-2: -INFINITY
+native_color: redis-bundle-1 allocation score on galera-bundle-0: -INFINITY
+native_color: redis-bundle-1 allocation score on galera-bundle-1: -INFINITY
+native_color: redis-bundle-1 allocation score on galera-bundle-2: -INFINITY
+native_color: redis-bundle-1 allocation score on metal-1: 0
+native_color: redis-bundle-1 allocation score on metal-2: 10000
+native_color: redis-bundle-1 allocation score on metal-3: 0
+native_color: redis-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-1 allocation score on redis-bundle-0: -INFINITY
+native_color: redis-bundle-1 allocation score on redis-bundle-1: -INFINITY
+native_color: redis-bundle-1 allocation score on redis-bundle-2: -INFINITY
+native_color: redis-bundle-2 allocation score on galera-bundle-0: -INFINITY
+native_color: redis-bundle-2 allocation score on galera-bundle-1: -INFINITY
+native_color: redis-bundle-2 allocation score on galera-bundle-2: -INFINITY
+native_color: redis-bundle-2 allocation score on metal-1: 0
+native_color: redis-bundle-2 allocation score on metal-2: 0
+native_color: redis-bundle-2 allocation score on metal-3: 10000
+native_color: redis-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-2 allocation score on redis-bundle-0: -INFINITY
+native_color: redis-bundle-2 allocation score on redis-bundle-1: -INFINITY
+native_color: redis-bundle-2 allocation score on redis-bundle-2: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on metal-1: 0
+native_color: redis-bundle-docker-0 allocation score on metal-2: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on metal-3: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on metal-1: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on metal-2: 0
+native_color: redis-bundle-docker-1 allocation score on metal-3: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on metal-1: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on metal-2: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on metal-3: 0
+native_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY
+native_color: redis:0 allocation score on galera-bundle-0: -INFINITY
+native_color: redis:0 allocation score on galera-bundle-1: -INFINITY
+native_color: redis:0 allocation score on galera-bundle-2: -INFINITY
+native_color: redis:0 allocation score on metal-1: -INFINITY
+native_color: redis:0 allocation score on metal-2: -INFINITY
+native_color: redis:0 allocation score on metal-3: -INFINITY
+native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis:0 allocation score on redis-bundle-0: INFINITY
+native_color: redis:0 allocation score on redis-bundle-1: -INFINITY
+native_color: redis:0 allocation score on redis-bundle-2: -INFINITY
+native_color: redis:1 allocation score on galera-bundle-0: -INFINITY
+native_color: redis:1 allocation score on galera-bundle-1: -INFINITY
+native_color: redis:1 allocation score on galera-bundle-2: -INFINITY
+native_color: redis:1 allocation score on metal-1: -INFINITY
+native_color: redis:1 allocation score on metal-2: -INFINITY
+native_color: redis:1 allocation score on metal-3: -INFINITY
+native_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis:1 allocation score on redis-bundle-0: -INFINITY
+native_color: redis:1 allocation score on redis-bundle-1: INFINITY
+native_color: redis:1 allocation score on redis-bundle-2: -INFINITY
+native_color: redis:2 allocation score on galera-bundle-0: -INFINITY
+native_color: redis:2 allocation score on galera-bundle-1: -INFINITY
+native_color: redis:2 allocation score on galera-bundle-2: -INFINITY
+native_color: redis:2 allocation score on metal-1: -INFINITY
+native_color: redis:2 allocation score on metal-2: -INFINITY
+native_color: redis:2 allocation score on metal-3: -INFINITY
+native_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis:2 allocation score on redis-bundle-0: -INFINITY
+native_color: redis:2 allocation score on redis-bundle-1: -INFINITY
+native_color: redis:2 allocation score on redis-bundle-2: INFINITY
+native_color: storage:0 allocation score on metal-1: 0
+native_color: storage:0 allocation score on metal-2: 0
+native_color: storage:0 allocation score on metal-3: 0
+native_color: storage:0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: storage:1 allocation score on metal-1: -INFINITY
+native_color: storage:1 allocation score on metal-2: 0
+native_color: storage:1 allocation score on metal-3: 0
+native_color: storage:1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: storage:2 allocation score on metal-1: -INFINITY
+native_color: storage:2 allocation score on metal-2: -INFINITY
+native_color: storage:2 allocation score on metal-3: 0
+native_color: storage:2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: storage:3 allocation score on metal-1: -INFINITY
+native_color: storage:3 allocation score on metal-2: -INFINITY
+native_color: storage:3 allocation score on metal-3: -INFINITY
+native_color: storage:3 allocation score on rabbitmq-bundle-0: -INFINITY
+redis:0 promotion score on redis-bundle-0: 99
+redis:1 promotion score on redis-bundle-1: 99
+redis:2 promotion score on redis-bundle-2: 99
diff --git a/pengine/test10/bundle-order-startup-clone-2.summary b/pengine/test10/bundle-order-startup-clone-2.summary
new file mode 100644
index 0000000000..e23d9339ca
--- /dev/null
+++ b/pengine/test10/bundle-order-startup-clone-2.summary
@@ -0,0 +1,179 @@
+
+Current cluster status:
+Online: [ metal-1 metal-2 metal-3 ]
+RemoteOFFLINE: [ rabbitmq-bundle-0 ]
+
+ Clone Set: storage-clone [storage]
+ Stopped: [ metal-1 metal-2 metal-3 rabbitmq-bundle-0 ]
+ Docker container set: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
+ galera-bundle-0 (ocf::heartbeat:galera): Stopped
+ galera-bundle-1 (ocf::heartbeat:galera): Stopped
+ galera-bundle-2 (ocf::heartbeat:galera): Stopped
+ Docker container set: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
+ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped
+ haproxy-bundle-docker-1 (ocf::heartbeat:docker): Stopped
+ haproxy-bundle-docker-2 (ocf::heartbeat:docker): Stopped
+ Docker container set: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
+ redis-bundle-0 (ocf::heartbeat:redis): Stopped
+ redis-bundle-1 (ocf::heartbeat:redis): Stopped
+ redis-bundle-2 (ocf::heartbeat:redis): Stopped
+
+Transition Summary:
+ * Start storage:0 (metal-1)
+ * Start storage:1 (metal-2)
+ * Start storage:2 (metal-3)
+ * Start galera-bundle-docker-0 (metal-1)
+ * Start galera-bundle-0 (metal-1)
+ * Start galera:0 (galera-bundle-0)
+ * Start galera-bundle-docker-1 (metal-2)
+ * Start galera-bundle-1 (metal-2)
+ * Start galera:1 (galera-bundle-1)
+ * Start galera-bundle-docker-2 (metal-3)
+ * Start galera-bundle-2 (metal-3)
+ * Start galera:2 (galera-bundle-2)
+ * Start haproxy-bundle-docker-0 (metal-1)
+ * Start haproxy-bundle-docker-1 (metal-2)
+ * Start haproxy-bundle-docker-2 (metal-3)
+ * Start redis-bundle-docker-0 (metal-1)
+ * Start redis-bundle-0 (metal-1)
+ * Start redis:0 (redis-bundle-0)
+ * Promote redis:0 (Stopped -> Master redis-bundle-0)
+ * Start redis-bundle-docker-1 (metal-2)
+ * Start redis-bundle-1 (metal-2)
+ * Start redis:1 (redis-bundle-1)
+ * Promote redis:1 (Stopped -> Master redis-bundle-1)
+ * Start redis-bundle-docker-2 (metal-3)
+ * Start redis-bundle-2 (metal-3)
+ * Start redis:2 (redis-bundle-2)
+ * Promote redis:2 (Stopped -> Master redis-bundle-2)
+
+Executing cluster transition:
+ * Resource action: storage:0 monitor on metal-1
+ * Resource action: storage:1 monitor on metal-2
+ * Resource action: storage:2 monitor on metal-3
+ * Pseudo action: storage-clone_pre_notify_start_0
+ * Resource action: galera-bundle-docker-0 monitor on metal-3
+ * Resource action: galera-bundle-docker-0 monitor on metal-2
+ * Resource action: galera-bundle-docker-0 monitor on metal-1
+ * Resource action: galera-bundle-docker-1 monitor on metal-3
+ * Resource action: galera-bundle-docker-1 monitor on metal-2
+ * Resource action: galera-bundle-docker-1 monitor on metal-1
+ * Resource action: galera-bundle-docker-2 monitor on metal-3
+ * Resource action: galera-bundle-docker-2 monitor on metal-2
+ * Resource action: galera-bundle-docker-2 monitor on metal-1
+ * Resource action: haproxy-bundle-docker-0 monitor on metal-3
+ * Resource action: haproxy-bundle-docker-0 monitor on metal-2
+ * Resource action: haproxy-bundle-docker-0 monitor on metal-1
+ * Resource action: haproxy-bundle-docker-1 monitor on metal-3
+ * Resource action: haproxy-bundle-docker-1 monitor on metal-2
+ * Resource action: haproxy-bundle-docker-1 monitor on metal-1
+ * Resource action: haproxy-bundle-docker-2 monitor on metal-3
+ * Resource action: haproxy-bundle-docker-2 monitor on metal-2
+ * Resource action: haproxy-bundle-docker-2 monitor on metal-1
+ * Resource action: redis-bundle-docker-0 monitor on metal-3
+ * Resource action: redis-bundle-docker-0 monitor on metal-2
+ * Resource action: redis-bundle-docker-0 monitor on metal-1
+ * Resource action: redis-bundle-docker-1 monitor on metal-3
+ * Resource action: redis-bundle-docker-1 monitor on metal-2
+ * Resource action: redis-bundle-docker-1 monitor on metal-1
+ * Resource action: redis-bundle-docker-2 monitor on metal-3
+ * Resource action: redis-bundle-docker-2 monitor on metal-2
+ * Resource action: redis-bundle-docker-2 monitor on metal-1
+ * Pseudo action: redis-bundle_start_0
+ * Pseudo action: haproxy-bundle_start_0
+ * Pseudo action: storage-clone_confirmed-pre_notify_start_0
+ * Resource action: haproxy-bundle-docker-0 start on metal-1
+ * Resource action: haproxy-bundle-docker-1 start on metal-2
+ * Resource action: haproxy-bundle-docker-2 start on metal-3
+ * Resource action: redis-bundle-docker-0 start on metal-1
+ * Resource action: redis-bundle-0 start on metal-1
+ * Resource action: redis-bundle-docker-1 start on metal-2
+ * Resource action: redis-bundle-1 start on metal-2
+ * Resource action: redis-bundle-docker-2 start on metal-3
+ * Resource action: redis-bundle-2 start on metal-3
+ * Pseudo action: redis-bundle-master_start_0
+ * Pseudo action: haproxy-bundle_running_0
+ * Resource action: haproxy-bundle-docker-0 monitor=60000 on metal-1
+ * Resource action: haproxy-bundle-docker-1 monitor=60000 on metal-2
+ * Resource action: haproxy-bundle-docker-2 monitor=60000 on metal-3
+ * Resource action: redis:0 start on redis-bundle-0
+ * Resource action: redis-bundle-docker-0 monitor=60000 on metal-1
+ * Resource action: redis-bundle-0 monitor=60000 on metal-1
+ * Resource action: redis:1 start on redis-bundle-1
+ * Resource action: redis-bundle-docker-1 monitor=60000 on metal-2
+ * Resource action: redis-bundle-1 monitor=60000 on metal-2
+ * Resource action: redis:2 start on redis-bundle-2
+ * Resource action: redis-bundle-docker-2 monitor=60000 on metal-3
+ * Resource action: redis-bundle-2 monitor=60000 on metal-3
+ * Pseudo action: redis-bundle-master_running_0
+ * Pseudo action: redis-bundle_running_0
+ * Pseudo action: redis-bundle_promote_0
+ * Pseudo action: redis-bundle-master_promote_0
+ * Resource action: redis:0 promote on redis-bundle-0
+ * Resource action: redis:1 promote on redis-bundle-1
+ * Resource action: redis:2 promote on redis-bundle-2
+ * Pseudo action: redis-bundle-master_promoted_0
+ * Resource action: redis:0 monitor=20000 on redis-bundle-0
+ * Resource action: redis:1 monitor=20000 on redis-bundle-1
+ * Resource action: redis:2 monitor=20000 on redis-bundle-2
+ * Pseudo action: redis-bundle_promoted_0
+ * Pseudo action: storage-clone_start_0
+ * Resource action: storage:0 start on metal-1
+ * Resource action: storage:1 start on metal-2
+ * Resource action: storage:2 start on metal-3
+ * Pseudo action: storage-clone_running_0
+ * Pseudo action: storage-clone_post_notify_running_0
+ * Resource action: storage:0 notify on metal-1
+ * Resource action: storage:1 notify on metal-2
+ * Resource action: storage:2 notify on metal-3
+ * Pseudo action: storage-clone_confirmed-post_notify_running_0
+ * Pseudo action: galera-bundle_start_0
+ * Resource action: storage:0 monitor=30000 on metal-1
+ * Resource action: storage:1 monitor=30000 on metal-2
+ * Resource action: storage:2 monitor=30000 on metal-3
+ * Resource action: galera-bundle-docker-0 start on metal-1
+ * Resource action: galera-bundle-0 start on metal-1
+ * Resource action: galera-bundle-docker-1 start on metal-2
+ * Resource action: galera-bundle-1 start on metal-2
+ * Resource action: galera-bundle-docker-2 start on metal-3
+ * Resource action: galera-bundle-2 start on metal-3
+ * Pseudo action: galera-bundle-master_start_0
+ * Resource action: galera:0 start on galera-bundle-0
+ * Resource action: galera-bundle-docker-0 monitor=60000 on metal-1
+ * Resource action: galera-bundle-0 monitor=60000 on metal-1
+ * Resource action: galera:1 start on galera-bundle-1
+ * Resource action: galera-bundle-docker-1 monitor=60000 on metal-2
+ * Resource action: galera-bundle-1 monitor=60000 on metal-2
+ * Resource action: galera:2 start on galera-bundle-2
+ * Resource action: galera-bundle-docker-2 monitor=60000 on metal-3
+ * Resource action: galera-bundle-2 monitor=60000 on metal-3
+ * Pseudo action: galera-bundle-master_running_0
+ * Pseudo action: galera-bundle_running_0
+ * Resource action: galera:0 monitor=30000 on galera-bundle-0
+ * Resource action: galera:0 monitor=20000 on galera-bundle-0
+ * Resource action: galera:1 monitor=30000 on galera-bundle-1
+ * Resource action: galera:1 monitor=20000 on galera-bundle-1
+ * Resource action: galera:2 monitor=30000 on galera-bundle-2
+ * Resource action: galera:2 monitor=20000 on galera-bundle-2
+
+Revised cluster status:
+Online: [ metal-1 metal-2 metal-3 ]
+RemoteOFFLINE: [ rabbitmq-bundle-0 ]
+Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ]
+
+ Clone Set: storage-clone [storage]
+ Started: [ metal-1 metal-2 metal-3 ]
+ Stopped: [ rabbitmq-bundle-0 ]
+ Docker container set: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
+ galera-bundle-0 (ocf::heartbeat:galera): Slave metal-1
+ galera-bundle-1 (ocf::heartbeat:galera): Slave metal-2
+ galera-bundle-2 (ocf::heartbeat:galera): Slave metal-3
+ Docker container set: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
+ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started metal-1
+ haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started metal-2
+ haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started metal-3
+ Docker container set: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
+ redis-bundle-0 (ocf::heartbeat:redis): Master metal-1
+ redis-bundle-1 (ocf::heartbeat:redis): Master metal-2
+ redis-bundle-2 (ocf::heartbeat:redis): Master metal-3
+
diff --git a/pengine/test10/bundle-order-startup-clone-2.xml b/pengine/test10/bundle-order-startup-clone-2.xml
new file mode 100644
index 0000000000..e2c248ee3b
--- /dev/null
+++ b/pengine/test10/bundle-order-startup-clone-2.xml
@@ -0,0 +1,190 @@
+<cib crm_feature_set="3.0.12" validate-with="pacemaker-2.8" epoch="58" num_updates="129" admin_epoch="0" cib-last-written="Thu Jun 1 03:59:09 2017" update-origin="undercloud" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-10.el7-94ff4df"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/>
+ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
+ </cluster_property_set>
+ <cluster_property_set id="redis_replication">
+ <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="redis-bundle-0"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="metal-1">
+ <instance_attributes id="nodes-1">
+ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/>
+ <nvpair id="nodes-1-galera-role" name="galera-role" value="true"/>
+ <nvpair id="nodes-1-redis-role" name="redis-role" value="true"/>
+ <nvpair id="nodes-1-haproxy-role" name="haproxy-role" value="true"/>
+ <nvpair id="nodes-1-cinder-volume-role" name="cinder-volume-role" value="true"/>
+ </instance_attributes>
+ </node>
+ <node id="2" uname="metal-2">
+ <instance_attributes id="nodes-2">
+ <nvpair id="nodes-2-rabbitmq-role" name="rabbitmq-role" value="true"/>
+ <nvpair id="nodes-2-galera-role" name="galera-role" value="true"/>
+ <nvpair id="nodes-2-redis-role" name="redis-role" value="true"/>
+ <nvpair id="nodes-2-haproxy-role" name="haproxy-role" value="true"/>
+ <nvpair id="nodes-2-cinder-volume-role" name="cinder-volume-role" value="true"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="metal-3">
+ <instance_attributes id="nodes-3">
+ <nvpair id="nodes-3-rabbitmq-role" name="rabbitmq-role" value="true"/>
+ <nvpair id="nodes-3-galera-role" name="galera-role" value="true"/>
+ <nvpair id="nodes-3-redis-role" name="redis-role" value="true"/>
+ <nvpair id="nodes-3-haproxy-role" name="haproxy-role" value="true"/>
+ <nvpair id="nodes-3-cinder-volume-role" name="cinder-volume-role" value="true"/>
+ </instance_attributes>
+ </node>
+ <node id="rabbitmq-bundle-0" type="remote" uname="rabbitmq-bundle-0">
+ <instance_attributes id="nodes-rabbitmq-bundle-0">
+ <nvpair id="nodes-rabbitmq-bundle-0-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@metal"/>
+ </instance_attributes>
+ </node>
+ </nodes>
+ <resources>
+ <clone id="storage-clone">
+ <meta_attributes id="storage_ma">
+ <nvpair id="storage_globally-unique" name="globally-unique" value="false"/>
+ <nvpair id="storage_notify" name="notify" value="true"/>
+ </meta_attributes>
+ <primitive class="ocf" id="storage" provider="heartbeat" type="Filesystem">
+ <operations>
+ <op id="storage_mon" interval="30s" name="monitor" timeout="30s"/>
+ </operations>
+ <instance_attributes id="storage_ia">
+ <nvpair id="storage_attr_0" name="device" value="nfs:/share/drbd_www/data/"/>
+ <nvpair id="storage_attr_1" name="directory" value="/data/www"/>
+ <nvpair id="storage_attr_2" name="fstype" value="nfs"/>
+ </instance_attributes>
+ </primitive>
+ </clone>
+ <bundle id="galera-bundle">
+ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" masters="3" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" run-command="/bin/bash /usr/local/bin/kolla_start"/>
+ <network control-port="3123"/>
+ <storage>
+ <storage-mapping id="mysql-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/mysql.json" target-dir="/var/lib/kolla/config_files/config.json"/>
+ <storage-mapping id="mysql-cfg-data" options="ro" source-dir="/var/lib/config-data/mysql" target-dir="/var/lib/kolla/config_files/src"/>
+ <storage-mapping id="mysql-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
+ <storage-mapping id="mysql-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
+ <storage-mapping id="mysql-lib" options="rw" source-dir="/var/lib/mysql" target-dir="/var/lib/mysql"/>
+ <storage-mapping id="mysql-log-mariadb" options="rw" source-dir="/var/log/mariadb" target-dir="/var/log/mariadb"/>
+ <storage-mapping id="mysql-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
+ <storage-mapping id="mysql-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
+ <storage-mapping id="mysql-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
+ <storage-mapping id="mysql-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
+ <storage-mapping id="mysql-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
+ </storage>
+ <primitive class="ocf" id="galera" provider="heartbeat" type="galera">
+ <instance_attributes id="galera-instance_attributes">
+ <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/>
+ <nvpair id="galera-instance_attributes-cluster_host_map" name="cluster_host_map" value="galera-bundle-0:metal"/>
+ <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/>
+ <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://metal"/>
+ </instance_attributes>
+ <meta_attributes id="galera-meta_attributes">
+ <nvpair id="galera-meta_attributes-master-max" name="master-max" value="1"/>
+ <nvpair id="galera-meta_attributes-ordered" name="ordered" value="true"/>
+ </meta_attributes>
+ <operations>
+ <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
+ <op id="galera-monitor-interval-20" interval="20" name="monitor" timeout="30"/>
+ <op id="galera-monitor-interval-10" interval="10" name="monitor" role="Master" timeout="30"/>
+ <op id="galera-monitor-interval-30" interval="30" name="monitor" role="Slave" timeout="30"/>
+ <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/>
+ <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120"/>
+ <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120"/>
+ </operations>
+ </primitive>
+ </bundle>
+ <bundle id="haproxy-bundle">
+ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="3" run-command="/bin/bash /usr/local/bin/kolla_start"/>
+ <storage>
+ <storage-mapping id="haproxy-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/haproxy.json" target-dir="/var/lib/kolla/config_files/config.json"/>
+ <storage-mapping id="haproxy-cfg-data" options="ro" source-dir="/var/lib/config-data/haproxy/etc" target-dir="/etc"/>
+ <storage-mapping id="haproxy-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
+ <storage-mapping id="haproxy-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
+ <storage-mapping id="haproxy-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
+ <storage-mapping id="haproxy-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
+ <storage-mapping id="haproxy-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
+ <storage-mapping id="haproxy-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
+ <storage-mapping id="haproxy-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
+ </storage>
+ </bundle>
+ <bundle id="redis-bundle">
+ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" masters="3" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" run-command="/bin/bash /usr/local/bin/kolla_start"/>
+ <network control-port="3124"/>
+ <storage>
+ <storage-mapping id="redis-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/redis.json" target-dir="/var/lib/kolla/config_files/config.json"/>
+ <storage-mapping id="redis-cfg-data-redis" options="ro" source-dir="/var/lib/config-data/redis/etc/redis" target-dir="/etc/redis"/>
+ <storage-mapping id="redis-cfg-data-redis-conf" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf" target-dir="/etc/redis.conf"/>
+ <storage-mapping id="redis-cfg-data-redis-conf-puppet" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf.puppet" target-dir="/etc/redis.conf.puppet"/>
+ <storage-mapping id="redis-cfg-data-redis-sentinel" options="ro" source-dir="/var/lib/config-data/redis/etc/redis-sentinel.conf" target-dir="/etc/redis-sentinel.conf"/>
+ <storage-mapping id="redis-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
+ <storage-mapping id="redis-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
+ <storage-mapping id="redis-lib" options="rw" source-dir="/var/lib/redis" target-dir="/var/lib/redis"/>
+ <storage-mapping id="redis-log" options="rw" source-dir="/var/log/redis" target-dir="/var/log/redis"/>
+ <storage-mapping id="redis-run" options="rw" source-dir="/var/run/redis" target-dir="/var/run/redis"/>
+ <storage-mapping id="redis-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
+ <storage-mapping id="redis-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
+ <storage-mapping id="redis-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
+ <storage-mapping id="redis-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
+ <storage-mapping id="redis-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
+ </storage>
+ <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
+ <instance_attributes id="redis-instance_attributes">
+ <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
+ </instance_attributes>
+ <meta_attributes id="redis-meta_attributes">
+ <nvpair id="redis-meta_attributes-interleave" name="interleave" value="true"/>
+ <nvpair id="redis-meta_attributes-notify" name="notify" value="true"/>
+ <nvpair id="redis-meta_attributes-ordered" name="ordered" value="true"/>
+ </meta_attributes>
+ <operations>
+ <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
+ <op id="redis-monitor-interval-45" interval="45" name="monitor" timeout="60"/>
+ <op id="redis-monitor-interval-20" interval="20" name="monitor" role="Master" timeout="60"/>
+ <op id="redis-monitor-interval-60" interval="60" name="monitor" role="Slave" timeout="60"/>
+ <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120"/>
+ <op id="redis-start-interval-0s" interval="0s" name="start" timeout="200s"/>
+ <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
+ </operations>
+ </primitive>
+ </bundle>
+ </resources>
+ <constraints>
+ <rsc_location id="location-redis-bundle" resource-discovery="exclusive" rsc="redis-bundle">
+ <rule id="location-redis-bundle-rule" score="0">
+ <expression attribute="redis-role" id="location-redis-bundle-rule-expr" operation="eq" value="true"/>
+ </rule>
+ </rsc_location>
+ <rsc_location id="location-galera-bundle" resource-discovery="exclusive" rsc="galera-bundle">
+ <rule id="location-galera-bundle-rule" score="0">
+ <expression attribute="galera-role" id="location-galera-bundle-rule-expr" operation="eq" value="true"/>
+ </rule>
+ </rsc_location>
+ <rsc_location id="location-redis-promote-1" rsc="redis-bundle" role="Master" node="redis-bundle-0" score="100"/>
+ <rsc_location id="location-redis-promote-2" rsc="redis-bundle" role="Master" node="redis-bundle-1" score="100"/>
+ <rsc_location id="location-redis-promote-3" rsc="redis-bundle" role="Master" node="redis-bundle-2" score="100"/>
+ <rsc_colocation id="colocation-1" rsc="redis-bundle" score="INFINITY" with-rsc="haproxy-bundle"/>
+ <rsc_colocation id="colocation-2" rsc="galera-bundle" score="INFINITY" with-rsc="storage-clone"/>
+ <rsc_order first="redis-bundle" first-action="promote" id="order-1" kind="Mandatory" then="storage-clone" then-action="start"/>
+ <rsc_order first="storage-clone" first-action="start" id="order-2" kind="Mandatory" then="galera-bundle" then-action="start"/>
+ <rsc_order first="haproxy-bundle" first-action="start" id="order-3" kind="Mandatory" then="storage-clone" then-action="start"/>
+ </constraints>
+ <rsc_defaults>
+ <meta_attributes id="rsc_defaults-options">
+ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
+ </meta_attributes>
+ </rsc_defaults>
+ </configuration>
+ <status>
+ <node_state id="1" uname="metal-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member"/>
+ <node_state id="2" uname="metal-2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member"/>
+ <node_state id="3" uname="metal-3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member"/>
+ </status>
+</cib>
diff --git a/pengine/test10/bundle-order-startup-clone.dot b/pengine/test10/bundle-order-startup-clone.dot
new file mode 100644
index 0000000000..92f019f146
--- /dev/null
+++ b/pengine/test10/bundle-order-startup-clone.dot
@@ -0,0 +1,119 @@
+digraph "g" {
+"galera-bundle-0_monitor_60000 metal-1" [ style=dashed color="red" fontcolor="black"]
+"galera-bundle-0_start_0 metal-1" -> "galera-bundle-0_monitor_60000 metal-1" [ style = dashed]
+"galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_20000 galera-bundle-0" [ style = dashed]
+"galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_30000 galera-bundle-0" [ style = dashed]
+"galera-bundle-0_start_0 metal-1" -> "galera:0_start_0 galera-bundle-0" [ style = dashed]
+"galera-bundle-0_start_0 metal-1" [ style=dashed color="red" fontcolor="black"]
+"galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = dashed]
+"galera-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = dashed]
+"galera-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = dashed]
+"galera-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-0_monitor_60000 metal-1" [ style=dashed color="red" fontcolor="black"]
+"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle-0_start_0 metal-1" [ style = dashed]
+"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle-docker-0_monitor_60000 metal-1" [ style = dashed]
+"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle_running_0" [ style = dashed]
+"galera-bundle-docker-0_start_0 metal-1" -> "galera:0_start_0 galera-bundle-0" [ style = dashed]
+"galera-bundle-docker-0_start_0 metal-1" [ style=dashed color="red" fontcolor="black"]
+"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = dashed]
+"galera-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"]
+"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = dashed]
+"galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = dashed]
+"galera-bundle-master_start_0" [ style=dashed color="red" fontcolor="orange"]
+"galera-bundle_running_0" [ style=dashed color="red" fontcolor="orange"]
+"galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 metal-1" [ style = dashed]
+"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = dashed]
+"galera-bundle_start_0" [ style=dashed color="red" fontcolor="orange"]
+"galera:0_monitor_20000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
+"galera:0_monitor_30000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
+"galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = dashed]
+"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = dashed]
+"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = dashed]
+"galera:0_start_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
+"haproxy-bundle-docker-0_monitor_0 metal-1" -> "haproxy-bundle-docker-0_start_0 metal-2" [ style = bold]
+"haproxy-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-0_monitor_0 metal-2" -> "haproxy-bundle-docker-0_start_0 metal-2" [ style = bold]
+"haproxy-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-0_monitor_0 metal-3" -> "haproxy-bundle-docker-0_start_0 metal-2" [ style = bold]
+"haproxy-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-0_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle-docker-0_start_0 metal-2" -> "haproxy-bundle-docker-0_monitor_60000 metal-2" [ style = bold]
+"haproxy-bundle-docker-0_start_0 metal-2" -> "haproxy-bundle_running_0" [ style = bold]
+"haproxy-bundle-docker-0_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"haproxy-bundle_running_0" -> "storage-clone_start_0" [ style = dashed]
+"haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
+"haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 metal-2" [ style = bold]
+"haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-0_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-0_start_0 metal-2" -> "redis-bundle-0_monitor_60000 metal-2" [ style = bold]
+"redis-bundle-0_start_0 metal-2" -> "redis:0_monitor_45000 redis-bundle-0" [ style = bold]
+"redis-bundle-0_start_0 metal-2" -> "redis:0_monitor_60000 redis-bundle-0" [ style = bold]
+"redis-bundle-0_start_0 metal-2" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
+"redis-bundle-0_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-2" [ style = bold]
+"redis-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-2" [ style = bold]
+"redis-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-2" [ style = bold]
+"redis-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-0_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-0_start_0 metal-2" -> "redis-bundle-0_start_0 metal-2" [ style = bold]
+"redis-bundle-docker-0_start_0 metal-2" -> "redis-bundle-docker-0_monitor_60000 metal-2" [ style = bold]
+"redis-bundle-docker-0_start_0 metal-2" -> "redis-bundle_running_0" [ style = bold]
+"redis-bundle-docker-0_start_0 metal-2" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
+"redis-bundle-docker-0_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = bold]
+"redis-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = bold]
+"redis-bundle-master_start_0" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
+"redis-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle_start_0" -> "redis-bundle-docker-0_start_0 metal-2" [ style = bold]
+"redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = bold]
+"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
+"redis:0_monitor_45000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
+"redis:0_monitor_60000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
+"redis:0_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = bold]
+"redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_45000 redis-bundle-0" [ style = bold]
+"redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_60000 redis-bundle-0" [ style = bold]
+"redis:0_start_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
+"storage-clone_confirmed-post_notify_running_0" -> "galera-bundle_start_0" [ style = dashed]
+"storage-clone_confirmed-post_notify_running_0" -> "storage:0_monitor_30000 metal-1" [ style = dashed]
+"storage-clone_confirmed-post_notify_running_0" -> "storage:1_monitor_30000 metal-2" [ style = dashed]
+"storage-clone_confirmed-post_notify_running_0" -> "storage:2_monitor_30000 metal-3" [ style = dashed]
+"storage-clone_confirmed-post_notify_running_0" [ style=dashed color="red" fontcolor="orange"]
+"storage-clone_confirmed-pre_notify_start_0" -> "storage-clone_post_notify_running_0" [ style = dashed]
+"storage-clone_confirmed-pre_notify_start_0" -> "storage-clone_start_0" [ style = dashed]
+"storage-clone_confirmed-pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"]
+"storage-clone_post_notify_running_0" -> "storage-clone_confirmed-post_notify_running_0" [ style = dashed]
+"storage-clone_post_notify_running_0" [ style=dashed color="red" fontcolor="orange"]
+"storage-clone_pre_notify_start_0" -> "storage-clone_confirmed-pre_notify_start_0" [ style = dashed]
+"storage-clone_pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"]
+"storage-clone_running_0" -> "storage-clone_post_notify_running_0" [ style = dashed]
+"storage-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
+"storage-clone_start_0" -> "storage-clone_running_0" [ style = dashed]
+"storage-clone_start_0" -> "storage:0_start_0 metal-1" [ style = dashed]
+"storage-clone_start_0" -> "storage:1_start_0 metal-2" [ style = dashed]
+"storage-clone_start_0" -> "storage:2_start_0 metal-3" [ style = dashed]
+"storage-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
+"storage:0_monitor_0 metal-1" -> "storage-clone_start_0" [ style = dashed]
+"storage:0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"storage:0_monitor_30000 metal-1" [ style=dashed color="red" fontcolor="black"]
+"storage:0_start_0 metal-1" -> "storage-clone_running_0" [ style = dashed]
+"storage:0_start_0 metal-1" -> "storage:0_monitor_30000 metal-1" [ style = dashed]
+"storage:0_start_0 metal-1" [ style=dashed color="red" fontcolor="black"]
+"storage:1_monitor_0 metal-2" -> "storage-clone_start_0" [ style = dashed]
+"storage:1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"storage:1_monitor_30000 metal-2" [ style=dashed color="red" fontcolor="black"]
+"storage:1_start_0 metal-2" -> "storage-clone_running_0" [ style = dashed]
+"storage:1_start_0 metal-2" -> "storage:1_monitor_30000 metal-2" [ style = dashed]
+"storage:1_start_0 metal-2" [ style=dashed color="red" fontcolor="black"]
+"storage:2_monitor_0 metal-3" -> "storage-clone_start_0" [ style = dashed]
+"storage:2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"storage:2_monitor_30000 metal-3" [ style=dashed color="red" fontcolor="black"]
+"storage:2_start_0 metal-3" -> "storage-clone_running_0" [ style = dashed]
+"storage:2_start_0 metal-3" -> "storage:2_monitor_30000 metal-3" [ style = dashed]
+"storage:2_start_0 metal-3" [ style=dashed color="red" fontcolor="black"]
+}
diff --git a/pengine/test10/bundle-order-startup-clone.exp b/pengine/test10/bundle-order-startup-clone.exp
new file mode 100644
index 0000000000..c736cb9ce9
--- /dev/null
+++ b/pengine/test10/bundle-order-startup-clone.exp
@@ -0,0 +1,327 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="2" operation="monitor" operation_key="storage:0_monitor_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="6" operation="monitor" operation_key="storage:1_monitor_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <rsc_op id="10" operation="monitor" operation_key="storage:2_monitor_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <rsc_op id="11" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="4">
+ <action_set>
+ <rsc_op id="7" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <rsc_op id="3" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="6">
+ <action_set>
+ <rsc_op id="56" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_60000" on_node="metal-2" on_node_uuid="2">
+ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="55" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <rsc_op id="55" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="4" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="8" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="12" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="57" operation="start" operation_key="haproxy-bundle_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="8">
+ <action_set>
+ <rsc_op id="12" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="9">
+ <action_set>
+ <rsc_op id="8" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="10">
+ <action_set>
+ <rsc_op id="4" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="11">
+ <action_set>
+ <rsc_op id="71" operation="monitor" operation_key="redis:0_monitor_60000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-2">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_role="Slave" CRM_meta_timeout="60000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="63" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="69" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="12">
+ <action_set>
+ <rsc_op id="70" operation="monitor" operation_key="redis:0_monitor_45000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-2">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="45000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="60000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="63" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="69" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="13">
+ <action_set>
+ <rsc_op id="69" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-2">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="200000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="61" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="63" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="72" operation="start" operation_key="redis-bundle-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="14">
+ <action_set>
+ <rsc_op id="62" operation="monitor" operation_key="redis-bundle-docker-0_monitor_60000" on_node="metal-2" on_node_uuid="2">
+ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="61" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="15">
+ <action_set>
+ <rsc_op id="61" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="5" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="9" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="13" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="65" operation="start" operation_key="redis-bundle_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="16">
+ <action_set>
+ <rsc_op id="13" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="17">
+ <action_set>
+ <rsc_op id="9" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="18">
+ <action_set>
+ <rsc_op id="5" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="19">
+ <action_set>
+ <rsc_op id="64" operation="monitor" operation_key="redis-bundle-0_monitor_60000" on_node="metal-2" on_node_uuid="2">
+ <primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="redis-bundle-docker-0" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="metal-2" port="3124"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="63" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="20">
+ <action_set>
+ <rsc_op id="63" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="redis-bundle-docker-0" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="metal-2" port="3124"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="61" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="21" priority="1000000">
+ <action_set>
+ <pseudo_event id="73" operation="running" operation_key="redis-bundle-master_running_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="69" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="72" operation="start" operation_key="redis-bundle-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="22">
+ <action_set>
+ <pseudo_event id="72" operation="start" operation_key="redis-bundle-master_start_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="65" operation="start" operation_key="redis-bundle_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="23" priority="1000000">
+ <action_set>
+ <pseudo_event id="66" operation="running" operation_key="redis-bundle_running_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="61" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="73" operation="running" operation_key="redis-bundle-master_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="24">
+ <action_set>
+ <pseudo_event id="65" operation="start" operation_key="redis-bundle_start_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="25" priority="1000000">
+ <action_set>
+ <pseudo_event id="58" operation="running" operation_key="haproxy-bundle_running_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="55" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="26">
+ <action_set>
+ <pseudo_event id="57" operation="start" operation_key="haproxy-bundle_start_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/bundle-order-startup-clone.scores b/pengine/test10/bundle-order-startup-clone.scores
new file mode 100644
index 0000000000..d907861ed1
--- /dev/null
+++ b/pengine/test10/bundle-order-startup-clone.scores
@@ -0,0 +1,174 @@
+Allocation scores:
+clone_color: galera-bundle-master allocation score on galera-bundle-0: 0
+clone_color: galera-bundle-master allocation score on metal-1: -INFINITY
+clone_color: galera-bundle-master allocation score on metal-2: -INFINITY
+clone_color: galera-bundle-master allocation score on metal-3: -INFINITY
+clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: galera:0 allocation score on galera-bundle-0: INFINITY
+clone_color: galera:0 allocation score on metal-1: -INFINITY
+clone_color: galera:0 allocation score on metal-2: -INFINITY
+clone_color: galera:0 allocation score on metal-3: -INFINITY
+clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
+clone_color: redis-bundle-master allocation score on metal-1: -INFINITY
+clone_color: redis-bundle-master allocation score on metal-2: -INFINITY
+clone_color: redis-bundle-master allocation score on metal-3: -INFINITY
+clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: redis-bundle-master allocation score on redis-bundle-0: 0
+clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY
+clone_color: redis:0 allocation score on metal-1: -INFINITY
+clone_color: redis:0 allocation score on metal-2: -INFINITY
+clone_color: redis:0 allocation score on metal-3: -INFINITY
+clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: redis:0 allocation score on redis-bundle-0: INFINITY
+clone_color: storage-clone allocation score on metal-1: 0
+clone_color: storage-clone allocation score on metal-2: 0
+clone_color: storage-clone allocation score on metal-3: 0
+clone_color: storage-clone allocation score on rabbitmq-bundle-0: 0
+clone_color: storage:0 allocation score on metal-1: 0
+clone_color: storage:0 allocation score on metal-2: 0
+clone_color: storage:0 allocation score on metal-3: 0
+clone_color: storage:0 allocation score on rabbitmq-bundle-0: 0
+clone_color: storage:1 allocation score on metal-1: 0
+clone_color: storage:1 allocation score on metal-2: 0
+clone_color: storage:1 allocation score on metal-3: 0
+clone_color: storage:1 allocation score on rabbitmq-bundle-0: 0
+clone_color: storage:2 allocation score on metal-1: 0
+clone_color: storage:2 allocation score on metal-2: 0
+clone_color: storage:2 allocation score on metal-3: 0
+clone_color: storage:2 allocation score on rabbitmq-bundle-0: 0
+clone_color: storage:3 allocation score on metal-1: 0
+clone_color: storage:3 allocation score on metal-2: 0
+clone_color: storage:3 allocation score on metal-3: 0
+clone_color: storage:3 allocation score on rabbitmq-bundle-0: 0
+container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle allocation score on metal-1: 0
+container_color: galera-bundle allocation score on metal-2: 0
+container_color: galera-bundle allocation score on metal-3: 0
+container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-0 allocation score on metal-1: 0
+container_color: galera-bundle-0 allocation score on metal-2: 0
+container_color: galera-bundle-0 allocation score on metal-3: 0
+container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-docker-0 allocation score on metal-1: 0
+container_color: galera-bundle-docker-0 allocation score on metal-2: 0
+container_color: galera-bundle-docker-0 allocation score on metal-3: 0
+container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-master allocation score on metal-1: 0
+container_color: galera-bundle-master allocation score on metal-2: 0
+container_color: galera-bundle-master allocation score on metal-3: 0
+container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: 0
+container_color: galera:0 allocation score on galera-bundle-0: -INFINITY
+container_color: galera:0 allocation score on metal-1: 0
+container_color: galera:0 allocation score on metal-2: 0
+container_color: galera:0 allocation score on metal-3: 0
+container_color: galera:0 allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle allocation score on metal-1: 0
+container_color: haproxy-bundle allocation score on metal-1: 0
+container_color: haproxy-bundle allocation score on metal-2: 0
+container_color: haproxy-bundle allocation score on metal-2: 0
+container_color: haproxy-bundle allocation score on metal-3: 0
+container_color: haproxy-bundle allocation score on metal-3: 0
+container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: 0
+container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle allocation score on metal-1: 0
+container_color: redis-bundle allocation score on metal-2: 0
+container_color: redis-bundle allocation score on metal-3: 0
+container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-0 allocation score on metal-1: 0
+container_color: redis-bundle-0 allocation score on metal-2: 0
+container_color: redis-bundle-0 allocation score on metal-3: 0
+container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on metal-1: 0
+container_color: redis-bundle-docker-0 allocation score on metal-2: 0
+container_color: redis-bundle-docker-0 allocation score on metal-3: 0
+container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-master allocation score on metal-1: 0
+container_color: redis-bundle-master allocation score on metal-2: 0
+container_color: redis-bundle-master allocation score on metal-3: 0
+container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: 0
+container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY
+container_color: redis:0 allocation score on galera-bundle-0: -INFINITY
+container_color: redis:0 allocation score on metal-1: 0
+container_color: redis:0 allocation score on metal-2: 0
+container_color: redis:0 allocation score on metal-3: 0
+container_color: redis:0 allocation score on rabbitmq-bundle-0: 0
+container_color: redis:0 allocation score on redis-bundle-0: -INFINITY
+galera:0 promotion score on galera-bundle-0: -1
+native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-0 allocation score on metal-1: 10000
+native_color: galera-bundle-0 allocation score on metal-2: 0
+native_color: galera-bundle-0 allocation score on metal-3: 0
+native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on metal-1: 0
+native_color: galera-bundle-docker-0 allocation score on metal-2: 0
+native_color: galera-bundle-docker-0 allocation score on metal-3: 0
+native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera:0 allocation score on galera-bundle-0: INFINITY
+native_color: galera:0 allocation score on metal-1: -INFINITY
+native_color: galera:0 allocation score on metal-2: -INFINITY
+native_color: galera:0 allocation score on metal-3: -INFINITY
+native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+native_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
+native_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
+native_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
+native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
+native_color: redis-bundle-0 allocation score on metal-1: 0
+native_color: redis-bundle-0 allocation score on metal-2: 10000
+native_color: redis-bundle-0 allocation score on metal-3: 0
+native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on metal-1: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on metal-2: 0
+native_color: redis-bundle-docker-0 allocation score on metal-3: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
+native_color: redis:0 allocation score on galera-bundle-0: -INFINITY
+native_color: redis:0 allocation score on metal-1: -INFINITY
+native_color: redis:0 allocation score on metal-2: -INFINITY
+native_color: redis:0 allocation score on metal-3: -INFINITY
+native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis:0 allocation score on redis-bundle-0: INFINITY
+native_color: storage:0 allocation score on metal-1: 0
+native_color: storage:0 allocation score on metal-2: 0
+native_color: storage:0 allocation score on metal-3: 0
+native_color: storage:0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: storage:1 allocation score on metal-1: -INFINITY
+native_color: storage:1 allocation score on metal-2: 0
+native_color: storage:1 allocation score on metal-3: 0
+native_color: storage:1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: storage:2 allocation score on metal-1: -INFINITY
+native_color: storage:2 allocation score on metal-2: -INFINITY
+native_color: storage:2 allocation score on metal-3: 0
+native_color: storage:2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: storage:3 allocation score on metal-1: -INFINITY
+native_color: storage:3 allocation score on metal-2: -INFINITY
+native_color: storage:3 allocation score on metal-3: -INFINITY
+native_color: storage:3 allocation score on rabbitmq-bundle-0: -INFINITY
+redis:0 promotion score on redis-bundle-0: -1
diff --git a/pengine/test10/bundle-order-startup-clone.summary b/pengine/test10/bundle-order-startup-clone.summary
new file mode 100644
index 0000000000..f3f8be0270
--- /dev/null
+++ b/pengine/test10/bundle-order-startup-clone.summary
@@ -0,0 +1,69 @@
+
+Current cluster status:
+Online: [ metal-1 metal-2 metal-3 ]
+RemoteOFFLINE: [ rabbitmq-bundle-0 ]
+
+ Clone Set: storage-clone [storage]
+ Stopped: [ metal-1 metal-2 metal-3 rabbitmq-bundle-0 ]
+ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
+ galera-bundle-0 (ocf::heartbeat:galera): Stopped
+ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
+ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped
+ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
+ redis-bundle-0 (ocf::heartbeat:redis): Stopped
+
+Transition Summary:
+ * Start storage:0 (metal-1 - blocked)
+ * Start storage:1 (metal-2 - blocked)
+ * Start storage:2 (metal-3 - blocked)
+ * Start galera-bundle-docker-0 (metal-1 - blocked)
+ * Start galera-bundle-0 (metal-1 - blocked)
+ * Start galera:0 (galera-bundle-0 - blocked)
+ * Start haproxy-bundle-docker-0 (metal-2)
+ * Start redis-bundle-docker-0 (metal-2)
+ * Start redis-bundle-0 (metal-2)
+ * Start redis:0 (redis-bundle-0)
+
+Executing cluster transition:
+ * Resource action: storage:0 monitor on metal-1
+ * Resource action: storage:1 monitor on metal-2
+ * Resource action: storage:2 monitor on metal-3
+ * Resource action: galera-bundle-docker-0 monitor on metal-3
+ * Resource action: galera-bundle-docker-0 monitor on metal-2
+ * Resource action: galera-bundle-docker-0 monitor on metal-1
+ * Resource action: haproxy-bundle-docker-0 monitor on metal-3
+ * Resource action: haproxy-bundle-docker-0 monitor on metal-2
+ * Resource action: haproxy-bundle-docker-0 monitor on metal-1
+ * Resource action: redis-bundle-docker-0 monitor on metal-3
+ * Resource action: redis-bundle-docker-0 monitor on metal-2
+ * Resource action: redis-bundle-docker-0 monitor on metal-1
+ * Pseudo action: redis-bundle_start_0
+ * Pseudo action: haproxy-bundle_start_0
+ * Resource action: haproxy-bundle-docker-0 start on metal-2
+ * Resource action: redis-bundle-docker-0 start on metal-2
+ * Resource action: redis-bundle-0 start on metal-2
+ * Pseudo action: redis-bundle-master_start_0
+ * Pseudo action: haproxy-bundle_running_0
+ * Resource action: haproxy-bundle-docker-0 monitor=60000 on metal-2
+ * Resource action: redis:0 start on redis-bundle-0
+ * Resource action: redis-bundle-docker-0 monitor=60000 on metal-2
+ * Resource action: redis-bundle-0 monitor=60000 on metal-2
+ * Pseudo action: redis-bundle-master_running_0
+ * Pseudo action: redis-bundle_running_0
+ * Resource action: redis:0 monitor=60000 on redis-bundle-0
+ * Resource action: redis:0 monitor=45000 on redis-bundle-0
+
+Revised cluster status:
+Online: [ metal-1 metal-2 metal-3 ]
+RemoteOFFLINE: [ rabbitmq-bundle-0 ]
+Containers: [ redis-bundle-0:redis-bundle-docker-0 ]
+
+ Clone Set: storage-clone [storage]
+ Stopped: [ metal-1 metal-2 metal-3 rabbitmq-bundle-0 ]
+ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
+ galera-bundle-0 (ocf::heartbeat:galera): Stopped
+ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
+ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started metal-2
+ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
+ redis-bundle-0 (ocf::heartbeat:redis): Slave metal-2
+
diff --git a/pengine/test10/bundle-order-startup-clone.xml b/pengine/test10/bundle-order-startup-clone.xml
new file mode 100644
index 0000000000..d24c707a91
--- /dev/null
+++ b/pengine/test10/bundle-order-startup-clone.xml
@@ -0,0 +1,187 @@
+<cib crm_feature_set="3.0.12" validate-with="pacemaker-2.8" epoch="58" num_updates="129" admin_epoch="0" cib-last-written="Thu Jun 1 03:59:09 2017" update-origin="undercloud" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-10.el7-94ff4df"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/>
+ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
+ </cluster_property_set>
+ <cluster_property_set id="redis_replication">
+ <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="redis-bundle-0"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="metal-1">
+ <instance_attributes id="nodes-1">
+ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/>
+ <nvpair id="nodes-1-galera-role" name="galera-role" value="true"/>
+ <nvpair id="nodes-1-redis-role" name="redis-role" value="true"/>
+ <nvpair id="nodes-1-haproxy-role" name="haproxy-role" value="true"/>
+ <nvpair id="nodes-1-cinder-volume-role" name="cinder-volume-role" value="true"/>
+ </instance_attributes>
+ </node>
+ <node id="2" uname="metal-2">
+ <instance_attributes id="nodes-2">
+ <nvpair id="nodes-2-rabbitmq-role" name="rabbitmq-role" value="true"/>
+ <nvpair id="nodes-2-galera-role" name="galera-role" value="true"/>
+ <nvpair id="nodes-2-redis-role" name="redis-role" value="true"/>
+ <nvpair id="nodes-2-haproxy-role" name="haproxy-role" value="true"/>
+ <nvpair id="nodes-2-cinder-volume-role" name="cinder-volume-role" value="true"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="metal-3">
+ <instance_attributes id="nodes-3">
+ <nvpair id="nodes-3-rabbitmq-role" name="rabbitmq-role" value="true"/>
+ <nvpair id="nodes-3-galera-role" name="galera-role" value="true"/>
+ <nvpair id="nodes-3-redis-role" name="redis-role" value="true"/>
+ <nvpair id="nodes-3-haproxy-role" name="haproxy-role" value="true"/>
+ <nvpair id="nodes-3-cinder-volume-role" name="cinder-volume-role" value="true"/>
+ </instance_attributes>
+ </node>
+ <node id="rabbitmq-bundle-0" type="remote" uname="rabbitmq-bundle-0">
+ <instance_attributes id="nodes-rabbitmq-bundle-0">
+ <nvpair id="nodes-rabbitmq-bundle-0-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@metal"/>
+ </instance_attributes>
+ </node>
+ </nodes>
+ <resources>
+ <clone id="storage-clone">
+ <meta_attributes id="storage_ma">
+ <nvpair id="storage_globally-unique" name="globally-unique" value="false"/>
+ <nvpair id="storage_notify" name="notify" value="true"/>
+ </meta_attributes>
+ <primitive class="ocf" id="storage" provider="heartbeat" type="Filesystem">
+ <operations>
+ <op id="storage_mon" interval="30s" name="monitor" timeout="30s"/>
+ </operations>
+ <instance_attributes id="storage_ia">
+ <nvpair id="storage_attr_0" name="device" value="nfs:/share/drbd_www/data/"/>
+ <nvpair id="storage_attr_1" name="directory" value="/data/www"/>
+ <nvpair id="storage_attr_2" name="fstype" value="nfs"/>
+ </instance_attributes>
+ </primitive>
+ </clone>
+ <bundle id="galera-bundle">
+ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
+ <network control-port="3123"/>
+ <storage>
+ <storage-mapping id="mysql-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/mysql.json" target-dir="/var/lib/kolla/config_files/config.json"/>
+ <storage-mapping id="mysql-cfg-data" options="ro" source-dir="/var/lib/config-data/mysql" target-dir="/var/lib/kolla/config_files/src"/>
+ <storage-mapping id="mysql-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
+ <storage-mapping id="mysql-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
+ <storage-mapping id="mysql-lib" options="rw" source-dir="/var/lib/mysql" target-dir="/var/lib/mysql"/>
+ <storage-mapping id="mysql-log-mariadb" options="rw" source-dir="/var/log/mariadb" target-dir="/var/log/mariadb"/>
+ <storage-mapping id="mysql-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
+ <storage-mapping id="mysql-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
+ <storage-mapping id="mysql-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
+ <storage-mapping id="mysql-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
+ <storage-mapping id="mysql-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
+ </storage>
+ <primitive class="ocf" id="galera" provider="heartbeat" type="galera">
+ <instance_attributes id="galera-instance_attributes">
+ <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/>
+ <nvpair id="galera-instance_attributes-cluster_host_map" name="cluster_host_map" value="galera-bundle-0:metal"/>
+ <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/>
+ <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://metal"/>
+ </instance_attributes>
+ <meta_attributes id="galera-meta_attributes">
+ <nvpair id="galera-meta_attributes-master-max" name="master-max" value="1"/>
+ <nvpair id="galera-meta_attributes-ordered" name="ordered" value="true"/>
+ </meta_attributes>
+ <operations>
+ <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
+ <op id="galera-monitor-interval-20" interval="20" name="monitor" timeout="30"/>
+ <op id="galera-monitor-interval-10" interval="10" name="monitor" role="Master" timeout="30"/>
+ <op id="galera-monitor-interval-30" interval="30" name="monitor" role="Slave" timeout="30"/>
+ <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/>
+ <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120"/>
+ <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120"/>
+ </operations>
+ </primitive>
+ </bundle>
+ <bundle id="haproxy-bundle">
+ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
+ <storage>
+ <storage-mapping id="haproxy-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/haproxy.json" target-dir="/var/lib/kolla/config_files/config.json"/>
+ <storage-mapping id="haproxy-cfg-data" options="ro" source-dir="/var/lib/config-data/haproxy/etc" target-dir="/etc"/>
+ <storage-mapping id="haproxy-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
+ <storage-mapping id="haproxy-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
+ <storage-mapping id="haproxy-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
+ <storage-mapping id="haproxy-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
+ <storage-mapping id="haproxy-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
+ <storage-mapping id="haproxy-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
+ <storage-mapping id="haproxy-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
+ </storage>
+ </bundle>
+ <bundle id="redis-bundle">
+ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
+ <network control-port="3124"/>
+ <storage>
+ <storage-mapping id="redis-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/redis.json" target-dir="/var/lib/kolla/config_files/config.json"/>
+ <storage-mapping id="redis-cfg-data-redis" options="ro" source-dir="/var/lib/config-data/redis/etc/redis" target-dir="/etc/redis"/>
+ <storage-mapping id="redis-cfg-data-redis-conf" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf" target-dir="/etc/redis.conf"/>
+ <storage-mapping id="redis-cfg-data-redis-conf-puppet" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf.puppet" target-dir="/etc/redis.conf.puppet"/>
+ <storage-mapping id="redis-cfg-data-redis-sentinel" options="ro" source-dir="/var/lib/config-data/redis/etc/redis-sentinel.conf" target-dir="/etc/redis-sentinel.conf"/>
+ <storage-mapping id="redis-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
+ <storage-mapping id="redis-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
+ <storage-mapping id="redis-lib" options="rw" source-dir="/var/lib/redis" target-dir="/var/lib/redis"/>
+ <storage-mapping id="redis-log" options="rw" source-dir="/var/log/redis" target-dir="/var/log/redis"/>
+ <storage-mapping id="redis-run" options="rw" source-dir="/var/run/redis" target-dir="/var/run/redis"/>
+ <storage-mapping id="redis-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
+ <storage-mapping id="redis-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
+ <storage-mapping id="redis-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
+ <storage-mapping id="redis-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
+ <storage-mapping id="redis-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
+ </storage>
+ <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
+ <instance_attributes id="redis-instance_attributes">
+ <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
+ </instance_attributes>
+ <meta_attributes id="redis-meta_attributes">
+ <nvpair id="redis-meta_attributes-interleave" name="interleave" value="true"/>
+ <nvpair id="redis-meta_attributes-notify" name="notify" value="true"/>
+ <nvpair id="redis-meta_attributes-ordered" name="ordered" value="true"/>
+ </meta_attributes>
+ <operations>
+ <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
+ <op id="redis-monitor-interval-45" interval="45" name="monitor" timeout="60"/>
+ <op id="redis-monitor-interval-20" interval="20" name="monitor" role="Master" timeout="60"/>
+ <op id="redis-monitor-interval-60" interval="60" name="monitor" role="Slave" timeout="60"/>
+ <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120"/>
+ <op id="redis-start-interval-0s" interval="0s" name="start" timeout="200s"/>
+ <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
+ </operations>
+ </primitive>
+ </bundle>
+ </resources>
+ <constraints>
+ <rsc_location id="location-redis-bundle" resource-discovery="exclusive" rsc="redis-bundle">
+ <rule id="location-redis-bundle-rule" score="0">
+ <expression attribute="redis-role" id="location-redis-bundle-rule-expr" operation="eq" value="true"/>
+ </rule>
+ </rsc_location>
+ <rsc_location id="location-galera-bundle" resource-discovery="exclusive" rsc="galera-bundle">
+ <rule id="location-galera-bundle-rule" score="0">
+ <expression attribute="galera-role" id="location-galera-bundle-rule-expr" operation="eq" value="true"/>
+ </rule>
+ </rsc_location>
+ <rsc_colocation id="colocation-1" rsc="redis-bundle" score="INFINITY" with-rsc="haproxy-bundle"/>
+ <rsc_colocation id="colocation-2" rsc="galera-bundle" score="INFINITY" with-rsc="storage-clone"/>
+ <rsc_order first="redis-bundle" first-action="promote" id="order-1" kind="Mandatory" then="storage-clone" then-action="start"/>
+ <rsc_order first="storage-clone" first-action="start" id="order-2" kind="Mandatory" then="galera-bundle" then-action="start"/>
+ <rsc_order first="haproxy-bundle" first-action="start" id="order-3" kind="Mandatory" then="storage-clone" then-action="start"/>
+ </constraints>
+ <rsc_defaults>
+ <meta_attributes id="rsc_defaults-options">
+ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
+ </meta_attributes>
+ </rsc_defaults>
+ </configuration>
+ <status>
+ <node_state id="1" uname="metal-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member"/>
+ <node_state id="2" uname="metal-2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member"/>
+ <node_state id="3" uname="metal-3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member"/>
+ </status>
+</cib>
diff --git a/pengine/test10/bundle-order-stop-clone.dot b/pengine/test10/bundle-order-stop-clone.dot
new file mode 100644
index 0000000000..6545fb8e61
--- /dev/null
+++ b/pengine/test10/bundle-order-stop-clone.dot
@@ -0,0 +1,80 @@
+digraph "g" {
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+"galera-bundle-0_monitor_60000 metal-1" [ style=dashed color="red" fontcolor="black"]
+"galera-bundle-0_start_0 metal-1" -> "galera-bundle-0_monitor_60000 metal-1" [ style = dashed]
+"galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_20000 galera-bundle-0" [ style = dashed]
+"galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_30000 galera-bundle-0" [ style = dashed]
+"galera-bundle-0_start_0 metal-1" -> "galera:0_start_0 galera-bundle-0" [ style = dashed]
+"galera-bundle-0_start_0 metal-1" [ style=dashed color="red" fontcolor="black"]
+"galera-bundle-0_stop_0 metal-1" -> "all_stopped" [ style = bold]
+"galera-bundle-0_stop_0 metal-1" -> "galera-bundle-0_start_0 metal-1" [ style = dashed]
+"galera-bundle-0_stop_0 metal-1" -> "galera-bundle-docker-0_stop_0 metal-1" [ style = bold]
+"galera-bundle-0_stop_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-docker-0_stop_0 metal-1" -> "all_stopped" [ style = bold]
+"galera-bundle-docker-0_stop_0 metal-1" -> "galera-bundle_stopped_0" [ style = bold]
+"galera-bundle-docker-0_stop_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold]
+"galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
+"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold]
+"galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = dashed]
+"galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
+"galera-bundle-master_stop_0" -> "galera-bundle-master_stopped_0" [ style = bold]
+"galera-bundle-master_stop_0" -> "galera:0_stop_0 galera-bundle-0" [ style = bold]
+"galera-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"]
+"galera-bundle-master_stopped_0" -> "galera-bundle-master_start_0" [ style = bold]
+"galera-bundle-master_stopped_0" -> "galera-bundle_stopped_0" [ style = bold]
+"galera-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
+"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold]
+"galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
+"galera-bundle_stop_0" -> "galera-bundle-docker-0_stop_0 metal-1" [ style = bold]
+"galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold]
+"galera-bundle_stop_0" -> "galera:0_stop_0 galera-bundle-0" [ style = bold]
+"galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
+"galera-bundle_stopped_0" -> "galera-bundle_start_0" [ style = bold]
+"galera-bundle_stopped_0" -> "storage-clone_stop_0" [ style = bold]
+"galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"galera:0_monitor_20000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
+"galera:0_monitor_30000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
+"galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = dashed]
+"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = dashed]
+"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = dashed]
+"galera:0_start_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
+"galera:0_stop_0 galera-bundle-0" -> "all_stopped" [ style = bold]
+"galera:0_stop_0 galera-bundle-0" -> "galera-bundle-0_stop_0 metal-1" [ style = bold]
+"galera:0_stop_0 galera-bundle-0" -> "galera-bundle-master_stopped_0" [ style = bold]
+"galera:0_stop_0 galera-bundle-0" -> "galera:0_start_0 galera-bundle-0" [ style = dashed]
+"galera:0_stop_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
+"storage-clone_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold]
+"storage-clone_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"storage-clone_confirmed-pre_notify_stop_0" -> "storage-clone_post_notify_stopped_0" [ style = bold]
+"storage-clone_confirmed-pre_notify_stop_0" -> "storage-clone_stop_0" [ style = bold]
+"storage-clone_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
+"storage-clone_post_notify_stopped_0" -> "storage-clone_confirmed-post_notify_stopped_0" [ style = bold]
+"storage-clone_post_notify_stopped_0" -> "storage:1_post_notify_stop_0 metal-2" [ style = bold]
+"storage-clone_post_notify_stopped_0" -> "storage:2_post_notify_stop_0 metal-3" [ style = bold]
+"storage-clone_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"storage-clone_pre_notify_stop_0" -> "storage-clone_confirmed-pre_notify_stop_0" [ style = bold]
+"storage-clone_pre_notify_stop_0" -> "storage:0_pre_notify_stop_0 metal-1" [ style = bold]
+"storage-clone_pre_notify_stop_0" -> "storage:1_pre_notify_stop_0 metal-2" [ style = bold]
+"storage-clone_pre_notify_stop_0" -> "storage:2_pre_notify_stop_0 metal-3" [ style = bold]
+"storage-clone_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
+"storage-clone_stop_0" -> "storage-clone_stopped_0" [ style = bold]
+"storage-clone_stop_0" -> "storage:0_stop_0 metal-1" [ style = bold]
+"storage-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"storage-clone_stopped_0" -> "storage-clone_post_notify_stopped_0" [ style = bold]
+"storage-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"storage:0_pre_notify_stop_0 metal-1" -> "storage-clone_confirmed-pre_notify_stop_0" [ style = bold]
+"storage:0_pre_notify_stop_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"storage:0_stop_0 metal-1" -> "all_stopped" [ style = bold]
+"storage:0_stop_0 metal-1" -> "storage-clone_stopped_0" [ style = bold]
+"storage:0_stop_0 metal-1" [ style=bold color="green" fontcolor="black"]
+"storage:1_post_notify_stop_0 metal-2" -> "storage-clone_confirmed-post_notify_stopped_0" [ style = bold]
+"storage:1_post_notify_stop_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"storage:1_pre_notify_stop_0 metal-2" -> "storage-clone_confirmed-pre_notify_stop_0" [ style = bold]
+"storage:1_pre_notify_stop_0 metal-2" [ style=bold color="green" fontcolor="black"]
+"storage:2_post_notify_stop_0 metal-3" -> "storage-clone_confirmed-post_notify_stopped_0" [ style = bold]
+"storage:2_post_notify_stop_0 metal-3" [ style=bold color="green" fontcolor="black"]
+"storage:2_pre_notify_stop_0 metal-3" -> "storage-clone_confirmed-pre_notify_stop_0" [ style = bold]
+"storage:2_pre_notify_stop_0 metal-3" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/pengine/test10/bundle-order-stop-clone.exp b/pengine/test10/bundle-order-stop-clone.exp
new file mode 100644
index 0000000000..ac0ae0532b
--- /dev/null
+++ b/pengine/test10/bundle-order-stop-clone.exp
@@ -0,0 +1,345 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="140" operation="notify" operation_key="storage:0_pre_notify_stop_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="pre" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="42" operation="notify" operation_key="storage-clone_pre_notify_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="29" operation="stop" operation_key="storage:0_stop_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="40" operation="stop" operation_key="storage-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2" priority="1000000">
+ <action_set>
+ <rsc_op id="142" operation="notify" operation_key="storage:1_post_notify_stop_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="post" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="44" operation="notify" operation_key="storage-clone_post_notify_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <rsc_op id="141" operation="notify" operation_key="storage:1_pre_notify_stop_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="pre" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="42" operation="notify" operation_key="storage-clone_pre_notify_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4" priority="1000000">
+ <action_set>
+ <rsc_op id="144" operation="notify" operation_key="storage:2_post_notify_stop_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="post" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="44" operation="notify" operation_key="storage-clone_post_notify_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <rsc_op id="143" operation="notify" operation_key="storage:2_pre_notify_stop_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="pre" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="42" operation="notify" operation_key="storage-clone_pre_notify_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="6" priority="1000000">
+ <action_set>
+ <pseudo_event id="45" operation="notified" operation_key="storage-clone_confirmed-post_notify_stopped_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="44" operation="notify" operation_key="storage-clone_post_notify_stopped_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="142" operation="notify" operation_key="storage:1_post_notify_stop_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="144" operation="notify" operation_key="storage:2_post_notify_stop_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7" priority="1000000">
+ <action_set>
+ <pseudo_event id="44" operation="notify" operation_key="storage-clone_post_notify_stopped_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="41" operation="stopped" operation_key="storage-clone_stopped_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-pre_notify_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="8">
+ <action_set>
+ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-pre_notify_stop_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="42" operation="notify" operation_key="storage-clone_pre_notify_stop_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="140" operation="notify" operation_key="storage:0_pre_notify_stop_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="141" operation="notify" operation_key="storage:1_pre_notify_stop_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="143" operation="notify" operation_key="storage:2_pre_notify_stop_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="9">
+ <action_set>
+ <pseudo_event id="42" operation="notify" operation_key="storage-clone_pre_notify_stop_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="10" priority="1000000">
+ <action_set>
+ <pseudo_event id="41" operation="stopped" operation_key="storage-clone_stopped_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="29" operation="stop" operation_key="storage:0_stop_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="40" operation="stop" operation_key="storage-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="11">
+ <action_set>
+ <pseudo_event id="40" operation="stop" operation_key="storage-clone_stop_0">
+ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-pre_notify_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="60" operation="stopped" operation_key="galera-bundle_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="12">
+ <action_set>
+ <rsc_op id="62" operation="stop" operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1">
+ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="59" operation="stop" operation_key="galera-bundle_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="69" operation="stop" operation_key="galera-bundle-master_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="13">
+ <action_set>
+ <rsc_op id="46" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="47" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="59" operation="stop" operation_key="galera-bundle_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="14">
+ <action_set>
+ <rsc_op id="47" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="#uname" port="3123"/>
+ <downed>
+ <node id="galera-bundle-0"/>
+ </downed>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="62" operation="stop" operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="15" priority="1000000">
+ <action_set>
+ <pseudo_event id="70" operation="stopped" operation_key="galera-bundle-master_stopped_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="62" operation="stop" operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="69" operation="stop" operation_key="galera-bundle-master_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="16">
+ <action_set>
+ <pseudo_event id="69" operation="stop" operation_key="galera-bundle-master_stop_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="59" operation="stop" operation_key="galera-bundle_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="17" priority="1000000">
+ <action_set>
+ <pseudo_event id="68" operation="running" operation_key="galera-bundle-master_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="67" operation="start" operation_key="galera-bundle-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="18">
+ <action_set>
+ <pseudo_event id="67" operation="start" operation_key="galera-bundle-master_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="57" operation="start" operation_key="galera-bundle_start_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="70" operation="stopped" operation_key="galera-bundle-master_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="19" priority="1000000">
+ <action_set>
+ <pseudo_event id="60" operation="stopped" operation_key="galera-bundle_stopped_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="46" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="70" operation="stopped" operation_key="galera-bundle-master_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="20">
+ <action_set>
+ <pseudo_event id="59" operation="stop" operation_key="galera-bundle_stop_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="21" priority="1000000">
+ <action_set>
+ <pseudo_event id="58" operation="running" operation_key="galera-bundle_running_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="68" operation="running" operation_key="galera-bundle-master_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="22">
+ <action_set>
+ <pseudo_event id="57" operation="start" operation_key="galera-bundle_start_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="60" operation="stopped" operation_key="galera-bundle_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="23">
+ <action_set>
+ <pseudo_event id="28" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="29" operation="stop" operation_key="storage:0_stop_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="45" operation="notified" operation_key="storage-clone_confirmed-post_notify_stopped_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="46" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="47" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="62" operation="stop" operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/bundle-order-stop-clone.scores b/pengine/test10/bundle-order-stop-clone.scores
new file mode 100644
index 0000000000..df53f3265f
--- /dev/null
+++ b/pengine/test10/bundle-order-stop-clone.scores
@@ -0,0 +1,591 @@
+Allocation scores:
+clone_color: galera-bundle-master allocation score on galera-bundle-0: 0
+clone_color: galera-bundle-master allocation score on galera-bundle-1: 0
+clone_color: galera-bundle-master allocation score on galera-bundle-2: 0
+clone_color: galera-bundle-master allocation score on metal-1: -INFINITY
+clone_color: galera-bundle-master allocation score on metal-2: -INFINITY
+clone_color: galera-bundle-master allocation score on metal-3: -INFINITY
+clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: galera:0 allocation score on galera-bundle-0: INFINITY
+clone_color: galera:0 allocation score on galera-bundle-1: -INFINITY
+clone_color: galera:0 allocation score on galera-bundle-2: -INFINITY
+clone_color: galera:0 allocation score on metal-1: -INFINITY
+clone_color: galera:0 allocation score on metal-2: -INFINITY
+clone_color: galera:0 allocation score on metal-3: -INFINITY
+clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: galera:1 allocation score on galera-bundle-0: -INFINITY
+clone_color: galera:1 allocation score on galera-bundle-1: INFINITY
+clone_color: galera:1 allocation score on galera-bundle-2: -INFINITY
+clone_color: galera:1 allocation score on metal-1: -INFINITY
+clone_color: galera:1 allocation score on metal-2: -INFINITY
+clone_color: galera:1 allocation score on metal-3: -INFINITY
+clone_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: galera:2 allocation score on galera-bundle-0: -INFINITY
+clone_color: galera:2 allocation score on galera-bundle-1: -INFINITY
+clone_color: galera:2 allocation score on galera-bundle-2: INFINITY
+clone_color: galera:2 allocation score on metal-1: -INFINITY
+clone_color: galera:2 allocation score on metal-2: -INFINITY
+clone_color: galera:2 allocation score on metal-3: -INFINITY
+clone_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
+clone_color: redis-bundle-master allocation score on galera-bundle-1: -INFINITY
+clone_color: redis-bundle-master allocation score on galera-bundle-2: -INFINITY
+clone_color: redis-bundle-master allocation score on metal-1: -INFINITY
+clone_color: redis-bundle-master allocation score on metal-2: -INFINITY
+clone_color: redis-bundle-master allocation score on metal-3: -INFINITY
+clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: redis-bundle-master allocation score on redis-bundle-0: 0
+clone_color: redis-bundle-master allocation score on redis-bundle-1: 0
+clone_color: redis-bundle-master allocation score on redis-bundle-2: 0
+clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY
+clone_color: redis:0 allocation score on galera-bundle-1: -INFINITY
+clone_color: redis:0 allocation score on galera-bundle-2: -INFINITY
+clone_color: redis:0 allocation score on metal-1: -INFINITY
+clone_color: redis:0 allocation score on metal-2: -INFINITY
+clone_color: redis:0 allocation score on metal-3: -INFINITY
+clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: redis:0 allocation score on redis-bundle-0: INFINITY
+clone_color: redis:0 allocation score on redis-bundle-1: -INFINITY
+clone_color: redis:0 allocation score on redis-bundle-2: -INFINITY
+clone_color: redis:1 allocation score on galera-bundle-0: -INFINITY
+clone_color: redis:1 allocation score on galera-bundle-1: -INFINITY
+clone_color: redis:1 allocation score on galera-bundle-2: -INFINITY
+clone_color: redis:1 allocation score on metal-1: -INFINITY
+clone_color: redis:1 allocation score on metal-2: -INFINITY
+clone_color: redis:1 allocation score on metal-3: -INFINITY
+clone_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: redis:1 allocation score on redis-bundle-0: -INFINITY
+clone_color: redis:1 allocation score on redis-bundle-1: INFINITY
+clone_color: redis:1 allocation score on redis-bundle-2: -INFINITY
+clone_color: redis:2 allocation score on galera-bundle-0: -INFINITY
+clone_color: redis:2 allocation score on galera-bundle-1: -INFINITY
+clone_color: redis:2 allocation score on galera-bundle-2: -INFINITY
+clone_color: redis:2 allocation score on metal-1: -INFINITY
+clone_color: redis:2 allocation score on metal-2: -INFINITY
+clone_color: redis:2 allocation score on metal-3: -INFINITY
+clone_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY
+clone_color: redis:2 allocation score on redis-bundle-0: -INFINITY
+clone_color: redis:2 allocation score on redis-bundle-1: -INFINITY
+clone_color: redis:2 allocation score on redis-bundle-2: INFINITY
+clone_color: storage-clone allocation score on metal-1: -INFINITY
+clone_color: storage-clone allocation score on metal-2: 0
+clone_color: storage-clone allocation score on metal-3: 0
+clone_color: storage-clone allocation score on rabbitmq-bundle-0: 0
+clone_color: storage:0 allocation score on metal-1: -INFINITY
+clone_color: storage:0 allocation score on metal-2: 0
+clone_color: storage:0 allocation score on metal-3: 0
+clone_color: storage:0 allocation score on rabbitmq-bundle-0: 0
+clone_color: storage:1 allocation score on metal-1: -INFINITY
+clone_color: storage:1 allocation score on metal-2: INFINITY
+clone_color: storage:1 allocation score on metal-3: 0
+clone_color: storage:1 allocation score on rabbitmq-bundle-0: 0
+clone_color: storage:2 allocation score on metal-1: -INFINITY
+clone_color: storage:2 allocation score on metal-2: 0
+clone_color: storage:2 allocation score on metal-3: INFINITY
+clone_color: storage:2 allocation score on rabbitmq-bundle-0: 0
+clone_color: storage:3 allocation score on metal-1: -INFINITY
+clone_color: storage:3 allocation score on metal-2: 0
+clone_color: storage:3 allocation score on metal-3: 0
+clone_color: storage:3 allocation score on rabbitmq-bundle-0: 0
+container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle allocation score on metal-1: 0
+container_color: galera-bundle allocation score on metal-2: 0
+container_color: galera-bundle allocation score on metal-3: 0
+container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle-0 allocation score on metal-1: INFINITY
+container_color: galera-bundle-0 allocation score on metal-2: 0
+container_color: galera-bundle-0 allocation score on metal-3: 0
+container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-1 allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-1 allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle-1 allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle-1 allocation score on metal-1: 0
+container_color: galera-bundle-1 allocation score on metal-2: INFINITY
+container_color: galera-bundle-1 allocation score on metal-3: 0
+container_color: galera-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-2 allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-2 allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle-2 allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle-2 allocation score on metal-1: 0
+container_color: galera-bundle-2 allocation score on metal-2: 0
+container_color: galera-bundle-2 allocation score on metal-3: INFINITY
+container_color: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle-docker-0 allocation score on metal-1: INFINITY
+container_color: galera-bundle-docker-0 allocation score on metal-2: 0
+container_color: galera-bundle-docker-0 allocation score on metal-3: 0
+container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle-docker-1 allocation score on metal-1: 0
+container_color: galera-bundle-docker-1 allocation score on metal-2: INFINITY
+container_color: galera-bundle-docker-1 allocation score on metal-3: 0
+container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle-docker-2 allocation score on metal-1: 0
+container_color: galera-bundle-docker-2 allocation score on metal-2: 0
+container_color: galera-bundle-docker-2 allocation score on metal-3: INFINITY
+container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY
+container_color: galera-bundle-master allocation score on galera-bundle-1: -INFINITY
+container_color: galera-bundle-master allocation score on galera-bundle-2: -INFINITY
+container_color: galera-bundle-master allocation score on metal-1: 0
+container_color: galera-bundle-master allocation score on metal-2: 0
+container_color: galera-bundle-master allocation score on metal-3: 0
+container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: 0
+container_color: galera:0 allocation score on galera-bundle-0: -INFINITY
+container_color: galera:0 allocation score on galera-bundle-1: -INFINITY
+container_color: galera:0 allocation score on galera-bundle-2: -INFINITY
+container_color: galera:0 allocation score on metal-1: 0
+container_color: galera:0 allocation score on metal-2: 0
+container_color: galera:0 allocation score on metal-3: 0
+container_color: galera:0 allocation score on rabbitmq-bundle-0: 0
+container_color: galera:1 allocation score on galera-bundle-0: -INFINITY
+container_color: galera:1 allocation score on galera-bundle-1: -INFINITY
+container_color: galera:1 allocation score on galera-bundle-2: -INFINITY
+container_color: galera:1 allocation score on metal-1: 0
+container_color: galera:1 allocation score on metal-2: 0
+container_color: galera:1 allocation score on metal-3: 0
+container_color: galera:1 allocation score on rabbitmq-bundle-0: 0
+container_color: galera:2 allocation score on galera-bundle-0: -INFINITY
+container_color: galera:2 allocation score on galera-bundle-1: -INFINITY
+container_color: galera:2 allocation score on galera-bundle-2: -INFINITY
+container_color: galera:2 allocation score on metal-1: 0
+container_color: galera:2 allocation score on metal-2: 0
+container_color: galera:2 allocation score on metal-3: 0
+container_color: galera:2 allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle allocation score on metal-1: 0
+container_color: haproxy-bundle allocation score on metal-1: 0
+container_color: haproxy-bundle allocation score on metal-1: 0
+container_color: haproxy-bundle allocation score on metal-1: 0
+container_color: haproxy-bundle allocation score on metal-2: 0
+container_color: haproxy-bundle allocation score on metal-2: 0
+container_color: haproxy-bundle allocation score on metal-2: 0
+container_color: haproxy-bundle allocation score on metal-2: 0
+container_color: haproxy-bundle allocation score on metal-3: 0
+container_color: haproxy-bundle allocation score on metal-3: 0
+container_color: haproxy-bundle allocation score on metal-3: 0
+container_color: haproxy-bundle allocation score on metal-3: 0
+container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on metal-1: INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on metal-1: INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on metal-1: INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on metal-1: INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on metal-1: 0
+container_color: haproxy-bundle-docker-1 allocation score on metal-2: INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on metal-2: INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on metal-2: INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on metal-2: INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
+container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: 0
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-1: 0
+container_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-2: 0
+container_color: haproxy-bundle-docker-2 allocation score on metal-3: INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-3: INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-3: INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on metal-3: INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: 0
+container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle allocation score on metal-1: 0
+container_color: redis-bundle allocation score on metal-2: 0
+container_color: redis-bundle allocation score on metal-3: 0
+container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle allocation score on redis-bundle-2: -INFINITY
+container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-0 allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle-0 allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle-0 allocation score on metal-1: INFINITY
+container_color: redis-bundle-0 allocation score on metal-2: 0
+container_color: redis-bundle-0 allocation score on metal-3: 0
+container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-0 allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-0 allocation score on redis-bundle-2: -INFINITY
+container_color: redis-bundle-1 allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-1 allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle-1 allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle-1 allocation score on metal-1: 0
+container_color: redis-bundle-1 allocation score on metal-2: INFINITY
+container_color: redis-bundle-1 allocation score on metal-3: 0
+container_color: redis-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle-1 allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-1 allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-1 allocation score on redis-bundle-2: -INFINITY
+container_color: redis-bundle-2 allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-2 allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle-2 allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle-2 allocation score on metal-1: 0
+container_color: redis-bundle-2 allocation score on metal-2: 0
+container_color: redis-bundle-2 allocation score on metal-3: INFINITY
+container_color: redis-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle-2 allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-2 allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-2 allocation score on redis-bundle-2: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on metal-1: INFINITY
+container_color: redis-bundle-docker-0 allocation score on metal-2: 0
+container_color: redis-bundle-docker-0 allocation score on metal-3: 0
+container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on metal-1: 0
+container_color: redis-bundle-docker-1 allocation score on metal-2: INFINITY
+container_color: redis-bundle-docker-1 allocation score on metal-3: 0
+container_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on metal-1: 0
+container_color: redis-bundle-docker-2 allocation score on metal-2: 0
+container_color: redis-bundle-docker-2 allocation score on metal-3: INFINITY
+container_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY
+container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
+container_color: redis-bundle-master allocation score on galera-bundle-1: -INFINITY
+container_color: redis-bundle-master allocation score on galera-bundle-2: -INFINITY
+container_color: redis-bundle-master allocation score on metal-1: 0
+container_color: redis-bundle-master allocation score on metal-2: 0
+container_color: redis-bundle-master allocation score on metal-3: 0
+container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: 0
+container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-master allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-master allocation score on redis-bundle-2: -INFINITY
+container_color: redis:0 allocation score on galera-bundle-0: -INFINITY
+container_color: redis:0 allocation score on galera-bundle-1: -INFINITY
+container_color: redis:0 allocation score on galera-bundle-2: -INFINITY
+container_color: redis:0 allocation score on metal-1: 0
+container_color: redis:0 allocation score on metal-2: 0
+container_color: redis:0 allocation score on metal-3: 0
+container_color: redis:0 allocation score on rabbitmq-bundle-0: 0
+container_color: redis:0 allocation score on redis-bundle-0: -INFINITY
+container_color: redis:0 allocation score on redis-bundle-1: -INFINITY
+container_color: redis:0 allocation score on redis-bundle-2: -INFINITY
+container_color: redis:1 allocation score on galera-bundle-0: -INFINITY
+container_color: redis:1 allocation score on galera-bundle-1: -INFINITY
+container_color: redis:1 allocation score on galera-bundle-2: -INFINITY
+container_color: redis:1 allocation score on metal-1: 0
+container_color: redis:1 allocation score on metal-2: 0
+container_color: redis:1 allocation score on metal-3: 0
+container_color: redis:1 allocation score on rabbitmq-bundle-0: 0
+container_color: redis:1 allocation score on redis-bundle-0: -INFINITY
+container_color: redis:1 allocation score on redis-bundle-1: -INFINITY
+container_color: redis:1 allocation score on redis-bundle-2: -INFINITY
+container_color: redis:2 allocation score on galera-bundle-0: -INFINITY
+container_color: redis:2 allocation score on galera-bundle-1: -INFINITY
+container_color: redis:2 allocation score on galera-bundle-2: -INFINITY
+container_color: redis:2 allocation score on metal-1: 0
+container_color: redis:2 allocation score on metal-2: 0
+container_color: redis:2 allocation score on metal-3: 0
+container_color: redis:2 allocation score on rabbitmq-bundle-0: 0
+container_color: redis:2 allocation score on redis-bundle-0: -INFINITY
+container_color: redis:2 allocation score on redis-bundle-1: -INFINITY
+container_color: redis:2 allocation score on redis-bundle-2: -INFINITY
+galera:0 promotion score on galera-bundle-0: -1
+galera:1 promotion score on galera-bundle-1: -1
+galera:2 promotion score on galera-bundle-2: -1
+native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY
+native_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY
+native_color: galera-bundle-0 allocation score on metal-1: INFINITY
+native_color: galera-bundle-0 allocation score on metal-2: -10000
+native_color: galera-bundle-0 allocation score on metal-3: -10000
+native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera-bundle-1 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-1 allocation score on galera-bundle-1: -INFINITY
+native_color: galera-bundle-1 allocation score on galera-bundle-2: -INFINITY
+native_color: galera-bundle-1 allocation score on metal-1: 0
+native_color: galera-bundle-1 allocation score on metal-2: INFINITY
+native_color: galera-bundle-1 allocation score on metal-3: 0
+native_color: galera-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera-bundle-2 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-2 allocation score on galera-bundle-1: -INFINITY
+native_color: galera-bundle-2 allocation score on galera-bundle-2: -INFINITY
+native_color: galera-bundle-2 allocation score on metal-1: 0
+native_color: galera-bundle-2 allocation score on metal-2: 0
+native_color: galera-bundle-2 allocation score on metal-3: INFINITY
+native_color: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on metal-1: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on metal-1: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on metal-2: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on metal-2: 0
+native_color: galera-bundle-docker-0 allocation score on metal-3: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on metal-3: 0
+native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+native_color: galera-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+native_color: galera-bundle-docker-1 allocation score on metal-1: -INFINITY
+native_color: galera-bundle-docker-1 allocation score on metal-2: INFINITY
+native_color: galera-bundle-docker-1 allocation score on metal-3: 0
+native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+native_color: galera-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+native_color: galera-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+native_color: galera-bundle-docker-2 allocation score on metal-1: -INFINITY
+native_color: galera-bundle-docker-2 allocation score on metal-2: -INFINITY
+native_color: galera-bundle-docker-2 allocation score on metal-3: INFINITY
+native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera:0 allocation score on galera-bundle-0: INFINITY
+native_color: galera:0 allocation score on galera-bundle-1: -INFINITY
+native_color: galera:0 allocation score on galera-bundle-2: -INFINITY
+native_color: galera:0 allocation score on metal-1: -INFINITY
+native_color: galera:0 allocation score on metal-2: -INFINITY
+native_color: galera:0 allocation score on metal-3: -INFINITY
+native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera:1 allocation score on galera-bundle-0: -INFINITY
+native_color: galera:1 allocation score on galera-bundle-1: INFINITY
+native_color: galera:1 allocation score on galera-bundle-2: -INFINITY
+native_color: galera:1 allocation score on metal-1: -INFINITY
+native_color: galera:1 allocation score on metal-2: -INFINITY
+native_color: galera:1 allocation score on metal-3: -INFINITY
+native_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: galera:2 allocation score on galera-bundle-0: -INFINITY
+native_color: galera:2 allocation score on galera-bundle-1: -INFINITY
+native_color: galera:2 allocation score on galera-bundle-2: INFINITY
+native_color: galera:2 allocation score on metal-1: -INFINITY
+native_color: galera:2 allocation score on metal-2: -INFINITY
+native_color: galera:2 allocation score on metal-3: -INFINITY
+native_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+native_color: haproxy-bundle-docker-0 allocation score on metal-1: INFINITY
+native_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
+native_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
+native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+native_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
+native_color: haproxy-bundle-docker-1 allocation score on metal-2: INFINITY
+native_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
+native_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+native_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
+native_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
+native_color: haproxy-bundle-docker-2 allocation score on metal-3: INFINITY
+native_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
+native_color: redis-bundle-0 allocation score on galera-bundle-1: -INFINITY
+native_color: redis-bundle-0 allocation score on galera-bundle-2: -INFINITY
+native_color: redis-bundle-0 allocation score on metal-1: INFINITY
+native_color: redis-bundle-0 allocation score on metal-2: 0
+native_color: redis-bundle-0 allocation score on metal-3: 0
+native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
+native_color: redis-bundle-0 allocation score on redis-bundle-1: -INFINITY
+native_color: redis-bundle-0 allocation score on redis-bundle-2: -INFINITY
+native_color: redis-bundle-1 allocation score on galera-bundle-0: -INFINITY
+native_color: redis-bundle-1 allocation score on galera-bundle-1: -INFINITY
+native_color: redis-bundle-1 allocation score on galera-bundle-2: -INFINITY
+native_color: redis-bundle-1 allocation score on metal-1: 0
+native_color: redis-bundle-1 allocation score on metal-2: INFINITY
+native_color: redis-bundle-1 allocation score on metal-3: 0
+native_color: redis-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-1 allocation score on redis-bundle-0: -INFINITY
+native_color: redis-bundle-1 allocation score on redis-bundle-1: -INFINITY
+native_color: redis-bundle-1 allocation score on redis-bundle-2: -INFINITY
+native_color: redis-bundle-2 allocation score on galera-bundle-0: -INFINITY
+native_color: redis-bundle-2 allocation score on galera-bundle-1: -INFINITY
+native_color: redis-bundle-2 allocation score on galera-bundle-2: -INFINITY
+native_color: redis-bundle-2 allocation score on metal-1: 0
+native_color: redis-bundle-2 allocation score on metal-2: 0
+native_color: redis-bundle-2 allocation score on metal-3: INFINITY
+native_color: redis-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-2 allocation score on redis-bundle-0: -INFINITY
+native_color: redis-bundle-2 allocation score on redis-bundle-1: -INFINITY
+native_color: redis-bundle-2 allocation score on redis-bundle-2: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on metal-1: INFINITY
+native_color: redis-bundle-docker-0 allocation score on metal-2: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on metal-3: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on metal-1: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on metal-2: INFINITY
+native_color: redis-bundle-docker-1 allocation score on metal-3: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on metal-1: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on metal-2: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on metal-3: INFINITY
+native_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY
+native_color: redis:0 allocation score on galera-bundle-0: -INFINITY
+native_color: redis:0 allocation score on galera-bundle-1: -INFINITY
+native_color: redis:0 allocation score on galera-bundle-2: -INFINITY
+native_color: redis:0 allocation score on metal-1: -INFINITY
+native_color: redis:0 allocation score on metal-2: -INFINITY
+native_color: redis:0 allocation score on metal-3: -INFINITY
+native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis:0 allocation score on redis-bundle-0: INFINITY
+native_color: redis:0 allocation score on redis-bundle-1: -INFINITY
+native_color: redis:0 allocation score on redis-bundle-2: -INFINITY
+native_color: redis:1 allocation score on galera-bundle-0: -INFINITY
+native_color: redis:1 allocation score on galera-bundle-1: -INFINITY
+native_color: redis:1 allocation score on galera-bundle-2: -INFINITY
+native_color: redis:1 allocation score on metal-1: -INFINITY
+native_color: redis:1 allocation score on metal-2: -INFINITY
+native_color: redis:1 allocation score on metal-3: -INFINITY
+native_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis:1 allocation score on redis-bundle-0: -INFINITY
+native_color: redis:1 allocation score on redis-bundle-1: INFINITY
+native_color: redis:1 allocation score on redis-bundle-2: -INFINITY
+native_color: redis:2 allocation score on galera-bundle-0: -INFINITY
+native_color: redis:2 allocation score on galera-bundle-1: -INFINITY
+native_color: redis:2 allocation score on galera-bundle-2: -INFINITY
+native_color: redis:2 allocation score on metal-1: -INFINITY
+native_color: redis:2 allocation score on metal-2: -INFINITY
+native_color: redis:2 allocation score on metal-3: -INFINITY
+native_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: redis:2 allocation score on redis-bundle-0: -INFINITY
+native_color: redis:2 allocation score on redis-bundle-1: -INFINITY
+native_color: redis:2 allocation score on redis-bundle-2: INFINITY
+native_color: storage:0 allocation score on metal-1: -INFINITY
+native_color: storage:0 allocation score on metal-2: -INFINITY
+native_color: storage:0 allocation score on metal-3: -INFINITY
+native_color: storage:0 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: storage:1 allocation score on metal-1: -INFINITY
+native_color: storage:1 allocation score on metal-2: INFINITY
+native_color: storage:1 allocation score on metal-3: 0
+native_color: storage:1 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: storage:2 allocation score on metal-1: -INFINITY
+native_color: storage:2 allocation score on metal-2: -INFINITY
+native_color: storage:2 allocation score on metal-3: INFINITY
+native_color: storage:2 allocation score on rabbitmq-bundle-0: -INFINITY
+native_color: storage:3 allocation score on metal-1: -INFINITY
+native_color: storage:3 allocation score on metal-2: -INFINITY
+native_color: storage:3 allocation score on metal-3: -INFINITY
+native_color: storage:3 allocation score on rabbitmq-bundle-0: -INFINITY
+redis:0 promotion score on redis-bundle-0: 99
+redis:1 promotion score on redis-bundle-1: 99
+redis:2 promotion score on redis-bundle-2: 99
diff --git a/pengine/test10/bundle-order-stop-clone.summary b/pengine/test10/bundle-order-stop-clone.summary
new file mode 100644
index 0000000000..404eecde99
--- /dev/null
+++ b/pengine/test10/bundle-order-stop-clone.summary
@@ -0,0 +1,75 @@
+
+Current cluster status:
+Online: [ metal-1 metal-2 metal-3 ]
+RemoteOFFLINE: [ rabbitmq-bundle-0 ]
+Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ]
+
+ Clone Set: storage-clone [storage]
+ Started: [ metal-1 metal-2 metal-3 ]
+ Stopped: [ rabbitmq-bundle-0 ]
+ Docker container set: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
+ galera-bundle-0 (ocf::heartbeat:galera): Slave metal-1
+ galera-bundle-1 (ocf::heartbeat:galera): Slave metal-2
+ galera-bundle-2 (ocf::heartbeat:galera): Slave metal-3
+ Docker container set: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
+ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started metal-1
+ haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started metal-2
+ haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started metal-3
+ Docker container set: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
+ redis-bundle-0 (ocf::heartbeat:redis): Master metal-1
+ redis-bundle-1 (ocf::heartbeat:redis): Master metal-2
+ redis-bundle-2 (ocf::heartbeat:redis): Master metal-3
+
+Transition Summary:
+ * Stop storage:0 (metal-1)
+ * Stop galera-bundle-docker-0 (metal-1)
+ * Stop galera-bundle-0 (Started metal-1)
+ * Stop galera:0 (Slave galera-bundle-0)
+
+Executing cluster transition:
+ * Pseudo action: storage-clone_pre_notify_stop_0
+ * Pseudo action: galera-bundle_stop_0
+ * Resource action: storage:0 notify on metal-1
+ * Resource action: storage:1 notify on metal-2
+ * Resource action: storage:2 notify on metal-3
+ * Pseudo action: storage-clone_confirmed-pre_notify_stop_0
+ * Pseudo action: galera-bundle-master_stop_0
+ * Resource action: galera:0 stop on galera-bundle-0
+ * Resource action: galera-bundle-0 stop on metal-1
+ * Pseudo action: galera-bundle-master_stopped_0
+ * Resource action: galera-bundle-docker-0 stop on metal-1
+ * Pseudo action: galera-bundle_stopped_0
+ * Pseudo action: galera-bundle_start_0
+ * Pseudo action: storage-clone_stop_0
+ * Pseudo action: galera-bundle-master_start_0
+ * Resource action: storage:0 stop on metal-1
+ * Pseudo action: storage-clone_stopped_0
+ * Pseudo action: galera-bundle-master_running_0
+ * Pseudo action: galera-bundle_running_0
+ * Pseudo action: storage-clone_post_notify_stopped_0
+ * Resource action: storage:1 notify on metal-2
+ * Resource action: storage:2 notify on metal-3
+ * Pseudo action: storage-clone_confirmed-post_notify_stopped_0
+ * Pseudo action: all_stopped
+
+Revised cluster status:
+Online: [ metal-1 metal-2 metal-3 ]
+RemoteOFFLINE: [ rabbitmq-bundle-0 ]
+Containers: [ galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ]
+
+ Clone Set: storage-clone [storage]
+ Started: [ metal-2 metal-3 ]
+ Stopped: [ metal-1 rabbitmq-bundle-0 ]
+ Docker container set: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
+ galera-bundle-0 (ocf::heartbeat:galera): Stopped
+ galera-bundle-1 (ocf::heartbeat:galera): Slave metal-2
+ galera-bundle-2 (ocf::heartbeat:galera): Slave metal-3
+ Docker container set: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
+ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started metal-1
+ haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started metal-2
+ haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started metal-3
+ Docker container set: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
+ redis-bundle-0 (ocf::heartbeat:redis): Master metal-1
+ redis-bundle-1 (ocf::heartbeat:redis): Master metal-2
+ redis-bundle-2 (ocf::heartbeat:redis): Master metal-3
+
diff --git a/pengine/test10/bundle-order-stop-clone.xml b/pengine/test10/bundle-order-stop-clone.xml
new file mode 100644
index 0000000000..60db64d716
--- /dev/null
+++ b/pengine/test10/bundle-order-stop-clone.xml
@@ -0,0 +1,398 @@
+<cib crm_feature_set="3.0.12" validate-with="pacemaker-2.8" epoch="58" num_updates="222" admin_epoch="0" cib-last-written="Thu Jun 1 03:59:09 2017" update-origin="undercloud" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-10.el7-94ff4df"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/>
+ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
+ </cluster_property_set>
+ <cluster_property_set id="redis_replication">
+ <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="redis-bundle-0"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="metal-1">
+ <instance_attributes id="nodes-1">
+ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/>
+ <nvpair id="nodes-1-galera-role" name="galera-role" value="true"/>
+ <nvpair id="nodes-1-redis-role" name="redis-role" value="true"/>
+ <nvpair id="nodes-1-haproxy-role" name="haproxy-role" value="true"/>
+ <nvpair id="nodes-1-cinder-volume-role" name="cinder-volume-role" value="true"/>
+ </instance_attributes>
+ </node>
+ <node id="2" uname="metal-2">
+ <instance_attributes id="nodes-2">
+ <nvpair id="nodes-2-rabbitmq-role" name="rabbitmq-role" value="true"/>
+ <nvpair id="nodes-2-galera-role" name="galera-role" value="true"/>
+ <nvpair id="nodes-2-redis-role" name="redis-role" value="true"/>
+ <nvpair id="nodes-2-haproxy-role" name="haproxy-role" value="true"/>
+ <nvpair id="nodes-2-cinder-volume-role" name="cinder-volume-role" value="true"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="metal-3">
+ <instance_attributes id="nodes-3">
+ <nvpair id="nodes-3-rabbitmq-role" name="rabbitmq-role" value="true"/>
+ <nvpair id="nodes-3-galera-role" name="galera-role" value="true"/>
+ <nvpair id="nodes-3-redis-role" name="redis-role" value="true"/>
+ <nvpair id="nodes-3-haproxy-role" name="haproxy-role" value="true"/>
+ <nvpair id="nodes-3-cinder-volume-role" name="cinder-volume-role" value="true"/>
+ </instance_attributes>
+ </node>
+ <node id="rabbitmq-bundle-0" type="remote" uname="rabbitmq-bundle-0">
+ <instance_attributes id="nodes-rabbitmq-bundle-0">
+ <nvpair id="nodes-rabbitmq-bundle-0-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@metal"/>
+ </instance_attributes>
+ </node>
+ </nodes>
+ <resources>
+ <clone id="storage-clone">
+ <meta_attributes id="storage_ma">
+ <nvpair id="storage_globally-unique" name="globally-unique" value="false"/>
+ <nvpair id="storage_notify" name="notify" value="true"/>
+ </meta_attributes>
+ <primitive class="ocf" id="storage" provider="heartbeat" type="Filesystem">
+ <operations>
+ <op id="storage_mon" interval="30s" name="monitor" timeout="30s"/>
+ </operations>
+ <instance_attributes id="storage_ia">
+ <nvpair id="storage_attr_0" name="device" value="nfs:/share/drbd_www/data/"/>
+ <nvpair id="storage_attr_1" name="directory" value="/data/www"/>
+ <nvpair id="storage_attr_2" name="fstype" value="nfs"/>
+ </instance_attributes>
+ </primitive>
+ </clone>
+ <bundle id="galera-bundle">
+ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" masters="3" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" run-command="/bin/bash /usr/local/bin/kolla_start"/>
+ <network control-port="3123"/>
+ <storage>
+ <storage-mapping id="mysql-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/mysql.json" target-dir="/var/lib/kolla/config_files/config.json"/>
+ <storage-mapping id="mysql-cfg-data" options="ro" source-dir="/var/lib/config-data/mysql" target-dir="/var/lib/kolla/config_files/src"/>
+ <storage-mapping id="mysql-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
+ <storage-mapping id="mysql-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
+ <storage-mapping id="mysql-lib" options="rw" source-dir="/var/lib/mysql" target-dir="/var/lib/mysql"/>
+ <storage-mapping id="mysql-log-mariadb" options="rw" source-dir="/var/log/mariadb" target-dir="/var/log/mariadb"/>
+ <storage-mapping id="mysql-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
+ <storage-mapping id="mysql-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
+ <storage-mapping id="mysql-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
+ <storage-mapping id="mysql-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
+ <storage-mapping id="mysql-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
+ </storage>
+ <primitive class="ocf" id="galera" provider="heartbeat" type="galera">
+ <instance_attributes id="galera-instance_attributes">
+ <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/>
+ <nvpair id="galera-instance_attributes-cluster_host_map" name="cluster_host_map" value="galera-bundle-0:metal"/>
+ <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/>
+ <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://metal"/>
+ </instance_attributes>
+ <meta_attributes id="galera-meta_attributes">
+ <nvpair id="galera-meta_attributes-master-max" name="master-max" value="1"/>
+ <nvpair id="galera-meta_attributes-ordered" name="ordered" value="true"/>
+ </meta_attributes>
+ <operations>
+ <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
+ <op id="galera-monitor-interval-20" interval="20" name="monitor" timeout="30"/>
+ <op id="galera-monitor-interval-10" interval="10" name="monitor" role="Master" timeout="30"/>
+ <op id="galera-monitor-interval-30" interval="30" name="monitor" role="Slave" timeout="30"/>
+ <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/>
+ <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120"/>
+ <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120"/>
+ </operations>
+ </primitive>
+ </bundle>
+ <bundle id="haproxy-bundle">
+ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="3" run-command="/bin/bash /usr/local/bin/kolla_start"/>
+ <storage>
+ <storage-mapping id="haproxy-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/haproxy.json" target-dir="/var/lib/kolla/config_files/config.json"/>
+ <storage-mapping id="haproxy-cfg-data" options="ro" source-dir="/var/lib/config-data/haproxy/etc" target-dir="/etc"/>
+ <storage-mapping id="haproxy-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
+ <storage-mapping id="haproxy-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
+ <storage-mapping id="haproxy-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
+ <storage-mapping id="haproxy-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
+ <storage-mapping id="haproxy-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
+ <storage-mapping id="haproxy-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
+ <storage-mapping id="haproxy-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
+ </storage>
+ </bundle>
+ <bundle id="redis-bundle">
+ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" masters="3" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" run-command="/bin/bash /usr/local/bin/kolla_start"/>
+ <network control-port="3124"/>
+ <storage>
+ <storage-mapping id="redis-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/redis.json" target-dir="/var/lib/kolla/config_files/config.json"/>
+ <storage-mapping id="redis-cfg-data-redis" options="ro" source-dir="/var/lib/config-data/redis/etc/redis" target-dir="/etc/redis"/>
+ <storage-mapping id="redis-cfg-data-redis-conf" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf" target-dir="/etc/redis.conf"/>
+ <storage-mapping id="redis-cfg-data-redis-conf-puppet" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf.puppet" target-dir="/etc/redis.conf.puppet"/>
+ <storage-mapping id="redis-cfg-data-redis-sentinel" options="ro" source-dir="/var/lib/config-data/redis/etc/redis-sentinel.conf" target-dir="/etc/redis-sentinel.conf"/>
+ <storage-mapping id="redis-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
+ <storage-mapping id="redis-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
+ <storage-mapping id="redis-lib" options="rw" source-dir="/var/lib/redis" target-dir="/var/lib/redis"/>
+ <storage-mapping id="redis-log" options="rw" source-dir="/var/log/redis" target-dir="/var/log/redis"/>
+ <storage-mapping id="redis-run" options="rw" source-dir="/var/run/redis" target-dir="/var/run/redis"/>
+ <storage-mapping id="redis-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
+ <storage-mapping id="redis-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
+ <storage-mapping id="redis-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
+ <storage-mapping id="redis-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
+ <storage-mapping id="redis-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
+ </storage>
+ <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
+ <instance_attributes id="redis-instance_attributes">
+ <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
+ </instance_attributes>
+ <meta_attributes id="redis-meta_attributes">
+ <nvpair id="redis-meta_attributes-interleave" name="interleave" value="true"/>
+ <nvpair id="redis-meta_attributes-notify" name="notify" value="true"/>
+ <nvpair id="redis-meta_attributes-ordered" name="ordered" value="true"/>
+ </meta_attributes>
+ <operations>
+ <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
+ <op id="redis-monitor-interval-45" interval="45" name="monitor" timeout="60"/>
+ <op id="redis-monitor-interval-20" interval="20" name="monitor" role="Master" timeout="60"/>
+ <op id="redis-monitor-interval-60" interval="60" name="monitor" role="Slave" timeout="60"/>
+ <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120"/>
+ <op id="redis-start-interval-0s" interval="0s" name="start" timeout="200s"/>
+ <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
+ </operations>
+ </primitive>
+ </bundle>
+ </resources>
+ <constraints>
+ <rsc_location id="location-redis-bundle" resource-discovery="exclusive" rsc="redis-bundle">
+ <rule id="location-redis-bundle-rule" score="0">
+ <expression attribute="redis-role" id="location-redis-bundle-rule-expr" operation="eq" value="true"/>
+ </rule>
+ </rsc_location>
+ <rsc_location id="location-galera-bundle" resource-discovery="exclusive" rsc="galera-bundle">
+ <rule id="location-galera-bundle-rule" score="0">
+ <expression attribute="galera-role" id="location-galera-bundle-rule-expr" operation="eq" value="true"/>
+ </rule>
+ </rsc_location>
+ <rsc_location id="storage-ban-1" rsc="storage-clone" node="metal-1" score="-INFINITY"/>
+ <rsc_location id="make-promote-now-1" rsc="redis-bundle" role="Master" node="redis-bundle-0" score="100"/>
+ <rsc_location id="make-promote-now-2" rsc="redis-bundle" role="Master" node="redis-bundle-1" score="100"/>
+ <rsc_location id="make-promote-now-3" rsc="redis-bundle" role="Master" node="redis-bundle-2" score="100"/>
+ <rsc_colocation id="colocation-1" rsc="redis-bundle" score="INFINITY" with-rsc="haproxy-bundle"/>
+ <rsc_colocation id="colocation-2" rsc="galera-bundle" score="INFINITY" with-rsc="storage-clone"/>
+ <rsc_order first="redis-bundle" first-action="promote" id="order-1" kind="Mandatory" then="storage-clone" then-action="start"/>
+ <rsc_order first="storage-clone" first-action="start" id="order-2" kind="Mandatory" then="galera-bundle" then-action="start"/>
+ <rsc_order first="haproxy-bundle" first-action="start" id="order-3" kind="Mandatory" then="storage-clone" then-action="start"/>
+ </constraints>
+ <rsc_defaults>
+ <meta_attributes id="rsc_defaults-options">
+ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
+ </meta_attributes>
+ </rsc_defaults>
+ </configuration>
+ <status>
+ <node_state id="1" uname="metal-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="storage:0" class="ocf" provider="heartbeat" type="Filesystem">
+ <lrm_rsc_op id="storage_last_0" operation_key="storage_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="011d6ab00aa8677f8496f1a3e768a370"/>
+ <lrm_rsc_op id="storage_post_notify_start_0" operation_key="storage_notify_0" operation="notify" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="011d6ab00aa8677f8496f1a3e768a370"/>
+ <lrm_rsc_op id="storage_monitor_30000" operation_key="storage_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="4" rc-code="0" op-status="0" interval="30000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="5fdd98b808a9cd8496a49fc679446175"/>
+ </lrm_resource>
+ <lrm_resource id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="ed3aad657e7994e60ac4dbd3a200b0b7"/>
+ <lrm_rsc_op id="galera-bundle-docker-0_monitor_60000" operation_key="galera-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="f7e456f82e276e07f726e6e86e93bbb3"/>
+ </lrm_resource>
+ <lrm_resource id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="galera-bundle-docker-1_last_0" operation_key="galera-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="810c6ca70f61efcf84f03ce4ebf1583e"/>
+ </lrm_resource>
+ <lrm_resource id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="galera-bundle-docker-2_last_0" operation_key="galera-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="d382f6658eb81d66313704072c2d9704"/>
+ </lrm_resource>
+ <lrm_resource id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="haproxy-bundle-docker-0_last_0" operation_key="haproxy-bundle-docker-0_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
+ <lrm_rsc_op id="haproxy-bundle-docker-0_monitor_60000" operation_key="haproxy-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="8e98096b4e2923c7be3ce5fa2646c524"/>
+ </lrm_resource>
+ <lrm_resource id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="haproxy-bundle-docker-1_last_0" operation_key="haproxy-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
+ </lrm_resource>
+ <lrm_resource id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="haproxy-bundle-docker-2_last_0" operation_key="haproxy-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="90ccdc75136c9ebfb1985f14df781477"/>
+ <lrm_rsc_op id="redis-bundle-docker-0_monitor_60000" operation_key="redis-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="81e9d490c493f8d22fab2822ec0bed8a"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="redis-bundle-docker-1_last_0" operation_key="redis-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="029dcdb8e6d039453207529ddf76e626"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="redis-bundle-docker-2_last_0" operation_key="redis-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="78dfacba4be0b97e9a6426b342bb2fdd"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote">
+ <lrm_rsc_op id="redis-bundle-0_last_0" operation_key="redis-bundle-0_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="cdc2746943a66a3b93738e3548dfc96d"/>
+ <lrm_rsc_op id="redis-bundle-0_monitor_60000" operation_key="redis-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="e1e2f8a6aaa00cc4f0e71adecd9e2301"/>
+ </lrm_resource>
+ <lrm_resource id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote">
+ <lrm_rsc_op id="galera-bundle-0_last_0" operation_key="galera-bundle-0_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="504a42401176c2e94483b1e091f79a5a"/>
+ <lrm_rsc_op id="galera-bundle-0_monitor_60000" operation_key="galera-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="9901da5f8c52b9011a1b27b74e2c4eee"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="2" uname="metal-2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="storage:1" class="ocf" provider="heartbeat" type="Filesystem">
+ <lrm_rsc_op id="storage_last_0" operation_key="storage_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="011d6ab00aa8677f8496f1a3e768a370"/>
+ <lrm_rsc_op id="storage_post_notify_start_0" operation_key="storage_notify_0" operation="notify" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="011d6ab00aa8677f8496f1a3e768a370"/>
+ <lrm_rsc_op id="storage_monitor_30000" operation_key="storage_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="4" rc-code="0" op-status="0" interval="30000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="5fdd98b808a9cd8496a49fc679446175"/>
+ </lrm_resource>
+ <lrm_resource id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="ed3aad657e7994e60ac4dbd3a200b0b7"/>
+ </lrm_resource>
+ <lrm_resource id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="galera-bundle-docker-1_last_0" operation_key="galera-bundle-docker-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="810c6ca70f61efcf84f03ce4ebf1583e"/>
+ <lrm_rsc_op id="galera-bundle-docker-1_monitor_60000" operation_key="galera-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="9d23f939271b60f65c5d55115613887d"/>
+ </lrm_resource>
+ <lrm_resource id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="galera-bundle-docker-2_last_0" operation_key="galera-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="d382f6658eb81d66313704072c2d9704"/>
+ </lrm_resource>
+ <lrm_resource id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="haproxy-bundle-docker-0_last_0" operation_key="haproxy-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
+ </lrm_resource>
+ <lrm_resource id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="haproxy-bundle-docker-1_last_0" operation_key="haproxy-bundle-docker-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
+ <lrm_rsc_op id="haproxy-bundle-docker-1_monitor_60000" operation_key="haproxy-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="8e98096b4e2923c7be3ce5fa2646c524"/>
+ </lrm_resource>
+ <lrm_resource id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="haproxy-bundle-docker-2_last_0" operation_key="haproxy-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="90ccdc75136c9ebfb1985f14df781477"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="redis-bundle-docker-1_last_0" operation_key="redis-bundle-docker-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="029dcdb8e6d039453207529ddf76e626"/>
+ <lrm_rsc_op id="redis-bundle-docker-1_monitor_60000" operation_key="redis-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="d37abb44bbe352fe0b7d7664151cc6ab"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="redis-bundle-docker-2_last_0" operation_key="redis-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="78dfacba4be0b97e9a6426b342bb2fdd"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-1" class="ocf" provider="pacemaker" type="remote">
+ <lrm_rsc_op id="redis-bundle-1_last_0" operation_key="redis-bundle-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="40c70f9d61dc1fe2dba5a6fd7d5bd567"/>
+ <lrm_rsc_op id="redis-bundle-1_monitor_60000" operation_key="redis-bundle-1_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="869db8f221b6f9b135143fdc50dad3f4"/>
+ </lrm_resource>
+ <lrm_resource id="galera-bundle-1" class="ocf" provider="pacemaker" type="remote">
+ <lrm_rsc_op id="galera-bundle-1_last_0" operation_key="galera-bundle-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="912f68441e818e51b92b60f340f6faa5"/>
+ <lrm_rsc_op id="galera-bundle-1_monitor_60000" operation_key="galera-bundle-1_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="5f3d272e783249eaddff8c313cb262f9"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="3" uname="metal-3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="storage:2" class="ocf" provider="heartbeat" type="Filesystem">
+ <lrm_rsc_op id="storage_last_0" operation_key="storage_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="011d6ab00aa8677f8496f1a3e768a370"/>
+ <lrm_rsc_op id="storage_post_notify_start_0" operation_key="storage_notify_0" operation="notify" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="011d6ab00aa8677f8496f1a3e768a370"/>
+ <lrm_rsc_op id="storage_monitor_30000" operation_key="storage_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="4" rc-code="0" op-status="0" interval="30000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="5fdd98b808a9cd8496a49fc679446175"/>
+ </lrm_resource>
+ <lrm_resource id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="ed3aad657e7994e60ac4dbd3a200b0b7"/>
+ </lrm_resource>
+ <lrm_resource id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="galera-bundle-docker-1_last_0" operation_key="galera-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="810c6ca70f61efcf84f03ce4ebf1583e"/>
+ </lrm_resource>
+ <lrm_resource id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="galera-bundle-docker-2_last_0" operation_key="galera-bundle-docker-2_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="d382f6658eb81d66313704072c2d9704"/>
+ <lrm_rsc_op id="galera-bundle-docker-2_monitor_60000" operation_key="galera-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="7c2c0a8373a43288eb9e4c59a8a11546"/>
+ </lrm_resource>
+ <lrm_resource id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="haproxy-bundle-docker-0_last_0" operation_key="haproxy-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
+ </lrm_resource>
+ <lrm_resource id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="haproxy-bundle-docker-1_last_0" operation_key="haproxy-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
+ </lrm_resource>
+ <lrm_resource id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="haproxy-bundle-docker-2_last_0" operation_key="haproxy-bundle-docker-2_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
+ <lrm_rsc_op id="haproxy-bundle-docker-2_monitor_60000" operation_key="haproxy-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="8e98096b4e2923c7be3ce5fa2646c524"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="90ccdc75136c9ebfb1985f14df781477"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="redis-bundle-docker-1_last_0" operation_key="redis-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="029dcdb8e6d039453207529ddf76e626"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
+ <lrm_rsc_op id="redis-bundle-docker-2_last_0" operation_key="redis-bundle-docker-2_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="78dfacba4be0b97e9a6426b342bb2fdd"/>
+ <lrm_rsc_op id="redis-bundle-docker-2_monitor_60000" operation_key="redis-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="24676507e61be2e5cc014443ba54c560"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-2" class="ocf" provider="pacemaker" type="remote">
+ <lrm_rsc_op id="redis-bundle-2_last_0" operation_key="redis-bundle-2_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="3c205c3ee79f4db4f37351207ebff95e"/>
+ <lrm_rsc_op id="redis-bundle-2_monitor_60000" operation_key="redis-bundle-2_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="3786b0a658dcf4c2d3df6b2cea5cc84b"/>
+ </lrm_resource>
+ <lrm_resource id="galera-bundle-2" class="ocf" provider="pacemaker" type="remote">
+ <lrm_rsc_op id="galera-bundle-2_last_0" operation_key="galera-bundle-2_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="b64c504325efb1370a8509833d3fe8ee"/>
+ <lrm_rsc_op id="galera-bundle-2_monitor_60000" operation_key="galera-bundle-2_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="011536401935dcba9470adbf1db4a633"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="redis-bundle-0" uname="redis-bundle-0">
+ <lrm id="redis-bundle-0">
+ <lrm_resources>
+ <lrm_resource id="redis:0" class="ocf" provider="heartbeat" type="redis">
+ <lrm_rsc_op id="redis_last_0" operation_key="redis_promote_0" operation="promote" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
+ <lrm_rsc_op id="redis_monitor_20000" operation_key="redis_monitor_20000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:8:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:8;3:-1:8:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="8" op-status="0" interval="20000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="redis-bundle-1" uname="redis-bundle-1">
+ <lrm id="redis-bundle-1">
+ <lrm_resources>
+ <lrm_resource id="redis:1" class="ocf" provider="heartbeat" type="redis">
+ <lrm_rsc_op id="redis_last_0" operation_key="redis_promote_0" operation="promote" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
+ <lrm_rsc_op id="redis_monitor_20000" operation_key="redis_monitor_20000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:8:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:8;3:-1:8:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="8" op-status="0" interval="20000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="redis-bundle-2" uname="redis-bundle-2">
+ <lrm id="redis-bundle-2">
+ <lrm_resources>
+ <lrm_resource id="redis:2" class="ocf" provider="heartbeat" type="redis">
+ <lrm_rsc_op id="redis_last_0" operation_key="redis_promote_0" operation="promote" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
+ <lrm_rsc_op id="redis_monitor_20000" operation_key="redis_monitor_20000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:8:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:8;3:-1:8:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="8" op-status="0" interval="20000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="galera-bundle-0" uname="galera-bundle-0">
+ <lrm id="galera-bundle-0">
+ <lrm_resources>
+ <lrm_resource id="galera:0" class="ocf" provider="heartbeat" type="galera">
+ <lrm_rsc_op id="galera_last_0" operation_key="galera_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="a9aaa400c381f8dd43ac3a0cbee94c90"/>
+ <lrm_rsc_op id="galera_monitor_30000" operation_key="galera_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="30000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="372bdbc835a0728d2a83ece1d27f89b8"/>
+ <lrm_rsc_op id="galera_monitor_20000" operation_key="galera_monitor_20000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="20000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="372bdbc835a0728d2a83ece1d27f89b8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="galera-bundle-1" uname="galera-bundle-1">
+ <lrm id="galera-bundle-1">
+ <lrm_resources>
+ <lrm_resource id="galera:1" class="ocf" provider="heartbeat" type="galera">
+ <lrm_rsc_op id="galera_last_0" operation_key="galera_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="a9aaa400c381f8dd43ac3a0cbee94c90"/>
+ <lrm_rsc_op id="galera_monitor_30000" operation_key="galera_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="30000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="372bdbc835a0728d2a83ece1d27f89b8"/>
+ <lrm_rsc_op id="galera_monitor_20000" operation_key="galera_monitor_20000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="20000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="372bdbc835a0728d2a83ece1d27f89b8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="galera-bundle-2" uname="galera-bundle-2">
+ <lrm id="galera-bundle-2">
+ <lrm_resources>
+ <lrm_resource id="galera:2" class="ocf" provider="heartbeat" type="galera">
+ <lrm_rsc_op id="galera_last_0" operation_key="galera_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="a9aaa400c381f8dd43ac3a0cbee94c90"/>
+ <lrm_rsc_op id="galera_monitor_30000" operation_key="galera_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="30000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="372bdbc835a0728d2a83ece1d27f89b8"/>
+ <lrm_rsc_op id="galera_monitor_20000" operation_key="galera_monitor_20000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="20000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="372bdbc835a0728d2a83ece1d27f89b8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ </status>
+</cib>