Page Menu
Home
ClusterLabs Projects
Search
Configure Global Search
Log In
Files
F3686913
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Size
135 KB
Referenced Files
None
Subscribers
None
View Options
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
index 2b53999f9d..b867c09e2a 100644
--- a/lib/pengine/utils.c
+++ b/lib/pengine/utils.c
@@ -1,1904 +1,1905 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <crm/common/util.h>
#include <glib.h>
#include <crm/pengine/rules.h>
#include <crm/pengine/internal.h>
pe_working_set_t *pe_dataset = NULL;
extern xmlNode *get_object_root(const char *object_type, xmlNode * the_root);
void print_str_str(gpointer key, gpointer value, gpointer user_data);
gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data);
void unpack_operation(action_t * action, xmlNode * xml_obj, resource_t * container,
pe_working_set_t * data_set);
static xmlNode *find_rsc_op_entry_helper(resource_t * rsc, const char *key,
gboolean include_disabled);
/*!
* \internal
* \brief Check whether we can fence a particular node
*
* \param[in] data_set Working set for cluster
* \param[in] node Name of node to check
*
* \return TRUE if node can be fenced, FALSE otherwise
*
* \note This function should only be called for cluster nodes and baremetal
* remote nodes; guest nodes are fenced by stopping their container
* resource, so fence execution requirements do not apply to them.
*/
bool pe_can_fence(pe_working_set_t * data_set, node_t *node)
{
if(is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
return FALSE; /* Turned off */
} else if (is_not_set(data_set->flags, pe_flag_have_stonith_resource)) {
return FALSE; /* No devices */
} else if (is_set(data_set->flags, pe_flag_have_quorum)) {
return TRUE;
} else if (data_set->no_quorum_policy == no_quorum_ignore) {
return TRUE;
} else if(node == NULL) {
return FALSE;
} else if(node->details->online) {
crm_notice("We can fence %s without quorum because they're in our membership", node->details->uname);
return TRUE;
}
crm_trace("Cannot fence %s", node->details->uname);
return FALSE;
}
node_t *
node_copy(const node_t *this_node)
{
node_t *new_node = NULL;
CRM_CHECK(this_node != NULL, return NULL);
new_node = calloc(1, sizeof(node_t));
CRM_ASSERT(new_node != NULL);
crm_trace("Copying %p (%s) to %p", this_node, this_node->details->uname, new_node);
new_node->rsc_discover_mode = this_node->rsc_discover_mode;
new_node->weight = this_node->weight;
new_node->fixed = this_node->fixed;
new_node->details = this_node->details;
return new_node;
}
/* any node in list1 or list2 and not in the other gets a score of -INFINITY */
void
node_list_exclude(GHashTable * hash, GListPtr list, gboolean merge_scores)
{
GHashTable *result = hash;
node_t *other_node = NULL;
GListPtr gIter = list;
GHashTableIter iter;
node_t *node = NULL;
g_hash_table_iter_init(&iter, hash);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
other_node = pe_find_node_id(list, node->details->id);
if (other_node == NULL) {
node->weight = -INFINITY;
} else if (merge_scores) {
node->weight = merge_weights(node->weight, other_node->weight);
}
}
for (; gIter != NULL; gIter = gIter->next) {
node_t *node = (node_t *) gIter->data;
other_node = pe_hash_table_lookup(result, node->details->id);
if (other_node == NULL) {
node_t *new_node = node_copy(node);
new_node->weight = -INFINITY;
g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
}
}
}
GHashTable *
node_hash_from_list(GListPtr list)
{
GListPtr gIter = list;
GHashTable *result = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, g_hash_destroy_str);
for (; gIter != NULL; gIter = gIter->next) {
node_t *node = (node_t *) gIter->data;
node_t *n = node_copy(node);
g_hash_table_insert(result, (gpointer) n->details->id, n);
}
return result;
}
GListPtr
node_list_dup(GListPtr list1, gboolean reset, gboolean filter)
{
GListPtr result = NULL;
GListPtr gIter = list1;
for (; gIter != NULL; gIter = gIter->next) {
node_t *new_node = NULL;
node_t *this_node = (node_t *) gIter->data;
if (filter && this_node->weight < 0) {
continue;
}
new_node = node_copy(this_node);
if (reset) {
new_node->weight = 0;
}
if (new_node != NULL) {
result = g_list_prepend(result, new_node);
}
}
return result;
}
gint
sort_node_uname(gconstpointer a, gconstpointer b)
{
const node_t *node_a = a;
const node_t *node_b = b;
return strcmp(node_a->details->uname, node_b->details->uname);
}
void
dump_node_scores_worker(int level, const char *file, const char *function, int line,
resource_t * rsc, const char *comment, GHashTable * nodes)
{
GHashTable *hash = nodes;
GHashTableIter iter;
node_t *node = NULL;
if (rsc) {
hash = rsc->allowed_nodes;
}
if (rsc && is_set(rsc->flags, pe_rsc_orphan)) {
/* Don't show the allocation scores for orphans */
return;
}
if (level == 0) {
char score[128];
int len = sizeof(score);
/* For now we want this in sorted order to keep the regression tests happy */
GListPtr gIter = NULL;
GListPtr list = g_hash_table_get_values(hash);
list = g_list_sort(list, sort_node_uname);
gIter = list;
for (; gIter != NULL; gIter = gIter->next) {
node_t *node = (node_t *) gIter->data;
/* This function is called a whole lot, use stack allocated score */
score2char_stack(node->weight, score, len);
if (rsc) {
printf("%s: %s allocation score on %s: %s\n",
comment, rsc->id, node->details->uname, score);
} else {
printf("%s: %s = %s\n", comment, node->details->uname, score);
}
}
g_list_free(list);
} else if (hash) {
char score[128];
int len = sizeof(score);
g_hash_table_iter_init(&iter, hash);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
/* This function is called a whole lot, use stack allocated score */
score2char_stack(node->weight, score, len);
if (rsc) {
do_crm_log_alias(LOG_TRACE, file, function, line,
"%s: %s allocation score on %s: %s", comment, rsc->id,
node->details->uname, score);
} else {
do_crm_log_alias(LOG_TRACE, file, function, line + 1, "%s: %s = %s", comment,
node->details->uname, score);
}
}
}
if (rsc && rsc->children) {
GListPtr gIter = NULL;
gIter = rsc->children;
for (; gIter != NULL; gIter = gIter->next) {
resource_t *child = (resource_t *) gIter->data;
dump_node_scores_worker(level, file, function, line, child, comment, nodes);
}
}
}
static void
append_dump_text(gpointer key, gpointer value, gpointer user_data)
{
char **dump_text = user_data;
int len = 0;
char *new_text = NULL;
len = strlen(*dump_text) + strlen(" ") + strlen(key) + strlen("=") + strlen(value) + 1;
new_text = calloc(1, len);
sprintf(new_text, "%s %s=%s", *dump_text, (char *)key, (char *)value);
free(*dump_text);
*dump_text = new_text;
}
void
dump_node_capacity(int level, const char *comment, node_t * node)
{
int len = 0;
char *dump_text = NULL;
len = strlen(comment) + strlen(": ") + strlen(node->details->uname) + strlen(" capacity:") + 1;
dump_text = calloc(1, len);
sprintf(dump_text, "%s: %s capacity:", comment, node->details->uname);
g_hash_table_foreach(node->details->utilization, append_dump_text, &dump_text);
if (level == 0) {
fprintf(stdout, "%s\n", dump_text);
} else {
crm_trace("%s", dump_text);
}
free(dump_text);
}
void
dump_rsc_utilization(int level, const char *comment, resource_t * rsc, node_t * node)
{
int len = 0;
char *dump_text = NULL;
len = strlen(comment) + strlen(": ") + strlen(rsc->id) + strlen(" utilization on ")
+ strlen(node->details->uname) + strlen(":") + 1;
dump_text = calloc(1, len);
sprintf(dump_text, "%s: %s utilization on %s:", comment, rsc->id, node->details->uname);
g_hash_table_foreach(rsc->utilization, append_dump_text, &dump_text);
if (level == 0) {
fprintf(stdout, "%s\n", dump_text);
} else {
crm_trace("%s", dump_text);
}
free(dump_text);
}
gint
sort_rsc_index(gconstpointer a, gconstpointer b)
{
const resource_t *resource1 = (const resource_t *)a;
const resource_t *resource2 = (const resource_t *)b;
if (a == NULL && b == NULL) {
return 0;
}
if (a == NULL) {
return 1;
}
if (b == NULL) {
return -1;
}
if (resource1->sort_index > resource2->sort_index) {
return -1;
}
if (resource1->sort_index < resource2->sort_index) {
return 1;
}
return 0;
}
gint
sort_rsc_priority(gconstpointer a, gconstpointer b)
{
const resource_t *resource1 = (const resource_t *)a;
const resource_t *resource2 = (const resource_t *)b;
if (a == NULL && b == NULL) {
return 0;
}
if (a == NULL) {
return 1;
}
if (b == NULL) {
return -1;
}
if (resource1->priority > resource2->priority) {
return -1;
}
if (resource1->priority < resource2->priority) {
return 1;
}
return 0;
}
action_t *
custom_action(resource_t * rsc, char *key, const char *task,
node_t * on_node, gboolean optional, gboolean save_action,
pe_working_set_t * data_set)
{
action_t *action = NULL;
GListPtr possible_matches = NULL;
CRM_CHECK(key != NULL, return NULL);
CRM_CHECK(task != NULL, free(key); return NULL);
if (save_action && rsc != NULL) {
possible_matches = find_actions(rsc->actions, key, on_node);
} else if(save_action) {
#if 0
action = g_hash_table_lookup(data_set->singletons, key);
#else
/* More expensive but takes 'node' into account */
possible_matches = find_actions(data_set->actions, key, on_node);
#endif
}
if(data_set->singletons == NULL) {
data_set->singletons = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL);
}
if (possible_matches != NULL) {
if (g_list_length(possible_matches) > 1) {
pe_warn("Action %s for %s on %s exists %d times",
task, rsc ? rsc->id : "<NULL>",
on_node ? on_node->details->uname : "<NULL>", g_list_length(possible_matches));
}
action = g_list_nth_data(possible_matches, 0);
pe_rsc_trace(rsc, "Found existing action (%d) %s for %s on %s",
action->id, task, rsc ? rsc->id : "<NULL>",
on_node ? on_node->details->uname : "<NULL>");
g_list_free(possible_matches);
}
if (action == NULL) {
if (save_action) {
pe_rsc_trace(rsc, "Creating%s action %d: %s for %s on %s %d",
optional ? "" : " mandatory", data_set->action_id, key,
rsc ? rsc->id : "<NULL>", on_node ? on_node->details->uname : "<NULL>", optional);
}
action = calloc(1, sizeof(action_t));
if (save_action) {
action->id = data_set->action_id++;
} else {
action->id = 0;
}
action->rsc = rsc;
CRM_ASSERT(task != NULL);
action->task = strdup(task);
if (on_node) {
action->node = node_copy(on_node);
}
action->uuid = strdup(key);
pe_set_action_bit(action, pe_action_runnable);
if (optional) {
pe_rsc_trace(rsc, "Set optional on %s", action->uuid);
pe_set_action_bit(action, pe_action_optional);
} else {
pe_clear_action_bit(action, pe_action_optional);
pe_rsc_trace(rsc, "Unset optional on %s", action->uuid);
}
/*
Implied by calloc()...
action->actions_before = NULL;
action->actions_after = NULL;
action->pseudo = FALSE;
action->dumped = FALSE;
action->processed = FALSE;
action->seen_count = 0;
*/
action->extra = g_hash_table_new_full(crm_str_hash, g_str_equal, free, free);
action->meta = g_hash_table_new_full(crm_str_hash, g_str_equal, free, free);
if (save_action) {
data_set->actions = g_list_prepend(data_set->actions, action);
if(rsc == NULL) {
g_hash_table_insert(data_set->singletons, action->uuid, action);
}
}
if (rsc != NULL) {
action->op_entry = find_rsc_op_entry_helper(rsc, key, TRUE);
unpack_operation(action, action->op_entry, rsc->container, data_set);
if (save_action) {
rsc->actions = g_list_prepend(rsc->actions, action);
}
}
if (save_action) {
pe_rsc_trace(rsc, "Action %d created", action->id);
}
}
if (optional == FALSE) {
pe_rsc_trace(rsc, "Unset optional on %s", action->uuid);
pe_clear_action_bit(action, pe_action_optional);
}
if (rsc != NULL) {
enum action_tasks a_task = text2task(action->task);
int warn_level = LOG_TRACE;
if (save_action) {
warn_level = LOG_WARNING;
}
if (is_set(action->flags, pe_action_have_node_attrs) == FALSE
&& action->node != NULL && action->op_entry != NULL) {
pe_set_action_bit(action, pe_action_have_node_attrs);
unpack_instance_attributes(data_set->input, action->op_entry, XML_TAG_ATTR_SETS,
action->node->details->attrs,
action->extra, NULL, FALSE, data_set->now);
}
if (is_set(action->flags, pe_action_pseudo)) {
/* leave untouched */
} else if (action->node == NULL) {
pe_rsc_trace(rsc, "Unset runnable on %s", action->uuid);
pe_clear_action_bit(action, pe_action_runnable);
} else if (is_not_set(rsc->flags, pe_rsc_managed)
&& g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL) == NULL) {
crm_debug("Action %s (unmanaged)", action->uuid);
pe_rsc_trace(rsc, "Set optional on %s", action->uuid);
pe_set_action_bit(action, pe_action_optional);
/* action->runnable = FALSE; */
- } else if (action->node->details->online == FALSE) {
+ } else if (action->node->details->online == FALSE
+ && !is_container_remote_node(action->node)) {
pe_clear_action_bit(action, pe_action_runnable);
do_crm_log(warn_level, "Action %s on %s is unrunnable (offline)",
action->uuid, action->node->details->uname);
if (is_set(action->rsc->flags, pe_rsc_managed)
&& save_action && a_task == stop_rsc) {
pe_fence_node(data_set, action->node, "because node is unclean");
}
} else if (action->node->details->pending) {
pe_clear_action_bit(action, pe_action_runnable);
do_crm_log(warn_level, "Action %s on %s is unrunnable (pending)",
action->uuid, action->node->details->uname);
} else if (action->needs == rsc_req_nothing) {
pe_rsc_trace(rsc, "Action %s does not require anything", action->uuid);
pe_set_action_bit(action, pe_action_runnable);
#if 0
/*
* No point checking this
* - if we don't have quorum we can't stonith anyway
*/
} else if (action->needs == rsc_req_stonith) {
crm_trace("Action %s requires only stonith", action->uuid);
action->runnable = TRUE;
#endif
} else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE
&& data_set->no_quorum_policy == no_quorum_stop) {
pe_clear_action_bit(action, pe_action_runnable);
crm_debug("%s\t%s (cancelled : quorum)", action->node->details->uname, action->uuid);
} else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE
&& data_set->no_quorum_policy == no_quorum_freeze) {
pe_rsc_trace(rsc, "Check resource is already active: %s %s %s %s", rsc->id, action->uuid, role2text(rsc->next_role), role2text(rsc->role));
if (rsc->fns->active(rsc, TRUE) == FALSE || rsc->next_role > rsc->role) {
pe_clear_action_bit(action, pe_action_runnable);
pe_rsc_debug(rsc, "%s\t%s (cancelled : quorum freeze)",
action->node->details->uname, action->uuid);
}
} else {
pe_rsc_trace(rsc, "Action %s is runnable", action->uuid);
pe_set_action_bit(action, pe_action_runnable);
}
if (save_action) {
switch (a_task) {
case stop_rsc:
set_bit(rsc->flags, pe_rsc_stopping);
break;
case start_rsc:
clear_bit(rsc->flags, pe_rsc_starting);
if (is_set(action->flags, pe_action_runnable)) {
set_bit(rsc->flags, pe_rsc_starting);
}
break;
default:
break;
}
}
}
free(key);
return action;
}
static const char *
unpack_operation_on_fail(action_t * action)
{
const char *value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL);
if (safe_str_eq(action->task, CRMD_ACTION_STOP) && safe_str_eq(value, "standby")) {
crm_config_err("on-fail=standby is not allowed for stop actions: %s", action->rsc->id);
return NULL;
} else if (safe_str_eq(action->task, CRMD_ACTION_DEMOTE) && !value) {
/* demote on_fail defaults to master monitor value if present */
xmlNode *operation = NULL;
const char *name = NULL;
const char *role = NULL;
const char *on_fail = NULL;
const char *interval = NULL;
const char *enabled = NULL;
CRM_CHECK(action->rsc != NULL, return NULL);
for (operation = __xml_first_child(action->rsc->ops_xml);
operation && !value; operation = __xml_next_element(operation)) {
if (!crm_str_eq((const char *)operation->name, "op", TRUE)) {
continue;
}
name = crm_element_value(operation, "name");
role = crm_element_value(operation, "role");
on_fail = crm_element_value(operation, XML_OP_ATTR_ON_FAIL);
enabled = crm_element_value(operation, "enabled");
interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
if (!on_fail) {
continue;
} else if (enabled && !crm_is_true(enabled)) {
continue;
} else if (safe_str_neq(name, "monitor") || safe_str_neq(role, "Master")) {
continue;
} else if (crm_get_interval(interval) <= 0) {
continue;
}
value = on_fail;
}
}
return value;
}
static xmlNode *
find_min_interval_mon(resource_t * rsc, gboolean include_disabled)
{
int number = 0;
int min_interval = -1;
const char *name = NULL;
const char *value = NULL;
const char *interval = NULL;
xmlNode *op = NULL;
xmlNode *operation = NULL;
for (operation = __xml_first_child(rsc->ops_xml); operation != NULL;
operation = __xml_next_element(operation)) {
if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
name = crm_element_value(operation, "name");
interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
value = crm_element_value(operation, "enabled");
if (!include_disabled && value && crm_is_true(value) == FALSE) {
continue;
}
if (safe_str_neq(name, RSC_STATUS)) {
continue;
}
number = crm_get_interval(interval);
if (number < 0) {
continue;
}
if (min_interval < 0 || number < min_interval) {
min_interval = number;
op = operation;
}
}
}
return op;
}
void
unpack_operation(action_t * action, xmlNode * xml_obj, resource_t * container,
pe_working_set_t * data_set)
{
int value_i = 0;
unsigned long long interval = 0;
unsigned long long start_delay = 0;
char *value_ms = NULL;
const char *value = NULL;
const char *field = NULL;
CRM_CHECK(action->rsc != NULL, return);
unpack_instance_attributes(data_set->input, data_set->op_defaults, XML_TAG_META_SETS, NULL,
action->meta, NULL, FALSE, data_set->now);
if (xml_obj) {
xmlAttrPtr xIter = NULL;
for (xIter = xml_obj->properties; xIter; xIter = xIter->next) {
const char *prop_name = (const char *)xIter->name;
const char *prop_value = crm_element_value(xml_obj, prop_name);
g_hash_table_replace(action->meta, strdup(prop_name), strdup(prop_value));
}
}
unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_META_SETS,
NULL, action->meta, NULL, FALSE, data_set->now);
unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_ATTR_SETS,
NULL, action->meta, NULL, FALSE, data_set->now);
g_hash_table_remove(action->meta, "id");
field = XML_LRM_ATTR_INTERVAL;
value = g_hash_table_lookup(action->meta, field);
if (value != NULL) {
interval = crm_get_interval(value);
if (interval > 0) {
value_ms = crm_itoa(interval);
g_hash_table_replace(action->meta, strdup(field), value_ms);
} else {
g_hash_table_remove(action->meta, field);
}
}
/* Begin compatibility code ("requires" set on start action not resource) */
value = g_hash_table_lookup(action->meta, "requires");
if (safe_str_neq(action->task, RSC_START)
&& safe_str_neq(action->task, RSC_PROMOTE)) {
action->needs = rsc_req_nothing;
value = "nothing (not start/promote)";
} else if (safe_str_eq(value, "nothing")) {
action->needs = rsc_req_nothing;
} else if (safe_str_eq(value, "quorum")) {
action->needs = rsc_req_quorum;
} else if (safe_str_eq(value, "unfencing")) {
action->needs = rsc_req_stonith;
set_bit(action->rsc->flags, pe_rsc_needs_unfencing);
if (is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
crm_notice("%s requires unfencing but fencing is disabled", action->rsc->id);
}
} else if (is_set(data_set->flags, pe_flag_stonith_enabled)
&& safe_str_eq(value, "fencing")) {
action->needs = rsc_req_stonith;
if (is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
crm_notice("%s requires fencing but fencing is disabled", action->rsc->id);
}
/* End compatibility code */
} else if (is_set(action->rsc->flags, pe_rsc_needs_fencing)) {
action->needs = rsc_req_stonith;
value = "fencing (resource)";
} else if (is_set(action->rsc->flags, pe_rsc_needs_quorum)) {
action->needs = rsc_req_quorum;
value = "quorum (resource)";
} else {
action->needs = rsc_req_nothing;
value = "nothing (resource)";
}
pe_rsc_trace(action->rsc, "\tAction %s requires: %s", action->task, value);
value = unpack_operation_on_fail(action);
if (value == NULL) {
} else if (safe_str_eq(value, "block")) {
action->on_fail = action_fail_block;
g_hash_table_insert(action->meta, strdup(XML_OP_ATTR_ON_FAIL), strdup("block"));
} else if (safe_str_eq(value, "fence")) {
action->on_fail = action_fail_fence;
value = "node fencing";
if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
crm_config_err("Specifying on_fail=fence and" " stonith-enabled=false makes no sense");
action->on_fail = action_fail_stop;
action->fail_role = RSC_ROLE_STOPPED;
value = "stop resource";
}
} else if (safe_str_eq(value, "standby")) {
action->on_fail = action_fail_standby;
value = "node standby";
} else if (safe_str_eq(value, "ignore")
|| safe_str_eq(value, "nothing")) {
action->on_fail = action_fail_ignore;
value = "ignore";
} else if (safe_str_eq(value, "migrate")) {
action->on_fail = action_fail_migrate;
value = "force migration";
} else if (safe_str_eq(value, "stop")) {
action->on_fail = action_fail_stop;
action->fail_role = RSC_ROLE_STOPPED;
value = "stop resource";
} else if (safe_str_eq(value, "restart")) {
action->on_fail = action_fail_recover;
value = "restart (and possibly migrate)";
} else if (safe_str_eq(value, "restart-container")) {
if (container) {
action->on_fail = action_fail_restart_container;
value = "restart container (and possibly migrate)";
} else {
value = NULL;
}
} else {
pe_err("Resource %s: Unknown failure type (%s)", action->rsc->id, value);
value = NULL;
}
/* defaults */
if (value == NULL && container) {
action->on_fail = action_fail_restart_container;
value = "restart container (and possibly migrate) (default)";
/* for baremetal remote nodes, ensure that any failure that results in
* dropping an active connection to a remote node results in fencing of
* the remote node.
*
* There are only two action failures that don't result in fencing.
* 1. probes - probe failures are expected.
* 2. start - a start failure indicates that an active connection does not already
* exist. The user can set op on-fail=fence if they really want to fence start
* failures. */
} else if (value == NULL &&
is_rsc_baremetal_remote_node(action->rsc, data_set) &&
!(safe_str_eq(action->task, CRMD_ACTION_STATUS) && interval == 0) &&
(safe_str_neq(action->task, CRMD_ACTION_START))) {
if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
value = "fence baremetal remote node (default)";
} else {
value = "recover baremetal remote node connection (default)";
}
if (action->rsc->remote_reconnect_interval) {
action->fail_role = RSC_ROLE_STOPPED;
}
action->on_fail = action_fail_reset_remote;
} else if (value == NULL && safe_str_eq(action->task, CRMD_ACTION_STOP)) {
if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
action->on_fail = action_fail_fence;
value = "resource fence (default)";
} else {
action->on_fail = action_fail_block;
value = "resource block (default)";
}
} else if (value == NULL) {
action->on_fail = action_fail_recover;
value = "restart (and possibly migrate) (default)";
}
pe_rsc_trace(action->rsc, "\t%s failure handling: %s", action->task, value);
value = NULL;
if (xml_obj != NULL) {
value = g_hash_table_lookup(action->meta, "role_after_failure");
}
if (value != NULL && action->fail_role == RSC_ROLE_UNKNOWN) {
action->fail_role = text2role(value);
}
/* defaults */
if (action->fail_role == RSC_ROLE_UNKNOWN) {
if (safe_str_eq(action->task, CRMD_ACTION_PROMOTE)) {
action->fail_role = RSC_ROLE_SLAVE;
} else {
action->fail_role = RSC_ROLE_STARTED;
}
}
pe_rsc_trace(action->rsc, "\t%s failure results in: %s", action->task,
role2text(action->fail_role));
field = XML_OP_ATTR_START_DELAY;
value = g_hash_table_lookup(action->meta, field);
if (value != NULL) {
value_i = crm_get_msec(value);
if (value_i < 0) {
value_i = 0;
}
start_delay = value_i;
value_ms = crm_itoa(value_i);
g_hash_table_replace(action->meta, strdup(field), value_ms);
} else if (interval > 0 && g_hash_table_lookup(action->meta, XML_OP_ATTR_ORIGIN)) {
crm_time_t *origin = NULL;
value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ORIGIN);
origin = crm_time_new(value);
if (origin == NULL) {
crm_config_err("Operation %s contained an invalid " XML_OP_ATTR_ORIGIN ": %s",
ID(xml_obj), value);
} else {
crm_time_t *delay = NULL;
int rc = crm_time_compare(origin, data_set->now);
long long delay_s = 0;
int interval_s = (interval / 1000);
crm_trace("Origin: %s, interval: %d", value, interval_s);
/* If 'origin' is in the future, find the most recent "multiple" that occurred in the past */
while(rc > 0) {
crm_time_add_seconds(origin, -interval_s);
rc = crm_time_compare(origin, data_set->now);
}
/* Now find the first "multiple" that occurs after 'now' */
while (rc < 0) {
crm_time_add_seconds(origin, interval_s);
rc = crm_time_compare(origin, data_set->now);
}
delay = crm_time_calculate_duration(origin, data_set->now);
crm_time_log(LOG_TRACE, "origin", origin,
crm_time_log_date | crm_time_log_timeofday |
crm_time_log_with_timezone);
crm_time_log(LOG_TRACE, "now", data_set->now,
crm_time_log_date | crm_time_log_timeofday |
crm_time_log_with_timezone);
crm_time_log(LOG_TRACE, "delay", delay, crm_time_log_duration);
delay_s = crm_time_get_seconds(delay);
CRM_CHECK(delay_s >= 0, delay_s = 0);
start_delay = delay_s * 1000;
crm_info("Calculated a start delay of %llds for %s", delay_s, ID(xml_obj));
g_hash_table_replace(action->meta, strdup(XML_OP_ATTR_START_DELAY),
crm_itoa(start_delay));
crm_time_free(origin);
crm_time_free(delay);
}
}
field = XML_ATTR_TIMEOUT;
value = g_hash_table_lookup(action->meta, field);
if (value == NULL && xml_obj == NULL && safe_str_eq(action->task, RSC_STATUS) && interval == 0) {
xmlNode *min_interval_mon = find_min_interval_mon(action->rsc, FALSE);
if (min_interval_mon) {
value = crm_element_value(min_interval_mon, XML_ATTR_TIMEOUT);
pe_rsc_trace(action->rsc,
"\t%s uses the timeout value '%s' from the minimum interval monitor",
action->uuid, value);
}
}
if (value == NULL) {
value = pe_pref(data_set->config_hash, "default-action-timeout");
}
value_i = crm_get_msec(value);
if (value_i < 0) {
value_i = 0;
}
value_ms = crm_itoa(value_i);
g_hash_table_replace(action->meta, strdup(field), value_ms);
}
static xmlNode *
find_rsc_op_entry_helper(resource_t * rsc, const char *key, gboolean include_disabled)
{
unsigned long long number = 0;
gboolean do_retry = TRUE;
char *local_key = NULL;
const char *name = NULL;
const char *value = NULL;
const char *interval = NULL;
char *match_key = NULL;
xmlNode *op = NULL;
xmlNode *operation = NULL;
retry:
for (operation = __xml_first_child(rsc->ops_xml); operation != NULL;
operation = __xml_next_element(operation)) {
if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
name = crm_element_value(operation, "name");
interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
value = crm_element_value(operation, "enabled");
if (!include_disabled && value && crm_is_true(value) == FALSE) {
continue;
}
number = crm_get_interval(interval);
match_key = generate_op_key(rsc->id, name, number);
if (safe_str_eq(key, match_key)) {
op = operation;
}
free(match_key);
if (rsc->clone_name) {
match_key = generate_op_key(rsc->clone_name, name, number);
if (safe_str_eq(key, match_key)) {
op = operation;
}
free(match_key);
}
if (op != NULL) {
free(local_key);
return op;
}
}
}
free(local_key);
if (do_retry == FALSE) {
return NULL;
}
do_retry = FALSE;
if (strstr(key, CRMD_ACTION_MIGRATE) || strstr(key, CRMD_ACTION_MIGRATED)) {
local_key = generate_op_key(rsc->id, "migrate", 0);
key = local_key;
goto retry;
} else if (strstr(key, "_notify_")) {
local_key = generate_op_key(rsc->id, "notify", 0);
key = local_key;
goto retry;
}
return NULL;
}
xmlNode *
find_rsc_op_entry(resource_t * rsc, const char *key)
{
return find_rsc_op_entry_helper(rsc, key, FALSE);
}
void
print_node(const char *pre_text, node_t * node, gboolean details)
{
if (node == NULL) {
crm_trace("%s%s: <NULL>", pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": ");
return;
}
CRM_ASSERT(node->details);
crm_trace("%s%s%sNode %s: (weight=%d, fixed=%s)",
pre_text == NULL ? "" : pre_text,
pre_text == NULL ? "" : ": ",
node->details->online ? "" : "Unavailable/Unclean ",
node->details->uname, node->weight, node->fixed ? "True" : "False");
if (details) {
char *pe_mutable = strdup("\t\t");
GListPtr gIter = node->details->running_rsc;
crm_trace("\t\t===Node Attributes");
g_hash_table_foreach(node->details->attrs, print_str_str, pe_mutable);
free(pe_mutable);
crm_trace("\t\t=== Resources");
for (; gIter != NULL; gIter = gIter->next) {
resource_t *rsc = (resource_t *) gIter->data;
print_resource(LOG_DEBUG_4, "\t\t", rsc, FALSE);
}
}
}
/*
* Used by the HashTable for-loop
*/
void
print_str_str(gpointer key, gpointer value, gpointer user_data)
{
crm_trace("%s%s %s ==> %s",
user_data == NULL ? "" : (char *)user_data,
user_data == NULL ? "" : ": ", (char *)key, (char *)value);
}
void
print_resource(int log_level, const char *pre_text, resource_t * rsc, gboolean details)
{
long options = pe_print_log | pe_print_pending;
if (rsc == NULL) {
do_crm_log(log_level - 1, "%s%s: <NULL>",
pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": ");
return;
}
if (details) {
options |= pe_print_details;
}
rsc->fns->print(rsc, pre_text, options, &log_level);
}
void
pe_free_action(action_t * action)
{
if (action == NULL) {
return;
}
g_list_free_full(action->actions_before, free); /* action_warpper_t* */
g_list_free_full(action->actions_after, free); /* action_warpper_t* */
if (action->extra) {
g_hash_table_destroy(action->extra);
}
if (action->meta) {
g_hash_table_destroy(action->meta);
}
free(action->cancel_task);
free(action->task);
free(action->uuid);
free(action->node);
free(action);
}
GListPtr
find_recurring_actions(GListPtr input, node_t * not_on_node)
{
const char *value = NULL;
GListPtr result = NULL;
GListPtr gIter = input;
CRM_CHECK(input != NULL, return NULL);
for (; gIter != NULL; gIter = gIter->next) {
action_t *action = (action_t *) gIter->data;
value = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL);
if (value == NULL) {
/* skip */
} else if (safe_str_eq(value, "0")) {
/* skip */
} else if (safe_str_eq(CRMD_ACTION_CANCEL, action->task)) {
/* skip */
} else if (not_on_node == NULL) {
crm_trace("(null) Found: %s", action->uuid);
result = g_list_prepend(result, action);
} else if (action->node == NULL) {
/* skip */
} else if (action->node->details != not_on_node->details) {
crm_trace("Found: %s", action->uuid);
result = g_list_prepend(result, action);
}
}
return result;
}
enum action_tasks
get_complex_task(resource_t * rsc, const char *name, gboolean allow_non_atomic)
{
enum action_tasks task = text2task(name);
if (rsc == NULL) {
return task;
} else if (allow_non_atomic == FALSE || rsc->variant == pe_native) {
switch (task) {
case stopped_rsc:
case started_rsc:
case action_demoted:
case action_promoted:
crm_trace("Folding %s back into its atomic counterpart for %s", name, rsc->id);
return task - 1;
break;
default:
break;
}
}
return task;
}
action_t *
find_first_action(GListPtr input, const char *uuid, const char *task, node_t * on_node)
{
GListPtr gIter = NULL;
CRM_CHECK(uuid || task, return NULL);
for (gIter = input; gIter != NULL; gIter = gIter->next) {
action_t *action = (action_t *) gIter->data;
if (uuid != NULL && safe_str_neq(uuid, action->uuid)) {
continue;
} else if (task != NULL && safe_str_neq(task, action->task)) {
continue;
} else if (on_node == NULL) {
return action;
} else if (action->node == NULL) {
continue;
} else if (on_node->details == action->node->details) {
return action;
}
}
return NULL;
}
GListPtr
find_actions(GListPtr input, const char *key, const node_t *on_node)
{
GListPtr gIter = input;
GListPtr result = NULL;
CRM_CHECK(key != NULL, return NULL);
for (; gIter != NULL; gIter = gIter->next) {
action_t *action = (action_t *) gIter->data;
if (safe_str_neq(key, action->uuid)) {
crm_trace("%s does not match action %s", key, action->uuid);
continue;
} else if (on_node == NULL) {
crm_trace("Action %s matches (ignoring node)", key);
result = g_list_prepend(result, action);
} else if (action->node == NULL) {
crm_trace("Action %s matches (unallocated, assigning to %s)",
key, on_node->details->uname);
action->node = node_copy(on_node);
result = g_list_prepend(result, action);
} else if (on_node->details == action->node->details) {
crm_trace("Action %s on %s matches", key, on_node->details->uname);
result = g_list_prepend(result, action);
} else {
crm_trace("Action %s on node %s does not match requested node %s",
key, action->node->details->uname,
on_node->details->uname);
}
}
return result;
}
GListPtr
find_actions_exact(GListPtr input, const char *key, node_t * on_node)
{
GListPtr gIter = input;
GListPtr result = NULL;
CRM_CHECK(key != NULL, return NULL);
for (; gIter != NULL; gIter = gIter->next) {
action_t *action = (action_t *) gIter->data;
crm_trace("Matching %s against %s", key, action->uuid);
if (safe_str_neq(key, action->uuid)) {
crm_trace("Key mismatch: %s vs. %s", key, action->uuid);
continue;
} else if (on_node == NULL || action->node == NULL) {
crm_trace("on_node=%p, action->node=%p", on_node, action->node);
continue;
} else if (safe_str_eq(on_node->details->id, action->node->details->id)) {
result = g_list_prepend(result, action);
}
crm_trace("Node mismatch: %s vs. %s", on_node->details->id, action->node->details->id);
}
return result;
}
static void
resource_node_score(resource_t * rsc, node_t * node, int score, const char *tag)
{
node_t *match = NULL;
if (rsc->children) {
GListPtr gIter = rsc->children;
for (; gIter != NULL; gIter = gIter->next) {
resource_t *child_rsc = (resource_t *) gIter->data;
resource_node_score(child_rsc, node, score, tag);
}
}
pe_rsc_trace(rsc, "Setting %s for %s on %s: %d", tag, rsc->id, node->details->uname, score);
match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
if (match == NULL) {
match = node_copy(node);
match->weight = merge_weights(score, node->weight);
g_hash_table_insert(rsc->allowed_nodes, (gpointer) match->details->id, match);
}
match->weight = merge_weights(match->weight, score);
}
void
resource_location(resource_t * rsc, node_t * node, int score, const char *tag,
pe_working_set_t * data_set)
{
if (node != NULL) {
resource_node_score(rsc, node, score, tag);
} else if (data_set != NULL) {
GListPtr gIter = data_set->nodes;
for (; gIter != NULL; gIter = gIter->next) {
node_t *node_iter = (node_t *) gIter->data;
resource_node_score(rsc, node_iter, score, tag);
}
} else {
GHashTableIter iter;
node_t *node_iter = NULL;
g_hash_table_iter_init(&iter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node_iter)) {
resource_node_score(rsc, node_iter, score, tag);
}
}
if (node == NULL && score == -INFINITY) {
if (rsc->allocated_to) {
crm_info("Deallocating %s from %s", rsc->id, rsc->allocated_to->details->uname);
free(rsc->allocated_to);
rsc->allocated_to = NULL;
}
}
}
#define sort_return(an_int, why) do { \
free(a_uuid); \
free(b_uuid); \
crm_trace("%s (%d) %c %s (%d) : %s", \
a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=', \
b_xml_id, b_call_id, why); \
return an_int; \
} while(0)
gint
sort_op_by_callid(gconstpointer a, gconstpointer b)
{
int a_call_id = -1;
int b_call_id = -1;
char *a_uuid = NULL;
char *b_uuid = NULL;
const xmlNode *xml_a = a;
const xmlNode *xml_b = b;
const char *a_xml_id = crm_element_value_const(xml_a, XML_ATTR_ID);
const char *b_xml_id = crm_element_value_const(xml_b, XML_ATTR_ID);
if (safe_str_eq(a_xml_id, b_xml_id)) {
/* We have duplicate lrm_rsc_op entries in the status
* section which is unliklely to be a good thing
* - we can handle it easily enough, but we need to get
* to the bottom of why its happening.
*/
pe_err("Duplicate lrm_rsc_op entries named %s", a_xml_id);
sort_return(0, "duplicate");
}
crm_element_value_const_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id);
crm_element_value_const_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id);
if (a_call_id == -1 && b_call_id == -1) {
/* both are pending ops so it doesn't matter since
* stops are never pending
*/
sort_return(0, "pending");
} else if (a_call_id >= 0 && a_call_id < b_call_id) {
sort_return(-1, "call id");
} else if (b_call_id >= 0 && a_call_id > b_call_id) {
sort_return(1, "call id");
} else if (b_call_id >= 0 && a_call_id == b_call_id) {
/*
* The op and last_failed_op are the same
* Order on last-rc-change
*/
int last_a = -1;
int last_b = -1;
crm_element_value_const_int(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a);
crm_element_value_const_int(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b);
crm_trace("rc-change: %d vs %d", last_a, last_b);
if (last_a >= 0 && last_a < last_b) {
sort_return(-1, "rc-change");
} else if (last_b >= 0 && last_a > last_b) {
sort_return(1, "rc-change");
}
sort_return(0, "rc-change");
} else {
/* One of the inputs is a pending operation
* Attempt to use XML_ATTR_TRANSITION_MAGIC to determine its age relative to the other
*/
int a_id = -1;
int b_id = -1;
int dummy = -1;
const char *a_magic = crm_element_value_const(xml_a, XML_ATTR_TRANSITION_MAGIC);
const char *b_magic = crm_element_value_const(xml_b, XML_ATTR_TRANSITION_MAGIC);
CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
if(!decode_transition_magic(a_magic, &a_uuid, &a_id, &dummy, &dummy, &dummy, &dummy)) {
sort_return(0, "bad magic a");
}
if(!decode_transition_magic(b_magic, &b_uuid, &b_id, &dummy, &dummy, &dummy, &dummy)) {
sort_return(0, "bad magic b");
}
/* try to determine the relative age of the operation...
* some pending operations (ie. a start) may have been superseded
* by a subsequent stop
*
* [a|b]_id == -1 means its a shutdown operation and _always_ comes last
*/
if (safe_str_neq(a_uuid, b_uuid) || a_id == b_id) {
/*
* some of the logic in here may be redundant...
*
* if the UUID from the TE doesn't match then one better
* be a pending operation.
* pending operations don't survive between elections and joins
* because we query the LRM directly
*/
if (b_call_id == -1) {
sort_return(-1, "transition + call");
} else if (a_call_id == -1) {
sort_return(1, "transition + call");
}
} else if ((a_id >= 0 && a_id < b_id) || b_id == -1) {
sort_return(-1, "transition");
} else if ((b_id >= 0 && a_id > b_id) || a_id == -1) {
sort_return(1, "transition");
}
}
/* we should never end up here */
CRM_CHECK(FALSE, sort_return(0, "default"));
}
time_t
get_effective_time(pe_working_set_t * data_set)
{
if(data_set) {
if (data_set->now == NULL) {
crm_trace("Recording a new 'now'");
data_set->now = crm_time_new(NULL);
}
return crm_time_get_seconds_since_epoch(data_set->now);
}
crm_trace("Defaulting to 'now'");
return time(NULL);
}
gboolean
get_target_role(resource_t * rsc, enum rsc_role_e * role)
{
enum rsc_role_e local_role = RSC_ROLE_UNKNOWN;
const char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
CRM_CHECK(role != NULL, return FALSE);
if (value == NULL || safe_str_eq("started", value)
|| safe_str_eq("default", value)) {
return FALSE;
}
local_role = text2role(value);
if (local_role == RSC_ROLE_UNKNOWN) {
crm_config_err("%s: Unknown value for %s: %s", rsc->id, XML_RSC_ATTR_TARGET_ROLE, value);
return FALSE;
} else if (local_role > RSC_ROLE_STARTED) {
if (uber_parent(rsc)->variant == pe_master) {
if (local_role > RSC_ROLE_SLAVE) {
/* This is what we'd do anyway, just leave the default to avoid messing up the placement algorithm */
return FALSE;
}
} else {
crm_config_err("%s is not part of a master/slave resource, a %s of '%s' makes no sense",
rsc->id, XML_RSC_ATTR_TARGET_ROLE, value);
return FALSE;
}
}
*role = local_role;
return TRUE;
}
gboolean
order_actions(action_t * lh_action, action_t * rh_action, enum pe_ordering order)
{
GListPtr gIter = NULL;
action_wrapper_t *wrapper = NULL;
GListPtr list = NULL;
if (order == pe_order_none) {
return FALSE;
}
if (lh_action == NULL || rh_action == NULL) {
return FALSE;
}
crm_trace("Ordering Action %s before %s", lh_action->uuid, rh_action->uuid);
/* Ensure we never create a dependency on ourselves... its happened */
CRM_ASSERT(lh_action != rh_action);
/* Filter dups, otherwise update_action_states() has too much work to do */
gIter = lh_action->actions_after;
for (; gIter != NULL; gIter = gIter->next) {
action_wrapper_t *after = (action_wrapper_t *) gIter->data;
if (after->action == rh_action && (after->type & order)) {
return FALSE;
}
}
wrapper = calloc(1, sizeof(action_wrapper_t));
wrapper->action = rh_action;
wrapper->type = order;
list = lh_action->actions_after;
list = g_list_prepend(list, wrapper);
lh_action->actions_after = list;
wrapper = NULL;
/* order |= pe_order_implies_then; */
/* order ^= pe_order_implies_then; */
wrapper = calloc(1, sizeof(action_wrapper_t));
wrapper->action = lh_action;
wrapper->type = order;
list = rh_action->actions_before;
list = g_list_prepend(list, wrapper);
rh_action->actions_before = list;
return TRUE;
}
action_t *
get_pseudo_op(const char *name, pe_working_set_t * data_set)
{
action_t *op = NULL;
if(data_set->singletons) {
op = g_hash_table_lookup(data_set->singletons, name);
}
if (op == NULL) {
op = custom_action(NULL, strdup(name), name, NULL, TRUE, TRUE, data_set);
set_bit(op->flags, pe_action_pseudo);
set_bit(op->flags, pe_action_runnable);
}
return op;
}
void
destroy_ticket(gpointer data)
{
ticket_t *ticket = data;
if (ticket->state) {
g_hash_table_destroy(ticket->state);
}
free(ticket->id);
free(ticket);
}
ticket_t *
ticket_new(const char *ticket_id, pe_working_set_t * data_set)
{
ticket_t *ticket = NULL;
if (ticket_id == NULL || strlen(ticket_id) == 0) {
return NULL;
}
if (data_set->tickets == NULL) {
data_set->tickets =
g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, destroy_ticket);
}
ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
if (ticket == NULL) {
ticket = calloc(1, sizeof(ticket_t));
if (ticket == NULL) {
crm_err("Cannot allocate ticket '%s'", ticket_id);
return NULL;
}
crm_trace("Creaing ticket entry for %s", ticket_id);
ticket->id = strdup(ticket_id);
ticket->granted = FALSE;
ticket->last_granted = -1;
ticket->standby = FALSE;
ticket->state = g_hash_table_new_full(crm_str_hash, g_str_equal,
g_hash_destroy_str, g_hash_destroy_str);
g_hash_table_insert(data_set->tickets, strdup(ticket->id), ticket);
}
return ticket;
}
static void
filter_parameters(xmlNode * param_set, const char *param_string, bool need_present)
{
int len = 0;
char *name = NULL;
char *match = NULL;
if (param_set == NULL) {
return;
}
if (param_set) {
xmlAttrPtr xIter = param_set->properties;
while (xIter) {
const char *prop_name = (const char *)xIter->name;
xIter = xIter->next;
name = NULL;
len = strlen(prop_name) + 3;
name = malloc(len);
if(name) {
sprintf(name, " %s ", prop_name);
name[len - 1] = 0;
match = strstr(param_string, name);
}
if (need_present && match == NULL) {
crm_trace("%s not found in %s", prop_name, param_string);
xml_remove_prop(param_set, prop_name);
} else if (need_present == FALSE && match) {
crm_trace("%s found in %s", prop_name, param_string);
xml_remove_prop(param_set, prop_name);
}
free(name);
}
}
}
op_digest_cache_t *
rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node,
pe_working_set_t * data_set)
{
op_digest_cache_t *data = NULL;
GHashTable *local_rsc_params = NULL;
xmlNode *local_versioned_params = NULL;
action_t *action = NULL;
char *key = NULL;
int interval = 0;
const char *op_id = ID(xml_op);
const char *interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
const char *digest_all;
const char *digest_restart;
const char *secure_list;
const char *restart_list;
const char *op_version;
data = g_hash_table_lookup(node->details->digest_cache, op_id);
if (data) {
return data;
}
data = calloc(1, sizeof(op_digest_cache_t));
digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
secure_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_SECURE);
restart_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_RESTART);
op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
/* key is freed in custom_action */
interval = crm_parse_int(interval_s, "0");
key = generate_op_key(rsc->id, task, interval);
action = custom_action(rsc, key, task, node, TRUE, FALSE, data_set);
key = NULL;
local_rsc_params = g_hash_table_new_full(crm_str_hash, g_str_equal,
g_hash_destroy_str, g_hash_destroy_str);
get_rsc_attributes(local_rsc_params, rsc, node, data_set);
local_versioned_params = create_xml_node(NULL, XML_TAG_VER_ATTRS);
pe_get_versioned_attributes(local_versioned_params, rsc, node, data_set);
data->params_all = create_xml_node(NULL, XML_TAG_PARAMS);
g_hash_table_foreach(local_rsc_params, hash2field, data->params_all);
g_hash_table_foreach(action->extra, hash2field, data->params_all);
g_hash_table_foreach(rsc->parameters, hash2field, data->params_all);
g_hash_table_foreach(action->meta, hash2metafield, data->params_all);
filter_action_parameters(data->params_all, op_version);
crm_summarize_versioned_params(data->params_all, rsc->versioned_parameters);
crm_summarize_versioned_params(data->params_all, local_versioned_params);
data->digest_all_calc = calculate_operation_digest(data->params_all, op_version);
if (secure_list && is_set(data_set->flags, pe_flag_sanitized)) {
data->params_secure = copy_xml(data->params_all);
if (secure_list) {
filter_parameters(data->params_secure, secure_list, FALSE);
}
data->digest_secure_calc = calculate_operation_digest(data->params_secure, op_version);
}
if (digest_restart) {
data->params_restart = copy_xml(data->params_all);
if (restart_list) {
filter_parameters(data->params_restart, restart_list, TRUE);
}
data->digest_restart_calc = calculate_operation_digest(data->params_restart, op_version);
}
data->rc = RSC_DIGEST_MATCH;
if (digest_restart && strcmp(data->digest_restart_calc, digest_restart) != 0) {
data->rc = RSC_DIGEST_RESTART;
} else if (digest_all == NULL) {
/* it is unknown what the previous op digest was */
data->rc = RSC_DIGEST_UNKNOWN;
} else if (strcmp(digest_all, data->digest_all_calc) != 0) {
data->rc = RSC_DIGEST_ALL;
}
g_hash_table_insert(node->details->digest_cache, strdup(op_id), data);
g_hash_table_destroy(local_rsc_params);
free_xml(local_versioned_params);
pe_free_action(action);
return data;
}
const char *rsc_printable_id(resource_t *rsc)
{
if (is_not_set(rsc->flags, pe_rsc_unique)) {
return ID(rsc->xml);
}
return rsc->id;
}
void
clear_bit_recursive(resource_t * rsc, unsigned long long flag)
{
GListPtr gIter = rsc->children;
clear_bit(rsc->flags, flag);
for (; gIter != NULL; gIter = gIter->next) {
resource_t *child_rsc = (resource_t *) gIter->data;
clear_bit_recursive(child_rsc, flag);
}
}
void
set_bit_recursive(resource_t * rsc, unsigned long long flag)
{
GListPtr gIter = rsc->children;
set_bit(rsc->flags, flag);
for (; gIter != NULL; gIter = gIter->next) {
resource_t *child_rsc = (resource_t *) gIter->data;
set_bit_recursive(child_rsc, flag);
}
}
action_t *
pe_fence_op(node_t * node, const char *op, bool optional, pe_working_set_t * data_set)
{
char *key = NULL;
action_t *stonith_op = NULL;
if(op == NULL) {
op = data_set->stonith_action;
}
key = crm_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op);
if(data_set->singletons) {
stonith_op = g_hash_table_lookup(data_set->singletons, key);
}
if(stonith_op == NULL) {
stonith_op = custom_action(NULL, key, CRM_OP_FENCE, node, optional, TRUE, data_set);
add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname);
add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id);
add_hash_param(stonith_op->meta, "stonith_action", op);
} else {
free(key);
}
if(optional == FALSE) {
crm_trace("%s is no longer optional", stonith_op->uuid);
pe_clear_action_bit(stonith_op, pe_action_optional);
}
return stonith_op;
}
void
trigger_unfencing(
resource_t * rsc, node_t *node, const char *reason, action_t *dependency, pe_working_set_t * data_set)
{
if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) {
/* No resources require it */
return;
} else if (rsc != NULL && is_not_set(rsc->flags, pe_rsc_fence_device)) {
/* Wasn't a stonith device */
return;
} else if(node
&& node->details->online
&& node->details->unclean == FALSE
&& node->details->shutdown == FALSE) {
action_t *unfence = pe_fence_op(node, "on", FALSE, data_set);
crm_notice("Unfencing %s: %s", node->details->uname, reason);
if(dependency) {
order_actions(unfence, dependency, pe_order_optional);
}
} else if(rsc) {
GHashTableIter iter;
g_hash_table_iter_init(&iter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) {
trigger_unfencing(rsc, node, reason, dependency, data_set);
}
}
}
}
gboolean
add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref)
{
tag_t *tag = NULL;
GListPtr gIter = NULL;
gboolean is_existing = FALSE;
CRM_CHECK(tags && tag_name && obj_ref, return FALSE);
tag = g_hash_table_lookup(tags, tag_name);
if (tag == NULL) {
tag = calloc(1, sizeof(tag_t));
if (tag == NULL) {
return FALSE;
}
tag->id = strdup(tag_name);
tag->refs = NULL;
g_hash_table_insert(tags, strdup(tag_name), tag);
}
for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
const char *existing_ref = (const char *) gIter->data;
if (crm_str_eq(existing_ref, obj_ref, TRUE)){
is_existing = TRUE;
break;
}
}
if (is_existing == FALSE) {
tag->refs = g_list_append(tag->refs, strdup(obj_ref));
crm_trace("Added: tag=%s ref=%s", tag->id, obj_ref);
}
return TRUE;
}
diff --git a/pengine/graph.c b/pengine/graph.c
index f7f6511f43..7691a591bf 100644
--- a/pengine/graph.c
+++ b/pengine/graph.c
@@ -1,1555 +1,1556 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <crm/crm.h>
#include <crm/cib.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <glib.h>
#include <allocate.h>
#include <utils.h>
void update_colo_start_chain(action_t * action);
gboolean rsc_update_action(action_t * first, action_t * then, enum pe_ordering type);
static enum pe_action_flags
get_action_flags(action_t * action, node_t * node)
{
enum pe_action_flags flags = action->flags;
if (action->rsc) {
flags = action->rsc->cmds->action_flags(action, NULL);
if (action->rsc->variant >= pe_clone && node) {
/* We only care about activity on $node */
enum pe_action_flags clone_flags = action->rsc->cmds->action_flags(action, node);
/* Go to great lengths to ensure the correct value for pe_action_runnable...
*
* If we are a clone, then for _ordering_ constraints, it's only relevant
* if we are runnable _anywhere_.
*
* This only applies to _runnable_ though, and only for ordering constraints.
* If this function is ever used during colocation, then we'll need additional logic
*
* Not very satisfying, but it's logical and appears to work well.
*/
if (is_not_set(clone_flags, pe_action_runnable)
&& is_set(flags, pe_action_runnable)) {
pe_rsc_trace(action->rsc, "Fixing up runnable flag for %s", action->uuid);
set_bit(clone_flags, pe_action_runnable);
}
flags = clone_flags;
}
}
return flags;
}
static char *
convert_non_atomic_uuid(char *old_uuid, resource_t * rsc, gboolean allow_notify,
gboolean free_original)
{
int interval = 0;
char *uuid = NULL;
char *rid = NULL;
char *raw_task = NULL;
int task = no_action;
CRM_ASSERT(rsc);
pe_rsc_trace(rsc, "Processing %s", old_uuid);
if (old_uuid == NULL) {
return NULL;
} else if (strstr(old_uuid, "notify") != NULL) {
goto done; /* no conversion */
} else if (rsc->variant < pe_group) {
goto done; /* no conversion */
}
CRM_ASSERT(parse_op_key(old_uuid, &rid, &raw_task, &interval));
if (interval > 0) {
goto done; /* no conversion */
}
task = text2task(raw_task);
switch (task) {
case stop_rsc:
case start_rsc:
case action_notify:
case action_promote:
case action_demote:
break;
case stopped_rsc:
case started_rsc:
case action_notified:
case action_promoted:
case action_demoted:
task--;
break;
case monitor_rsc:
case shutdown_crm:
case stonith_node:
task = no_action;
break;
default:
crm_err("Unknown action: %s", raw_task);
task = no_action;
break;
}
if (task != no_action) {
if (is_set(rsc->flags, pe_rsc_notify) && allow_notify) {
uuid = generate_notify_key(rid, "confirmed-post", task2text(task + 1));
} else {
uuid = generate_op_key(rid, task2text(task + 1), 0);
}
pe_rsc_trace(rsc, "Converted %s -> %s", old_uuid, uuid);
}
done:
if (uuid == NULL) {
uuid = strdup(old_uuid);
}
if (free_original) {
free(old_uuid);
}
free(raw_task);
free(rid);
return uuid;
}
static action_t *
rsc_expand_action(action_t * action)
{
action_t *result = action;
if (action->rsc && action->rsc->variant >= pe_group) {
/* Expand 'start' -> 'started' */
char *uuid = NULL;
gboolean notify = FALSE;
if (action->rsc->parent == NULL) {
/* Only outermost resources have notification actions */
notify = is_set(action->rsc->flags, pe_rsc_notify);
}
uuid = convert_non_atomic_uuid(action->uuid, action->rsc, notify, FALSE);
if (uuid) {
pe_rsc_trace(action->rsc, "Converting %s to %s %d", action->uuid, uuid,
is_set(action->rsc->flags, pe_rsc_notify));
result = find_first_action(action->rsc->actions, uuid, NULL, NULL);
if (result == NULL) {
crm_err("Couldn't expand %s", action->uuid);
result = action;
}
free(uuid);
}
}
return result;
}
static enum pe_graph_flags
graph_update_action(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
enum pe_ordering type)
{
enum pe_graph_flags changed = pe_graph_none;
gboolean processed = FALSE;
/* TODO: Do as many of these in parallel as possible */
if (type & pe_order_implies_then) {
processed = TRUE;
if (then->rsc) {
changed |=
then->rsc->cmds->update_actions(first, then, node, flags & pe_action_optional,
pe_action_optional, pe_order_implies_then);
} else if (is_set(flags, pe_action_optional) == FALSE) {
if (update_action_flags(then, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__)) {
changed |= pe_graph_updated_then;
}
}
if (changed) {
pe_rsc_trace(then->rsc, "implies right: %s then %s: changed", first->uuid, then->uuid);
} else {
crm_trace("implies right: %s then %s %p", first->uuid, then->uuid, then->rsc);
}
}
if ((type & pe_order_restart) && then->rsc) {
enum pe_action_flags restart = (pe_action_optional | pe_action_runnable);
processed = TRUE;
changed |=
then->rsc->cmds->update_actions(first, then, node, flags, restart, pe_order_restart);
if (changed) {
pe_rsc_trace(then->rsc, "restart: %s then %s: changed", first->uuid, then->uuid);
} else {
crm_trace("restart: %s then %s", first->uuid, then->uuid);
}
}
if (type & pe_order_implies_first) {
processed = TRUE;
if (first->rsc) {
changed |=
first->rsc->cmds->update_actions(first, then, node, flags,
pe_action_optional, pe_order_implies_first);
} else if (is_set(flags, pe_action_optional) == FALSE) {
pe_rsc_trace(first->rsc, "first unrunnable: %s then %s", first->uuid, then->uuid);
if (update_action_flags(first, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__)) {
changed |= pe_graph_updated_first;
}
}
if (changed) {
pe_rsc_trace(then->rsc, "implies left: %s then %s: changed", first->uuid, then->uuid);
} else {
crm_trace("implies left: %s then %s", first->uuid, then->uuid);
}
}
if (type & pe_order_implies_first_master) {
processed = TRUE;
if (then->rsc) {
changed |=
then->rsc->cmds->update_actions(first, then, node, flags & pe_action_optional,
pe_action_optional, pe_order_implies_first_master);
}
if (changed) {
pe_rsc_trace(then->rsc,
"implies left when right rsc is Master role: %s then %s: changed",
first->uuid, then->uuid);
} else {
crm_trace("implies left when right rsc is Master role: %s then %s", first->uuid,
then->uuid);
}
}
if (type & pe_order_one_or_more) {
processed = TRUE;
if (then->rsc) {
changed |=
then->rsc->cmds->update_actions(first, then, node, flags,
pe_action_runnable, pe_order_one_or_more);
} else if (is_set(flags, pe_action_runnable)) {
/* alright. a "first" action is considered runnable, incremente
* the 'runnable_before' counter */
then->runnable_before++;
/* if the runnable before count for then exceeds the required number
* of "before" runnable actions... mark then as runnable */
if (then->runnable_before >= then->required_runnable_before) {
if (update_action_flags(then, pe_action_runnable, __FUNCTION__, __LINE__)) {
changed |= pe_graph_updated_then;
}
}
}
if (changed) {
pe_rsc_trace(then->rsc, "runnable_one_or_more: %s then %s: changed", first->uuid,
then->uuid);
} else {
crm_trace("runnable_one_or_more: %s then %s", first->uuid, then->uuid);
}
}
if (type & pe_order_runnable_left) {
processed = TRUE;
if (then->rsc) {
changed |=
then->rsc->cmds->update_actions(first, then, node, flags,
pe_action_runnable, pe_order_runnable_left);
} else if (is_set(flags, pe_action_runnable) == FALSE) {
pe_rsc_trace(then->rsc, "then unrunnable: %s then %s", first->uuid, then->uuid);
if (update_action_flags(then, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__)) {
changed |= pe_graph_updated_then;
}
}
if (changed) {
pe_rsc_trace(then->rsc, "runnable: %s then %s: changed", first->uuid, then->uuid);
} else {
crm_trace("runnable: %s then %s", first->uuid, then->uuid);
}
}
if (type & pe_order_implies_first_migratable) {
processed = TRUE;
if (then->rsc) {
changed |=
then->rsc->cmds->update_actions(first, then, node, flags,
pe_action_optional, pe_order_implies_first_migratable);
}
if (changed) {
pe_rsc_trace(then->rsc, "optional: %s then %s: changed", first->uuid, then->uuid);
} else {
crm_trace("optional: %s then %s", first->uuid, then->uuid);
}
}
if (type & pe_order_pseudo_left) {
processed = TRUE;
if (then->rsc) {
changed |=
then->rsc->cmds->update_actions(first, then, node, flags,
pe_action_optional, pe_order_pseudo_left);
}
if (changed) {
pe_rsc_trace(then->rsc, "optional: %s then %s: changed", first->uuid, then->uuid);
} else {
crm_trace("optional: %s then %s", first->uuid, then->uuid);
}
}
if (type & pe_order_optional) {
processed = TRUE;
if (then->rsc) {
changed |=
then->rsc->cmds->update_actions(first, then, node, flags,
pe_action_runnable, pe_order_optional);
}
if (changed) {
pe_rsc_trace(then->rsc, "optional: %s then %s: changed", first->uuid, then->uuid);
} else {
crm_trace("optional: %s then %s", first->uuid, then->uuid);
}
}
if (type & pe_order_asymmetrical) {
processed = TRUE;
if (then->rsc) {
changed |=
then->rsc->cmds->update_actions(first, then, node, flags,
pe_action_runnable, pe_order_asymmetrical);
}
if (changed) {
pe_rsc_trace(then->rsc, "asymmetrical: %s then %s: changed", first->uuid, then->uuid);
} else {
crm_trace("asymmetrical: %s then %s", first->uuid, then->uuid);
}
}
if ((first->flags & pe_action_runnable) && (type & pe_order_implies_then_printed)
&& (flags & pe_action_optional) == 0) {
processed = TRUE;
crm_trace("%s implies %s printed", first->uuid, then->uuid);
update_action_flags(then, pe_action_print_always, __FUNCTION__, __LINE__); /* don't care about changed */
}
if ((type & pe_order_implies_first_printed) && (flags & pe_action_optional) == 0) {
processed = TRUE;
crm_trace("%s implies %s printed", then->uuid, first->uuid);
update_action_flags(first, pe_action_print_always, __FUNCTION__, __LINE__); /* don't care about changed */
}
if ((type & pe_order_implies_then
|| type & pe_order_implies_first
|| type & pe_order_restart)
&& first->rsc
&& safe_str_eq(first->task, RSC_STOP)
&& is_not_set(first->rsc->flags, pe_rsc_managed)
&& is_set(first->rsc->flags, pe_rsc_block)
&& is_not_set(first->flags, pe_action_runnable)) {
if (update_action_flags(then, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__)) {
changed |= pe_graph_updated_then;
}
if (changed) {
pe_rsc_trace(then->rsc, "unmanaged left: %s then %s: changed", first->uuid, then->uuid);
} else {
crm_trace("unmanaged left: %s then %s", first->uuid, then->uuid);
}
}
if (processed == FALSE) {
crm_trace("Constraint 0x%.6x not applicable", type);
}
return changed;
}
static void
mark_start_blocked(resource_t *rsc)
{
GListPtr gIter = rsc->actions;
for (; gIter != NULL; gIter = gIter->next) {
action_t *action = (action_t *) gIter->data;
if (safe_str_neq(action->task, RSC_START)) {
continue;
}
if (is_set(action->flags, pe_action_runnable)) {
clear_bit(action->flags, pe_action_runnable);
update_colo_start_chain(action);
update_action(action);
}
}
}
void
update_colo_start_chain(action_t *action)
{
GListPtr gIter = NULL;
resource_t *rsc = NULL;
if (is_not_set(action->flags, pe_action_runnable) && safe_str_eq(action->task, RSC_START)) {
rsc = uber_parent(action->rsc);
}
if (rsc == NULL || rsc->rsc_cons_lhs == NULL) {
return;
}
/* if rsc has children, all the children need to have start set to
* unrunnable before we follow the colo chain for the parent. */
for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
resource_t *child = (resource_t *)gIter->data;
action_t *start = find_first_action(child->actions, NULL, RSC_START, NULL);
if (start == NULL || is_set(start->flags, pe_action_runnable)) {
return;
}
}
for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
rsc_colocation_t *colocate_with = (rsc_colocation_t *)gIter->data;
if (colocate_with->score == INFINITY) {
mark_start_blocked(colocate_with->rsc_lh);
}
}
}
gboolean
update_action(action_t * then)
{
GListPtr lpc = NULL;
enum pe_graph_flags changed = pe_graph_none;
int last_flags = then->flags;
crm_trace("Processing %s (%s %s %s)",
then->uuid,
is_set(then->flags, pe_action_optional) ? "optional" : "required",
is_set(then->flags, pe_action_runnable) ? "runnable" : "unrunnable",
is_set(then->flags,
pe_action_pseudo) ? "pseudo" : then->node ? then->node->details->uname : "");
if (is_set(then->flags, pe_action_requires_any)) {
/* initialize current known runnable before actions to 0
* from here as graph_update_action is called for each of
* then's before actions, this number will increment as
* runnable 'first' actions are encountered */
then->runnable_before = 0;
/* for backwards compatibility with previous options that use
* the 'requires_any' flag, initialize required to 1 if it is
* not set. */
if (then->required_runnable_before == 0) {
then->required_runnable_before = 1;
}
clear_bit(then->flags, pe_action_runnable);
/* We are relying on the pe_order_one_or_more clause of
* graph_update_action(), called as part of the:
*
* 'if (first == other->action)'
*
* block below, to set this back if appropriate
*/
}
for (lpc = then->actions_before; lpc != NULL; lpc = lpc->next) {
action_wrapper_t *other = (action_wrapper_t *) lpc->data;
action_t *first = other->action;
node_t *then_node = then->node;
node_t *first_node = first->node;
enum pe_action_flags then_flags = 0;
enum pe_action_flags first_flags = 0;
if (first->rsc && first->rsc->variant == pe_group && safe_str_eq(first->task, RSC_START)) {
first_node = first->rsc->fns->location(first->rsc, NULL, FALSE);
if (first_node) {
crm_trace("First: Found node %s for %s", first_node->details->uname, first->uuid);
}
}
if (then->rsc && then->rsc->variant == pe_group && safe_str_eq(then->task, RSC_START)) {
then_node = then->rsc->fns->location(then->rsc, NULL, FALSE);
if (then_node) {
crm_trace("Then: Found node %s for %s", then_node->details->uname, then->uuid);
}
}
/* Disable constraint if it only applies when on same node, but isn't */
if (is_set(other->type, pe_order_same_node) && first_node && then_node
&& (first_node->details != then_node->details)) {
crm_trace("Disabled constraint %s on %s -> %s on %s",
other->action->uuid, first_node->details->uname,
then->uuid, then_node->details->uname);
other->type = pe_order_none;
continue;
}
clear_bit(changed, pe_graph_updated_first);
if (first->rsc != then->rsc
&& first->rsc != NULL && then->rsc != NULL && first->rsc != then->rsc->parent) {
first = rsc_expand_action(first);
}
if (first != other->action) {
crm_trace("Ordering %s after %s instead of %s", then->uuid, first->uuid,
other->action->uuid);
}
first_flags = get_action_flags(first, then_node);
then_flags = get_action_flags(then, first_node);
crm_trace("Checking %s (%s %s %s) against %s (%s %s %s) filter=0x%.6x type=0x%.6x",
then->uuid,
is_set(then_flags, pe_action_optional) ? "optional" : "required",
is_set(then_flags, pe_action_runnable) ? "runnable" : "unrunnable",
is_set(then_flags,
pe_action_pseudo) ? "pseudo" : then->node ? then->node->details->
uname : "", first->uuid, is_set(first_flags,
pe_action_optional) ? "optional" : "required",
is_set(first_flags, pe_action_runnable) ? "runnable" : "unrunnable",
is_set(first_flags,
pe_action_pseudo) ? "pseudo" : first->node ? first->node->details->
uname : "", first_flags, other->type);
if (first == other->action) {
/*
* 'first' was not expanded (ie. from 'start' to 'running'), which could mean it:
* - has no associated resource,
* - was a primitive,
* - was pre-expanded (ie. 'running' instead of 'start')
*
* The third argument here to graph_update_action() is a node which is used under two conditions:
* - Interleaving, in which case first->node and
* then->node are equal (and NULL)
* - If 'then' is a clone, to limit the scope of the
* constraint to instances on the supplied node
*
*/
int otype = other->type;
node_t *node = then->node;
if(is_set(otype, pe_order_implies_then_on_node)) {
/* Normally we want the _whole_ 'then' clone to
* restart if 'first' is restarted, so then->node is
* needed.
*
* However for unfencing, we want to limit this to
* instances on the same node as 'first' (the
* unfencing operation), so first->node is supplied.
*
* Swap the node, from then on we can can treat it
* like any other 'pe_order_implies_then'
*/
clear_bit(otype, pe_order_implies_then_on_node);
set_bit(otype, pe_order_implies_then);
node = first->node;
}
clear_bit(first_flags, pe_action_pseudo);
changed |= graph_update_action(first, then, node, first_flags, otype);
/* 'first' was for a complex resource (clone, group, etc),
* create a new dependency if necessary
*/
} else if (order_actions(first, then, other->type)) {
/* This was the first time 'first' and 'then' were associated,
* start again to get the new actions_before list
*/
changed |= (pe_graph_updated_then | pe_graph_disable);
}
if (changed & pe_graph_disable) {
crm_trace("Disabled constraint %s -> %s", other->action->uuid, then->uuid);
clear_bit(changed, pe_graph_disable);
other->type = pe_order_none;
}
if (changed & pe_graph_updated_first) {
GListPtr lpc2 = NULL;
crm_trace("Updated %s (first %s %s %s), processing dependents ",
first->uuid,
is_set(first->flags, pe_action_optional) ? "optional" : "required",
is_set(first->flags, pe_action_runnable) ? "runnable" : "unrunnable",
is_set(first->flags,
pe_action_pseudo) ? "pseudo" : first->node ? first->node->details->
uname : "");
for (lpc2 = first->actions_after; lpc2 != NULL; lpc2 = lpc2->next) {
action_wrapper_t *other = (action_wrapper_t *) lpc2->data;
update_action(other->action);
}
update_action(first);
}
}
if (is_set(then->flags, pe_action_requires_any)) {
if (last_flags != then->flags) {
changed |= pe_graph_updated_then;
} else {
clear_bit(changed, pe_graph_updated_then);
}
}
if (changed & pe_graph_updated_then) {
crm_trace("Updated %s (then %s %s %s), processing dependents ",
then->uuid,
is_set(then->flags, pe_action_optional) ? "optional" : "required",
is_set(then->flags, pe_action_runnable) ? "runnable" : "unrunnable",
is_set(then->flags,
pe_action_pseudo) ? "pseudo" : then->node ? then->node->details->
uname : "");
if (is_set(last_flags, pe_action_runnable) && is_not_set(then->flags, pe_action_runnable)) {
update_colo_start_chain(then);
}
update_action(then);
for (lpc = then->actions_after; lpc != NULL; lpc = lpc->next) {
action_wrapper_t *other = (action_wrapper_t *) lpc->data;
update_action(other->action);
}
}
return FALSE;
}
gboolean
shutdown_constraints(node_t * node, action_t * shutdown_op, pe_working_set_t * data_set)
{
/* add the stop to the before lists so it counts as a pre-req
* for the shutdown
*/
GListPtr lpc = NULL;
for (lpc = data_set->actions; lpc != NULL; lpc = lpc->next) {
action_t *action = (action_t *) lpc->data;
if (action->rsc == NULL || action->node == NULL) {
continue;
} else if (action->node->details != node->details) {
continue;
} else if (is_set(action->rsc->flags, pe_rsc_maintenance)) {
pe_rsc_trace(action->rsc, "Skipping %s: maintenance mode", action->uuid);
continue;
} else if (node->details->maintenance) {
pe_rsc_trace(action->rsc, "Skipping %s: node %s is in maintenance mode",
action->uuid, node->details->uname);
continue;
} else if (safe_str_neq(action->task, RSC_STOP)) {
continue;
} else if (is_not_set(action->rsc->flags, pe_rsc_managed)
&& is_not_set(action->rsc->flags, pe_rsc_block)) {
/*
* If another action depends on this one, we may still end up blocking
*/
pe_rsc_trace(action->rsc, "Skipping %s: unmanaged", action->uuid);
continue;
}
pe_rsc_trace(action->rsc, "Ordering %s before shutdown on %s", action->uuid,
node->details->uname);
pe_clear_action_bit(action, pe_action_optional);
custom_action_order(action->rsc, NULL, action,
NULL, strdup(CRM_OP_SHUTDOWN), shutdown_op,
pe_order_optional | pe_order_runnable_left, data_set);
}
return TRUE;
}
/*!
* \internal
* \brief Order all actions appropriately relative to a fencing operation
*
* Ensure start operations of affected resources are ordered after fencing,
* imply stop and demote operations of affected resources by marking them as
* pseudo-actions, etc.
*
* \param[in] node Node to be fenced
* \param[in] stonith_op Fencing operation
* \param[in/out] data_set Working set of cluster
*/
gboolean
stonith_constraints(node_t * node, action_t * stonith_op, pe_working_set_t * data_set)
{
GListPtr r = NULL;
CRM_CHECK(stonith_op != NULL, return FALSE);
for (r = data_set->resources; r != NULL; r = r->next) {
rsc_stonith_ordering((resource_t *) r->data, stonith_op, data_set);
}
return TRUE;
}
static node_t *
get_router_node(action_t *action)
{
node_t *began_on = NULL;
node_t *ended_on = NULL;
node_t *router_node = NULL;
if (safe_str_eq(action->task, CRM_OP_FENCE) || is_remote_node(action->node) == FALSE) {
return NULL;
}
CRM_ASSERT(action->node->details->remote_rsc != NULL);
if (action->node->details->remote_rsc->running_on) {
began_on = action->node->details->remote_rsc->running_on->data;
}
ended_on = action->node->details->remote_rsc->allocated_to;
/* if there is only one location to choose from,
* this is easy. Check for those conditions first */
if (!began_on || !ended_on) {
/* remote rsc is either shutting down or starting up */
return began_on ? began_on : ended_on;
} else if (began_on->details == ended_on->details) {
/* remote rsc didn't move nodes. */
return began_on;
}
/* If we have get here, we know the remote resource
* began on one node and is moving to another node.
*
* This means some actions will get routed through the cluster
* node the connection rsc began on, and others are routed through
* the cluster node the connection rsc ends up on.
*
* 1. stop, demote, migrate actions of resources living in the remote
* node _MUST_ occur _BEFORE_ the connection can move (these actions
* are all required before the remote rsc stop action can occur.) In
* this case, we know these actions have to be routed through the initial
* cluster node the connection resource lived on before the move takes place.
*
* 2. Everything else (start, promote, monitor, probe, refresh, clear failcount
* delete ....) must occur after the resource starts on the node it is
* moving to.
*/
/* 1. before connection rsc moves. */
if (safe_str_eq(action->task, "stop") ||
safe_str_eq(action->task, "demote") ||
safe_str_eq(action->task, "migrate_from") ||
safe_str_eq(action->task, "migrate_to")) {
router_node = began_on;
/* 2. after connection rsc moves. */
} else {
router_node = ended_on;
}
return router_node;
}
/*!
* \internal
* \brief Add an XML node tag for a specified ID
*
* \param[in] id Node UUID to add
* \param[in,out] xml Parent XML tag to add to
*/
static void
add_node_to_xml_by_id(const char *id, xmlNode *xml)
{
xmlNode *node_xml;
node_xml = create_xml_node(xml, XML_CIB_TAG_NODE);
crm_xml_add(node_xml, XML_ATTR_UUID, id);
}
/*!
* \internal
* \brief Add an XML node tag for a specified node
*
* \param[in] node Node to add
* \param[in/out] xml XML to add node to
*/
static void
add_node_to_xml(const node_t *node, void *xml)
{
add_node_to_xml_by_id(node->details->id, (xmlNode *) xml);
}
/*!
* \internal
* \brief Add XML with nodes that an action is expected to bring down
*
* If a specified action is expected to bring any nodes down, add an XML block
* with their UUIDs. When a node is lost, this allows the crmd to determine
* whether it was expected.
*
* \param[in,out] xml Parent XML tag to add to
* \param[in] action Action to check for downed nodes
* \param[in] data_set Working set for cluster
*/
static void
add_downed_nodes(xmlNode *xml, const action_t *action,
const pe_working_set_t *data_set)
{
CRM_CHECK(xml && action && action->node && data_set, return);
if (safe_str_eq(action->task, CRM_OP_SHUTDOWN)) {
/* Shutdown makes the action's node down */
xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
add_node_to_xml_by_id(action->node->details->id, downed);
} else if (safe_str_eq(action->task, CRM_OP_FENCE)) {
/* Fencing makes the action's node and any hosted guest nodes down */
const char *fence = g_hash_table_lookup(action->meta, "stonith_action");
if (safe_str_eq(fence, "off") || safe_str_eq(fence, "reboot")) {
xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
add_node_to_xml_by_id(action->node->details->id, downed);
pe_foreach_guest_node(data_set, action->node, add_node_to_xml, downed);
}
} else if (action->rsc && action->rsc->is_remote_node
&& safe_str_eq(action->task, CRMD_ACTION_STOP)) {
/* Stopping a remote connection resource makes connected node down,
* unless it's part of a migration
*/
GListPtr iter;
action_t *input;
gboolean migrating = FALSE;
for (iter = action->actions_before; iter != NULL; iter = iter->next) {
input = ((action_wrapper_t *) iter->data)->action;
if (input->rsc && safe_str_eq(action->rsc->id, input->rsc->id)
&& safe_str_eq(input->task, CRMD_ACTION_MIGRATED)) {
migrating = TRUE;
break;
}
}
if (!migrating) {
xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
add_node_to_xml_by_id(action->rsc->id, downed);
}
}
}
static xmlNode *
action2xml(action_t * action, gboolean as_input, pe_working_set_t *data_set)
{
gboolean needs_node_info = TRUE;
xmlNode *action_xml = NULL;
xmlNode *args_xml = NULL;
if (action == NULL) {
return NULL;
}
if (safe_str_eq(action->task, CRM_OP_FENCE)) {
/* All fences need node info; guest node fences are pseudo-events */
action_xml = create_xml_node(NULL,
is_set(action->flags, pe_action_pseudo)?
XML_GRAPH_TAG_PSEUDO_EVENT :
XML_GRAPH_TAG_CRM_EVENT);
} else if (safe_str_eq(action->task, CRM_OP_SHUTDOWN)) {
action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);
} else if (safe_str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT)) {
action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);
} else if (safe_str_eq(action->task, CRM_OP_LRM_REFRESH)) {
action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);
/* } else if(safe_str_eq(action->task, RSC_PROBED)) { */
/* action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT); */
} else if (is_set(action->flags, pe_action_pseudo)) {
action_xml = create_xml_node(NULL, XML_GRAPH_TAG_PSEUDO_EVENT);
needs_node_info = FALSE;
} else {
action_xml = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
}
crm_xml_add_int(action_xml, XML_ATTR_ID, action->id);
crm_xml_add(action_xml, XML_LRM_ATTR_TASK, action->task);
if (action->rsc != NULL && action->rsc->clone_name != NULL) {
char *clone_key = NULL;
const char *interval_s = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL);
int interval = crm_parse_int(interval_s, "0");
if (safe_str_eq(action->task, RSC_NOTIFY)) {
const char *n_type = g_hash_table_lookup(action->meta, "notify_type");
const char *n_task = g_hash_table_lookup(action->meta, "notify_operation");
CRM_CHECK(n_type != NULL, crm_err("No notify type value found for %s", action->uuid));
CRM_CHECK(n_task != NULL,
crm_err("No notify operation value found for %s", action->uuid));
clone_key = generate_notify_key(action->rsc->clone_name, n_type, n_task);
} else if(action->cancel_task) {
clone_key = generate_op_key(action->rsc->clone_name, action->cancel_task, interval);
} else {
clone_key = generate_op_key(action->rsc->clone_name, action->task, interval);
}
CRM_CHECK(clone_key != NULL, crm_err("Could not generate a key for %s", action->uuid));
crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, clone_key);
crm_xml_add(action_xml, "internal_" XML_LRM_ATTR_TASK_KEY, action->uuid);
free(clone_key);
} else {
crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, action->uuid);
}
if (needs_node_info && action->node != NULL) {
node_t *router_node = get_router_node(action);
crm_xml_add(action_xml, XML_LRM_ATTR_TARGET, action->node->details->uname);
crm_xml_add(action_xml, XML_LRM_ATTR_TARGET_UUID, action->node->details->id);
if (router_node) {
crm_xml_add(action_xml, XML_LRM_ATTR_ROUTER_NODE, router_node->details->uname);
}
}
/* No details if this action is only being listed in the inputs section */
if (as_input) {
return action_xml;
}
/* List affected resource */
if (action->rsc) {
if (is_set(action->flags, pe_action_pseudo) == FALSE) {
int lpc = 0;
xmlNode *rsc_xml = create_xml_node(action_xml, crm_element_name(action->rsc->xml));
const char *attr_list[] = {
XML_AGENT_ATTR_CLASS,
XML_AGENT_ATTR_PROVIDER,
XML_ATTR_TYPE
};
if (is_set(action->rsc->flags, pe_rsc_orphan) && action->rsc->clone_name) {
/* Do not use the 'instance free' name here as that
* might interfere with the instance we plan to keep.
* Ie. if there are more than two named /anonymous/
* instances on a given node, we need to make sure the
* command goes to the right one.
*
* Keep this block, even when everyone is using
* 'instance free' anonymous clone names - it means
* we'll do the right thing if anyone toggles the
* unique flag to 'off'
*/
crm_debug("Using orphan clone name %s instead of %s", action->rsc->id,
action->rsc->clone_name);
crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->clone_name);
crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);
} else if (is_not_set(action->rsc->flags, pe_rsc_unique)) {
const char *xml_id = ID(action->rsc->xml);
crm_debug("Using anonymous clone name %s for %s (aka. %s)", xml_id, action->rsc->id,
action->rsc->clone_name);
/* ID is what we'd like client to use
* ID_LONG is what they might know it as instead
*
* ID_LONG is only strictly needed /here/ during the
* transition period until all nodes in the cluster
* are running the new software /and/ have rebooted
* once (meaning that they've only ever spoken to a DC
* supporting this feature).
*
* If anyone toggles the unique flag to 'on', the
* 'instance free' name will correspond to an orphan
* and fall into the claus above instead
*/
crm_xml_add(rsc_xml, XML_ATTR_ID, xml_id);
if (action->rsc->clone_name && safe_str_neq(xml_id, action->rsc->clone_name)) {
crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->clone_name);
} else {
crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);
}
} else {
CRM_ASSERT(action->rsc->clone_name == NULL);
crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->id);
}
for (lpc = 0; lpc < DIMOF(attr_list); lpc++) {
crm_xml_add(rsc_xml, attr_list[lpc],
g_hash_table_lookup(action->rsc->meta, attr_list[lpc]));
}
}
}
/* List any attributes in effect */
args_xml = create_xml_node(NULL, XML_TAG_ATTRS);
crm_xml_add(args_xml, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
g_hash_table_foreach(action->extra, hash2field, args_xml);
if (action->rsc != NULL && action->node) {
GHashTable *p = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
xmlNode *versioned_parameters = create_xml_node(NULL, XML_TAG_VER_ATTRS);
get_rsc_attributes(p, action->rsc, action->node, data_set);
g_hash_table_foreach(p, hash2smartfield, args_xml);
pe_get_versioned_attributes(versioned_parameters, action->rsc, action->node, data_set);
if (xml_has_children(versioned_parameters)) {
add_node_copy(action_xml, versioned_parameters);
}
g_hash_table_destroy(p);
free_xml(versioned_parameters);
} else if(action->rsc && action->rsc->variant <= pe_native) {
g_hash_table_foreach(action->rsc->parameters, hash2smartfield, args_xml);
if (xml_has_children(action->rsc->versioned_parameters)) {
add_node_copy(action_xml, action->rsc->versioned_parameters);
}
}
g_hash_table_foreach(action->meta, hash2metafield, args_xml);
if (action->rsc != NULL) {
int isolated = 0;
resource_t *parent = action->rsc;
while (parent != NULL) {
isolated |= parent->isolation_wrapper ? 1 : 0;
parent->cmds->append_meta(parent, args_xml);
parent = parent->parent;
}
if (isolated && action->node) {
char *nodeattr = crm_meta_name(XML_RSC_ATTR_ISOLATION_HOST);
crm_xml_add(args_xml, nodeattr, action->node->details->uname);
free(nodeattr);
}
} else if (safe_str_eq(action->task, CRM_OP_FENCE) && action->node) {
g_hash_table_foreach(action->node->details->attrs, hash2metafield, args_xml);
}
sorted_xml(args_xml, action_xml, FALSE);
free_xml(args_xml);
/* List any nodes this action is expected to make down */
if (needs_node_info && (action->node != NULL)) {
add_downed_nodes(action_xml, action, data_set);
}
crm_log_xml_trace(action_xml, "dumped action");
return action_xml;
}
static gboolean
should_dump_action(action_t * action)
{
CRM_CHECK(action != NULL, return FALSE);
if (is_set(action->flags, pe_action_dumped)) {
crm_trace("action %d (%s) was already dumped", action->id, action->uuid);
return FALSE;
} else if (is_set(action->flags, pe_action_pseudo) && safe_str_eq(action->task, CRM_OP_PROBED)) {
GListPtr lpc = NULL;
/* This is a horrible but convenient hack
*
* It mimimizes the number of actions with unsatisfied inputs
* (ie. not included in the graph)
*
* This in turn, means we can be more concise when printing
* aborted/incomplete graphs.
*
* It also makes it obvious which node is preventing
* probe_complete from running (presumably because it is only
* partially up)
*
* For these reasons we tolerate such perversions
*/
for (lpc = action->actions_after; lpc != NULL; lpc = lpc->next) {
action_wrapper_t *wrapper = (action_wrapper_t *) lpc->data;
if (is_not_set(wrapper->action->flags, pe_action_runnable)) {
/* Only interested in runnable operations */
} else if (safe_str_neq(wrapper->action->task, RSC_START)) {
/* Only interested in start operations */
} else if (is_set(wrapper->action->flags, pe_action_dumped)) {
crm_trace("action %d (%s) dependency of %s",
action->id, action->uuid, wrapper->action->uuid);
return TRUE;
} else if (should_dump_action(wrapper->action)) {
crm_trace("action %d (%s) dependency of %s",
action->id, action->uuid, wrapper->action->uuid);
return TRUE;
}
}
}
if (is_set(action->flags, pe_action_runnable) == FALSE) {
crm_trace("action %d (%s) was not runnable", action->id, action->uuid);
return FALSE;
} else if (is_set(action->flags, pe_action_optional)
&& is_set(action->flags, pe_action_print_always) == FALSE) {
crm_trace("action %d (%s) was optional", action->id, action->uuid);
return FALSE;
} else if (action->rsc != NULL && is_not_set(action->rsc->flags, pe_rsc_managed)) {
const char *interval = NULL;
interval = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL);
/* make sure probes and recurring monitors go through */
if (safe_str_neq(action->task, RSC_STATUS) && interval == NULL) {
crm_trace("action %d (%s) was for an unmanaged resource (%s)",
action->id, action->uuid, action->rsc->id);
return FALSE;
}
}
if (is_set(action->flags, pe_action_pseudo)
|| safe_str_eq(action->task, CRM_OP_FENCE)
|| safe_str_eq(action->task, CRM_OP_SHUTDOWN)) {
/* skip the next checks */
return TRUE;
}
if (action->node == NULL) {
pe_err("action %d (%s) was not allocated", action->id, action->uuid);
log_action(LOG_DEBUG, "Unallocated action", action, FALSE);
return FALSE;
- } else if (action->node->details->online == FALSE) {
+ } else if (action->node->details->online == FALSE
+ && !is_container_remote_node(action->node)) {
pe_err("action %d was (%s) scheduled for offline node", action->id, action->uuid);
log_action(LOG_DEBUG, "Action for offline node", action, FALSE);
return FALSE;
#if 0
/* but this would also affect resources that can be safely
* migrated before a fencing op
*/
} else if (action->node->details->unclean == FALSE) {
pe_err("action %d was (%s) scheduled for unclean node", action->id, action->uuid);
log_action(LOG_DEBUG, "Action for unclean node", action, FALSE);
return FALSE;
#endif
}
return TRUE;
}
/* lowest to highest */
static gint
sort_action_id(gconstpointer a, gconstpointer b)
{
const action_wrapper_t *action_wrapper2 = (const action_wrapper_t *)a;
const action_wrapper_t *action_wrapper1 = (const action_wrapper_t *)b;
if (a == NULL) {
return 1;
}
if (b == NULL) {
return -1;
}
if (action_wrapper1->action->id > action_wrapper2->action->id) {
return -1;
}
if (action_wrapper1->action->id < action_wrapper2->action->id) {
return 1;
}
return 0;
}
static gboolean
check_dump_input(int last_action, action_t * action, action_wrapper_t * wrapper)
{
int type = wrapper->type;
if (wrapper->state == pe_link_dumped) {
return TRUE;
} else if (wrapper->state == pe_link_dup) {
return FALSE;
}
type &= ~pe_order_implies_first_printed;
type &= ~pe_order_implies_then_printed;
type &= ~pe_order_optional;
if (wrapper->action->node
&& action->rsc && action->rsc->fillers
&& is_not_set(type, pe_order_preserve)
&& wrapper->action->node->details->remote_rsc
&& uber_parent(action->rsc) != uber_parent(wrapper->action->rsc)
) {
/* This prevents user-defined ordering constraints between
* resources in remote nodes and the resources that
* define/represent a remote node.
*
* There is no known valid reason to allow this sort of thing
* but if one arises, we'd need to change the
* action->rsc->fillers clause to be more specific, possibly
* to check that it contained wrapper->action->rsc
*/
crm_warn("Invalid ordering constraint between %s and %s",
wrapper->action->rsc->id, action->rsc->id);
wrapper->type = pe_order_none;
return FALSE;
}
if (last_action == wrapper->action->id) {
crm_trace("Input (%d) %s duplicated for %s",
wrapper->action->id, wrapper->action->uuid, action->uuid);
wrapper->state = pe_link_dup;
return FALSE;
} else if (wrapper->type == pe_order_none) {
crm_trace("Input (%d) %s suppressed for %s",
wrapper->action->id, wrapper->action->uuid, action->uuid);
return FALSE;
} else if (is_set(wrapper->action->flags, pe_action_runnable) == FALSE
&& type == pe_order_none && safe_str_neq(wrapper->action->uuid, CRM_OP_PROBED)) {
crm_trace("Input (%d) %s optional (ordering) for %s",
wrapper->action->id, wrapper->action->uuid, action->uuid);
return FALSE;
} else if (is_set(wrapper->action->flags, pe_action_runnable) == FALSE
&& is_set(type, pe_order_one_or_more)) {
crm_trace("Input (%d) %s optional (one-or-more) for %s",
wrapper->action->id, wrapper->action->uuid, action->uuid);
return FALSE;
} else if (is_set(action->flags, pe_action_pseudo)
&& (wrapper->type & pe_order_stonith_stop)) {
crm_trace("Input (%d) %s suppressed for %s",
wrapper->action->id, wrapper->action->uuid, action->uuid);
return FALSE;
} else if ((wrapper->type & pe_order_implies_first_migratable) && (is_set(wrapper->action->flags, pe_action_runnable) == FALSE)) {
return FALSE;
} else if ((wrapper->type & pe_order_apply_first_non_migratable)
&& (is_set(wrapper->action->flags, pe_action_migrate_runnable))) {
return FALSE;
} else if ((wrapper->type == pe_order_optional)
&& crm_ends_with(wrapper->action->uuid, "_stop_0")
&& is_set(wrapper->action->flags, pe_action_migrate_runnable)) {
/* for optional only ordering, ordering is not preserved for
* a stop action that is actually involved with a migration. */
return FALSE;
} else if (wrapper->type == pe_order_load) {
crm_trace("check load filter %s.%s -> %s.%s",
wrapper->action->uuid,
wrapper->action->node ? wrapper->action->node->details->uname : "", action->uuid,
action->node ? action->node->details->uname : "");
if (action->rsc && safe_str_eq(action->task, RSC_MIGRATE)) {
/* Remove the orders like the following if not relevant:
* "load_stopped_node2" -> "rscA_migrate_to node1"
* which were created also from: pengine/native.c: MigrateRsc()
* order_actions(other, then, other_w->type);
*/
/* For migrate_to ops, we care about where it has been
* allocated to, not where the action will be executed
*/
if (wrapper->action->node == NULL || action->rsc->allocated_to == NULL
|| wrapper->action->node->details != action->rsc->allocated_to->details) {
/* Check if the actions are for the same node, ignore otherwise */
crm_trace("load filter - migrate");
wrapper->type = pe_order_none;
return FALSE;
}
} else if (wrapper->action->node == NULL || action->node == NULL
|| wrapper->action->node->details != action->node->details) {
/* Check if the actions are for the same node, ignore otherwise */
crm_trace("load filter - node");
wrapper->type = pe_order_none;
return FALSE;
} else if (is_set(wrapper->action->flags, pe_action_optional)) {
/* Check if the pre-req is optional, ignore if so */
crm_trace("load filter - optional");
wrapper->type = pe_order_none;
return FALSE;
}
} else if (wrapper->type == pe_order_anti_colocation) {
crm_trace("check anti-colocation filter %s.%s -> %s.%s",
wrapper->action->uuid,
wrapper->action->node ? wrapper->action->node->details->uname : "",
action->uuid,
action->node ? action->node->details->uname : "");
if (wrapper->action->node && action->node
&& wrapper->action->node->details != action->node->details) {
/* Check if the actions are for the same node, ignore otherwise */
crm_trace("anti-colocation filter - node");
wrapper->type = pe_order_none;
return FALSE;
} else if (is_set(wrapper->action->flags, pe_action_optional)) {
/* Check if the pre-req is optional, ignore if so */
crm_trace("anti-colocation filter - optional");
wrapper->type = pe_order_none;
return FALSE;
}
} else if (wrapper->action->rsc
&& wrapper->action->rsc != action->rsc
&& is_set(wrapper->action->rsc->flags, pe_rsc_failed)
&& is_not_set(wrapper->action->rsc->flags, pe_rsc_managed)
&& crm_ends_with(wrapper->action->uuid, "_stop_0")
&& action->rsc && action->rsc->variant >= pe_clone) {
crm_warn("Ignoring requirement that %s complete before %s:"
" unmanaged failed resources cannot prevent clone shutdown",
wrapper->action->uuid, action->uuid);
return FALSE;
} else if (is_set(wrapper->action->flags, pe_action_dumped)
|| should_dump_action(wrapper->action)) {
crm_trace("Input (%d) %s should be dumped for %s", wrapper->action->id,
wrapper->action->uuid, action->uuid);
goto dump;
#if 0
} else if (is_set(wrapper->action->flags, pe_action_runnable)
&& is_set(wrapper->action->flags, pe_action_pseudo)
&& wrapper->action->rsc->variant != pe_native) {
crm_crit("Input (%d) %s should be dumped for %s",
wrapper->action->id, wrapper->action->uuid, action->uuid);
goto dump;
#endif
} else if (is_set(wrapper->action->flags, pe_action_optional) == TRUE
&& is_set(wrapper->action->flags, pe_action_print_always) == FALSE) {
crm_trace("Input (%d) %s optional for %s", wrapper->action->id,
wrapper->action->uuid, action->uuid);
crm_trace("Input (%d) %s n=%p p=%d r=%d o=%d a=%d f=0x%.6x",
wrapper->action->id, wrapper->action->uuid, wrapper->action->node,
is_set(wrapper->action->flags, pe_action_pseudo),
is_set(wrapper->action->flags, pe_action_runnable),
is_set(wrapper->action->flags, pe_action_optional),
is_set(wrapper->action->flags, pe_action_print_always), wrapper->type);
return FALSE;
}
dump:
return TRUE;
}
static gboolean
graph_has_loop(action_t * init_action, action_t * action, action_wrapper_t * wrapper)
{
GListPtr lpc = NULL;
gboolean has_loop = FALSE;
if (is_set(wrapper->action->flags, pe_action_tracking)) {
crm_trace("Breaking tracking loop: %s.%s -> %s.%s (0x%.6x)",
wrapper->action->uuid,
wrapper->action->node ? wrapper->action->node->details->uname : "",
action->uuid,
action->node ? action->node->details->uname : "",
wrapper->type);
return FALSE;
}
if (check_dump_input(-1, action, wrapper) == FALSE) {
return FALSE;
}
/* If there's any order like:
* "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
* rscA is being migrated from node1 to node2,
* while rscB is being migrated from node2 to node1.
* There will be potential graph loop.
* Break the order "load_stopped_node2" -> "rscA_migrate_to node1".
*/
crm_trace("Checking graph loop: %s.%s -> %s.%s (0x%.6x)",
wrapper->action->uuid,
wrapper->action->node ? wrapper->action->node->details->uname : "",
action->uuid,
action->node ? action->node->details->uname : "",
wrapper->type);
if (wrapper->action == init_action) {
crm_debug("Found graph loop: %s.%s ->...-> %s.%s",
action->uuid,
action->node ? action->node->details->uname : "",
init_action->uuid,
init_action->node ? init_action->node->details->uname : "");
return TRUE;
}
set_bit(wrapper->action->flags, pe_action_tracking);
for (lpc = wrapper->action->actions_before; lpc != NULL; lpc = lpc->next) {
action_wrapper_t *wrapper_before = (action_wrapper_t *) lpc->data;
if (graph_has_loop(init_action, wrapper->action, wrapper_before)) {
has_loop = TRUE;
goto done;
}
}
done:
clear_bit(wrapper->action->flags, pe_action_tracking);
return has_loop;
}
static gboolean
should_dump_input(int last_action, action_t * action, action_wrapper_t * wrapper)
{
wrapper->state = pe_link_not_dumped;
if (check_dump_input(last_action, action, wrapper) == FALSE) {
return FALSE;
}
if (wrapper->type == pe_order_load
&& action->rsc
&& safe_str_eq(action->task, RSC_MIGRATE)) {
crm_trace("Checking graph loop - load migrate: %s.%s -> %s.%s",
wrapper->action->uuid,
wrapper->action->node ? wrapper->action->node->details->uname : "",
action->uuid,
action->node ? action->node->details->uname : "");
if (graph_has_loop(action, action, wrapper)) {
/* Remove the orders like the following if they are introducing any graph loops:
* "load_stopped_node2" -> "rscA_migrate_to node1"
* which were created also from: pengine/native.c: MigrateRsc()
* order_actions(other, then, other_w->type);
*/
crm_debug("Breaking graph loop - load migrate: %s.%s -> %s.%s",
wrapper->action->uuid,
wrapper->action->node ? wrapper->action->node->details->uname : "",
action->uuid,
action->node ? action->node->details->uname : "");
wrapper->type = pe_order_none;
return FALSE;
}
}
crm_trace("Input (%d) %s n=%p p=%d r=%d o=%d a=%d f=0x%.6x dumped for %s",
wrapper->action->id,
wrapper->action->uuid,
wrapper->action->node,
is_set(wrapper->action->flags, pe_action_pseudo),
is_set(wrapper->action->flags, pe_action_runnable),
is_set(wrapper->action->flags, pe_action_optional),
is_set(wrapper->action->flags, pe_action_print_always), wrapper->type, action->uuid);
return TRUE;
}
void
graph_element_from_action(action_t * action, pe_working_set_t * data_set)
{
GListPtr lpc = NULL;
int last_action = -1;
int synapse_priority = 0;
xmlNode *syn = NULL;
xmlNode *set = NULL;
xmlNode *in = NULL;
xmlNode *input = NULL;
xmlNode *xml_action = NULL;
if (should_dump_action(action) == FALSE) {
return;
}
set_bit(action->flags, pe_action_dumped);
syn = create_xml_node(data_set->graph, "synapse");
set = create_xml_node(syn, "action_set");
in = create_xml_node(syn, "inputs");
crm_xml_add_int(syn, XML_ATTR_ID, data_set->num_synapse);
data_set->num_synapse++;
if (action->rsc != NULL) {
synapse_priority = action->rsc->priority;
}
if (action->priority > synapse_priority) {
synapse_priority = action->priority;
}
if (synapse_priority > 0) {
crm_xml_add_int(syn, XML_CIB_ATTR_PRIORITY, synapse_priority);
}
xml_action = action2xml(action, FALSE, data_set);
add_node_nocopy(set, crm_element_name(xml_action), xml_action);
action->actions_before = g_list_sort(action->actions_before, sort_action_id);
for (lpc = action->actions_before; lpc != NULL; lpc = lpc->next) {
action_wrapper_t *wrapper = (action_wrapper_t *) lpc->data;
if (should_dump_input(last_action, action, wrapper) == FALSE) {
continue;
}
wrapper->state = pe_link_dumped;
CRM_CHECK(last_action < wrapper->action->id,;
);
last_action = wrapper->action->id;
input = create_xml_node(in, "trigger");
xml_action = action2xml(wrapper->action, TRUE, data_set);
add_node_nocopy(input, crm_element_name(xml_action), xml_action);
}
}
diff --git a/pengine/utils.c b/pengine/utils.c
index a67e572700..755f1c8cf9 100644
--- a/pengine/utils.c
+++ b/pengine/utils.c
@@ -1,413 +1,424 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <crm/msg_xml.h>
#include <allocate.h>
#include <utils.h>
void
pe_free_ordering(GListPtr constraints)
{
GListPtr iterator = constraints;
while (iterator != NULL) {
order_constraint_t *order = iterator->data;
iterator = iterator->next;
free(order->lh_action_task);
free(order->rh_action_task);
free(order);
}
if (constraints != NULL) {
g_list_free(constraints);
}
}
void
pe_free_rsc_to_node(GListPtr constraints)
{
GListPtr iterator = constraints;
while (iterator != NULL) {
rsc_to_node_t *cons = iterator->data;
iterator = iterator->next;
g_list_free_full(cons->node_list_rh, free);
free(cons->id);
free(cons);
}
if (constraints != NULL) {
g_list_free(constraints);
}
}
rsc_to_node_t *
rsc2node_new(const char *id, resource_t * rsc,
int node_weight, const char *discover_mode,
node_t * foo_node, pe_working_set_t * data_set)
{
rsc_to_node_t *new_con = NULL;
if (rsc == NULL || id == NULL) {
pe_err("Invalid constraint %s for rsc=%p", crm_str(id), rsc);
return NULL;
} else if (foo_node == NULL) {
CRM_CHECK(node_weight == 0, return NULL);
}
new_con = calloc(1, sizeof(rsc_to_node_t));
if (new_con != NULL) {
new_con->id = strdup(id);
new_con->rsc_lh = rsc;
new_con->node_list_rh = NULL;
new_con->role_filter = RSC_ROLE_UNKNOWN;
if (discover_mode == NULL || safe_str_eq(discover_mode, "always")) {
new_con->discover_mode = discover_always;
} else if (safe_str_eq(discover_mode, "never")) {
new_con->discover_mode = discover_never;
} else if (safe_str_eq(discover_mode, "exclusive")) {
new_con->discover_mode = discover_exclusive;
rsc->exclusive_discover = TRUE;
} else {
pe_err("Invalid %s value %s in location constraint", XML_LOCATION_ATTR_DISCOVERY, discover_mode);
}
if (foo_node != NULL) {
node_t *copy = node_copy(foo_node);
copy->weight = merge_weights(node_weight, foo_node->weight);
new_con->node_list_rh = g_list_prepend(NULL, copy);
}
data_set->placement_constraints = g_list_prepend(data_set->placement_constraints, new_con);
rsc->rsc_location = g_list_prepend(rsc->rsc_location, new_con);
}
return new_con;
}
gboolean
can_run_resources(const node_t * node)
{
if (node == NULL) {
return FALSE;
}
#if 0
if (node->weight < 0) {
return FALSE;
}
#endif
if (node->details->online == FALSE
|| node->details->shutdown || node->details->unclean
|| node->details->standby || node->details->maintenance) {
crm_trace("%s: online=%d, unclean=%d, standby=%d, maintenance=%d",
node->details->uname, node->details->online,
node->details->unclean, node->details->standby, node->details->maintenance);
return FALSE;
}
return TRUE;
}
/* return -1 if 'a' is more preferred
* return 1 if 'b' is more preferred
*/
gint
sort_node_weight(gconstpointer a, gconstpointer b, gpointer data)
{
const node_t *node1 = (const node_t *)a;
const node_t *node2 = (const node_t *)b;
const node_t *active = (node_t *) data;
int node1_weight = 0;
int node2_weight = 0;
int result = 0;
if (a == NULL) {
return 1;
}
if (b == NULL) {
return -1;
}
node1_weight = node1->weight;
node2_weight = node2->weight;
if (can_run_resources(node1) == FALSE) {
node1_weight = -INFINITY;
}
if (can_run_resources(node2) == FALSE) {
node2_weight = -INFINITY;
}
if (node1_weight > node2_weight) {
crm_trace("%s (%d) > %s (%d) : weight",
node1->details->uname, node1_weight, node2->details->uname, node2_weight);
return -1;
}
if (node1_weight < node2_weight) {
crm_trace("%s (%d) < %s (%d) : weight",
node1->details->uname, node1_weight, node2->details->uname, node2_weight);
return 1;
}
crm_trace("%s (%d) == %s (%d) : weight",
node1->details->uname, node1_weight, node2->details->uname, node2_weight);
if (safe_str_eq(pe_dataset->placement_strategy, "minimal")) {
goto equal;
}
if (safe_str_eq(pe_dataset->placement_strategy, "balanced")) {
result = compare_capacity(node1, node2);
if (result != 0) {
return result;
}
}
/* now try to balance resources across the cluster */
if (node1->details->num_resources < node2->details->num_resources) {
crm_trace("%s (%d) < %s (%d) : resources",
node1->details->uname, node1->details->num_resources,
node2->details->uname, node2->details->num_resources);
return -1;
} else if (node1->details->num_resources > node2->details->num_resources) {
crm_trace("%s (%d) > %s (%d) : resources",
node1->details->uname, node1->details->num_resources,
node2->details->uname, node2->details->num_resources);
return 1;
}
if (active && active->details == node1->details) {
crm_trace("%s (%d) > %s (%d) : active",
node1->details->uname, node1->details->num_resources,
node2->details->uname, node2->details->num_resources);
return -1;
} else if (active && active->details == node2->details) {
crm_trace("%s (%d) > %s (%d) : active",
node1->details->uname, node1->details->num_resources,
node2->details->uname, node2->details->num_resources);
return 1;
}
equal:
crm_trace("%s = %s", node1->details->uname, node2->details->uname);
return strcmp(node1->details->uname, node2->details->uname);
}
void
native_deallocate(resource_t * rsc)
{
if (rsc->allocated_to) {
node_t *old = rsc->allocated_to;
crm_info("Deallocating %s from %s", rsc->id, old->details->uname);
set_bit(rsc->flags, pe_rsc_provisional);
rsc->allocated_to = NULL;
old->details->allocated_rsc = g_list_remove(old->details->allocated_rsc, rsc);
old->details->num_resources--;
/* old->count--; */
calculate_utilization(old->details->utilization, rsc->utilization, TRUE);
free(old);
}
}
gboolean
native_assign_node(resource_t * rsc, GListPtr nodes, node_t * chosen, gboolean force)
{
CRM_ASSERT(rsc->variant == pe_native);
- if (force == FALSE
- && chosen != NULL && (can_run_resources(chosen) == FALSE || chosen->weight < 0)) {
- crm_debug("All nodes for resource %s are unavailable"
- ", unclean or shutting down (%s: %d, %d)",
- rsc->id, chosen->details->uname, can_run_resources(chosen), chosen->weight);
- rsc->next_role = RSC_ROLE_STOPPED;
- chosen = NULL;
+ if (force == FALSE && chosen != NULL) {
+ bool unset = FALSE;
+
+ if(chosen->weight < 0) {
+ unset = TRUE;
+
+ // Allow the graph to assume that the remote resource will come up
+ } else if(can_run_resources(chosen) == FALSE && !is_container_remote_node(chosen)) {
+ unset = TRUE;
+ }
+
+ if(unset) {
+ crm_debug("All nodes for resource %s are unavailable"
+ ", unclean or shutting down (%s: %d, %d)",
+ rsc->id, chosen->details->uname, can_run_resources(chosen), chosen->weight);
+ rsc->next_role = RSC_ROLE_STOPPED;
+ chosen = NULL;
+ }
}
/* todo: update the old node for each resource to reflect its
* new resource count
*/
native_deallocate(rsc);
clear_bit(rsc->flags, pe_rsc_provisional);
if (chosen == NULL) {
GListPtr gIter = NULL;
char *rc_inactive = crm_itoa(PCMK_OCF_NOT_RUNNING);
crm_debug("Could not allocate a node for %s", rsc->id);
rsc->next_role = RSC_ROLE_STOPPED;
for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
action_t *op = (action_t *) gIter->data;
const char *interval = g_hash_table_lookup(op->meta, XML_LRM_ATTR_INTERVAL);
crm_debug("Processing %s", op->uuid);
if(safe_str_eq(RSC_STOP, op->task)) {
update_action_flags(op, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__);
} else if(safe_str_eq(RSC_START, op->task)) {
update_action_flags(op, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
/* set_bit(rsc->flags, pe_rsc_block); */
} else if(interval && safe_str_neq(interval, "0")) {
if(safe_str_eq(rc_inactive, g_hash_table_lookup(op->meta, XML_ATTR_TE_TARGET_RC))) {
/* This is a recurring monitor for the stopped state, leave it alone */
} else {
/* Normal monitor operation, cancel it */
update_action_flags(op, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
}
}
}
free(rc_inactive);
return FALSE;
}
crm_debug("Assigning %s to %s", chosen->details->uname, rsc->id);
rsc->allocated_to = node_copy(chosen);
chosen->details->allocated_rsc = g_list_prepend(chosen->details->allocated_rsc, rsc);
chosen->details->num_resources++;
chosen->count++;
calculate_utilization(chosen->details->utilization, rsc->utilization, FALSE);
dump_rsc_utilization(show_utilization ? 0 : utilization_log_level, __FUNCTION__, rsc, chosen);
return TRUE;
}
void
log_action(unsigned int log_level, const char *pre_text, action_t * action, gboolean details)
{
const char *node_uname = NULL;
const char *node_uuid = NULL;
if (action == NULL) {
crm_trace("%s%s: <NULL>", pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": ");
return;
}
if (is_set(action->flags, pe_action_pseudo)) {
node_uname = NULL;
node_uuid = NULL;
} else if (action->node != NULL) {
node_uname = action->node->details->uname;
node_uuid = action->node->details->id;
} else {
node_uname = "<none>";
node_uuid = NULL;
}
switch (text2task(action->task)) {
case stonith_node:
case shutdown_crm:
crm_trace("%s%s%sAction %d: %s%s%s%s%s%s",
pre_text == NULL ? "" : pre_text,
pre_text == NULL ? "" : ": ",
is_set(action->flags,
pe_action_pseudo) ? "Pseudo " : is_set(action->flags,
pe_action_optional) ?
"Optional " : is_set(action->flags,
pe_action_runnable) ? is_set(action->flags,
pe_action_processed)
? "" : "(Provisional) " : "!!Non-Startable!! ", action->id,
action->uuid, node_uname ? "\ton " : "",
node_uname ? node_uname : "", node_uuid ? "\t\t(" : "",
node_uuid ? node_uuid : "", node_uuid ? ")" : "");
break;
default:
crm_trace("%s%s%sAction %d: %s %s%s%s%s%s%s",
pre_text == NULL ? "" : pre_text,
pre_text == NULL ? "" : ": ",
is_set(action->flags,
pe_action_optional) ? "Optional " : is_set(action->flags,
pe_action_pseudo)
? "Pseudo " : is_set(action->flags,
pe_action_runnable) ? is_set(action->flags,
pe_action_processed)
? "" : "(Provisional) " : "!!Non-Startable!! ", action->id,
action->uuid, action->rsc ? action->rsc->id : "<none>",
node_uname ? "\ton " : "", node_uname ? node_uname : "",
node_uuid ? "\t\t(" : "", node_uuid ? node_uuid : "", node_uuid ? ")" : "");
break;
}
if (details) {
GListPtr gIter = NULL;
crm_trace("\t\t====== Preceding Actions");
gIter = action->actions_before;
for (; gIter != NULL; gIter = gIter->next) {
action_wrapper_t *other = (action_wrapper_t *) gIter->data;
log_action(log_level + 1, "\t\t", other->action, FALSE);
}
crm_trace("\t\t====== Subsequent Actions");
gIter = action->actions_after;
for (; gIter != NULL; gIter = gIter->next) {
action_wrapper_t *other = (action_wrapper_t *) gIter->data;
log_action(log_level + 1, "\t\t", other->action, FALSE);
}
crm_trace("\t\t====== End");
} else {
crm_trace("\t\t(seen=%d, before=%d, after=%d)",
action->seen_count,
g_list_length(action->actions_before), g_list_length(action->actions_after));
}
}
gboolean
can_run_any(GHashTable * nodes)
{
GHashTableIter iter;
node_t *node = NULL;
if (nodes == NULL) {
return FALSE;
}
g_hash_table_iter_init(&iter, nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
if (can_run_resources(node) && node->weight >= 0) {
return TRUE;
}
}
return FALSE;
}
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Mon, Apr 21, 6:11 PM (1 d, 49 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1665123
Default Alt Text
(135 KB)
Attached To
Mode
rP Pacemaker
Attached
Detach File
Event Timeline
Log In to Comment