diff --git a/pengine/master.c b/pengine/master.c index 8c39f492cc..3c3b06872f 100644 --- a/pengine/master.c +++ b/pengine/master.c @@ -1,1060 +1,1063 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #define VARIANT_CLONE 1 #include extern gint sort_clone_instance(gconstpointer a, gconstpointer b, gpointer data_set); static int master_score(resource_t * rsc, node_t * node, int not_set_value); static void child_promoting_constraints(clone_variant_data_t * clone_data, enum pe_ordering type, resource_t * rsc, resource_t * child, resource_t * last, pe_working_set_t * data_set) { if (child == NULL) { if (clone_data->ordered && last != NULL) { pe_rsc_trace(rsc, "Ordered version (last node)"); /* last child promote before promoted started */ new_rsc_order(last, RSC_PROMOTE, rsc, RSC_PROMOTED, type, data_set); } return; } /* child promote before global promoted */ new_rsc_order(child, RSC_PROMOTE, rsc, RSC_PROMOTED, type, data_set); /* global promote before child promote */ new_rsc_order(rsc, RSC_PROMOTE, child, RSC_PROMOTE, type, data_set); if (clone_data->ordered) { pe_rsc_trace(rsc, "Ordered version"); if (last == NULL) { /* global promote before first child promote */ last = rsc; } /* else: child/child relative promote */ order_start_start(last, child, type); new_rsc_order(last, RSC_PROMOTE, child, RSC_PROMOTE, type, data_set); } else { pe_rsc_trace(rsc, "Un-ordered version"); } } static void child_demoting_constraints(clone_variant_data_t * clone_data, enum pe_ordering type, resource_t * rsc, resource_t * child, resource_t * last, pe_working_set_t * data_set) { if (child == NULL) { if (clone_data->ordered && last != NULL) { pe_rsc_trace(rsc, "Ordered version (last node)"); /* global demote before first child demote */ new_rsc_order(rsc, RSC_DEMOTE, last, RSC_DEMOTE, pe_order_optional, data_set); } return; } /* child demote before global demoted */ new_rsc_order(child, RSC_DEMOTE, rsc, RSC_DEMOTED, pe_order_implies_then_printed, data_set); /* global demote before child demote */ new_rsc_order(rsc, RSC_DEMOTE, child, RSC_DEMOTE, pe_order_implies_first_printed, data_set); if (clone_data->ordered && last != NULL) { pe_rsc_trace(rsc, "Ordered version"); /* child/child relative demote */ new_rsc_order(child, RSC_DEMOTE, last, RSC_DEMOTE, type, data_set); } else if (clone_data->ordered) { pe_rsc_trace(rsc, "Ordered version (1st node)"); /* first child stop before global stopped */ new_rsc_order(child, RSC_DEMOTE, rsc, RSC_DEMOTED, type, data_set); } else { pe_rsc_trace(rsc, "Un-ordered version"); } } static void master_update_pseudo_status(resource_t * rsc, gboolean * demoting, gboolean * promoting) { GListPtr gIter = NULL; if (rsc->children) { gIter = rsc->children; for (; gIter != NULL; gIter = gIter->next) { resource_t *child = (resource_t *) gIter->data; master_update_pseudo_status(child, demoting, promoting); } return; } CRM_ASSERT(demoting != NULL); CRM_ASSERT(promoting != NULL); gIter = rsc->actions; for (; gIter != NULL; gIter = gIter->next) { action_t *action = (action_t *) gIter->data; if (*promoting && *demoting) { return; } else if (is_set(action->flags, pe_action_optional)) { continue; } else if (safe_str_eq(RSC_DEMOTE, action->task)) { *demoting = TRUE; } else if (safe_str_eq(RSC_PROMOTE, action->task)) { *promoting = TRUE; } } } static void apply_master_location(resource_t *child, GListPtr location_constraints, pe_node_t *chosen) { - for(GListPtr gIter = location_constraints; gIter != NULL; gIter = gIter->next) { - pe_node_t *cons_node = NULL; - rsc_to_node_t *cons = (rsc_to_node_t*)gIter->data; - - if(cons->role_filter == RSC_ROLE_MASTER) { - pe_rsc_trace(child, "Applying %s to %s", cons->id, child->id); - cons_node = pe_find_node_id(cons->node_list_rh, chosen->details->id); - } - if(cons_node != NULL) { - int new_priority = merge_weights(child->priority, cons_node->weight); - pe_rsc_trace(child, "\t%s[%s]: %d -> %d (%d)", child->id, cons_node->details->uname, - child->priority, new_priority, cons_node->weight); - child->priority = new_priority; + CRM_CHECK(child && chosen, return); + for (GListPtr gIter = location_constraints; gIter; gIter = gIter->next) { + pe_node_t *cons_node = NULL; + rsc_to_node_t *cons = (rsc_to_node_t *) gIter->data; + + if (cons->role_filter == RSC_ROLE_MASTER) { + pe_rsc_trace(child, "Applying %s to %s", cons->id, child->id); + cons_node = pe_find_node_id(cons->node_list_rh, chosen->details->id); + } + if (cons_node != NULL) { + int new_priority = merge_weights(child->priority, cons_node->weight); + + pe_rsc_trace(child, "\t%s[%s]: %d -> %d (%d)", + child->id, cons_node->details->uname, child->priority, + new_priority, cons_node->weight); + child->priority = new_priority; } } } static node_t * can_be_master(resource_t * rsc) { node_t *node = NULL; node_t *local_node = NULL; resource_t *parent = uber_parent(rsc); clone_variant_data_t *clone_data = NULL; #if 0 enum rsc_role_e role = RSC_ROLE_UNKNOWN; role = rsc->fns->state(rsc, FALSE); crm_info("%s role: %s", rsc->id, role2text(role)); #endif if (rsc->children) { GListPtr gIter = rsc->children; for (; gIter != NULL; gIter = gIter->next) { resource_t *child = (resource_t *) gIter->data; if (can_be_master(child) == NULL) { pe_rsc_trace(rsc, "Child %s of %s can't be promoted", child->id, rsc->id); return NULL; } } } node = rsc->fns->location(rsc, NULL, FALSE); if (node == NULL) { pe_rsc_trace(rsc, "%s cannot be master: not allocated", rsc->id); return NULL; } else if (is_not_set(rsc->flags, pe_rsc_managed)) { if (rsc->fns->state(rsc, TRUE) == RSC_ROLE_MASTER) { crm_notice("Forcing unmanaged master %s to remain promoted on %s", rsc->id, node->details->uname); } else { return NULL; } } else if (rsc->priority < 0) { pe_rsc_trace(rsc, "%s cannot be master: preference: %d", rsc->id, rsc->priority); return NULL; } else if (can_run_resources(node) == FALSE) { crm_trace("Node can't run any resources: %s", node->details->uname); return NULL; } get_clone_variant_data(clone_data, parent); local_node = pe_hash_table_lookup(parent->allowed_nodes, node->details->id); if (local_node == NULL) { crm_err("%s cannot run on %s: node not allowed", rsc->id, node->details->uname); return NULL; } else if (local_node->count < clone_data->master_node_max || is_not_set(rsc->flags, pe_rsc_managed)) { return local_node; } else { pe_rsc_trace(rsc, "%s cannot be master on %s: node full", rsc->id, node->details->uname); } return NULL; } static gint sort_master_instance(gconstpointer a, gconstpointer b, gpointer data_set) { int rc; enum rsc_role_e role1 = RSC_ROLE_UNKNOWN; enum rsc_role_e role2 = RSC_ROLE_UNKNOWN; const resource_t *resource1 = (const resource_t *)a; const resource_t *resource2 = (const resource_t *)b; CRM_ASSERT(resource1 != NULL); CRM_ASSERT(resource2 != NULL); role1 = resource1->fns->state(resource1, TRUE); role2 = resource2->fns->state(resource2, TRUE); rc = sort_rsc_index(a, b); if (rc != 0) { crm_trace("%s %c %s (index)", resource1->id, rc < 0 ? '<' : '>', resource2->id); return rc; } if (role1 > role2) { crm_trace("%s %c %s (role)", resource1->id, '<', resource2->id); return -1; } else if (role1 < role2) { crm_trace("%s %c %s (role)", resource1->id, '>', resource2->id); return 1; } return sort_clone_instance(a, b, data_set); } GHashTable * master_merge_weights(resource_t * rsc, const char *rhs, GHashTable * nodes, const char *attr, float factor, enum pe_weights flags) { return rsc_merge_weights(rsc, rhs, nodes, attr, factor, flags); } static void master_promotion_order(resource_t * rsc, pe_working_set_t * data_set) { GListPtr gIter = NULL; node_t *node = NULL; node_t *chosen = NULL; clone_variant_data_t *clone_data = NULL; char score[33]; size_t len = sizeof(score); get_clone_variant_data(clone_data, rsc); if (clone_data->merged_master_weights) { return; } clone_data->merged_master_weights = TRUE; pe_rsc_trace(rsc, "Merging weights for %s", rsc->id); set_bit(rsc->flags, pe_rsc_merging); for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { resource_t *child = (resource_t *) gIter->data; pe_rsc_trace(rsc, "Sort index: %s = %d", child->id, child->sort_index); } dump_node_scores(LOG_DEBUG_3, rsc, "Before", rsc->allowed_nodes); gIter = rsc->children; for (; gIter != NULL; gIter = gIter->next) { resource_t *child = (resource_t *) gIter->data; chosen = child->fns->location(child, NULL, FALSE); if (chosen == NULL || child->sort_index < 0) { pe_rsc_trace(rsc, "Skipping %s", child->id); continue; } node = (node_t *) pe_hash_table_lookup(rsc->allowed_nodes, chosen->details->id); CRM_ASSERT(node != NULL); /* adds in master preferences and rsc_location.role=Master */ score2char_stack(child->sort_index, score, len); pe_rsc_trace(rsc, "Adding %s to %s from %s", score, node->details->uname, child->id); node->weight = merge_weights(child->sort_index, node->weight); } dump_node_scores(LOG_DEBUG_3, rsc, "Middle", rsc->allowed_nodes); gIter = rsc->rsc_cons; for (; gIter != NULL; gIter = gIter->next) { rsc_colocation_t *constraint = (rsc_colocation_t *) gIter->data; /* (re-)adds location preferences of resources that the * master instance should/must be colocated with */ if (constraint->role_lh == RSC_ROLE_MASTER) { enum pe_weights flags = constraint->score == INFINITY ? 0 : pe_weights_rollback; pe_rsc_trace(rsc, "RHS: %s with %s: %d", constraint->rsc_lh->id, constraint->rsc_rh->id, constraint->score); rsc->allowed_nodes = constraint->rsc_rh->cmds->merge_weights(constraint->rsc_rh, rsc->id, rsc->allowed_nodes, constraint->node_attribute, (float)constraint->score / INFINITY, flags); } } gIter = rsc->rsc_cons_lhs; for (; gIter != NULL; gIter = gIter->next) { rsc_colocation_t *constraint = (rsc_colocation_t *) gIter->data; /* (re-)adds location preferences of resource that wish to be * colocated with the master instance */ if (constraint->role_rh == RSC_ROLE_MASTER) { pe_rsc_trace(rsc, "LHS: %s with %s: %d", constraint->rsc_lh->id, constraint->rsc_rh->id, constraint->score); rsc->allowed_nodes = constraint->rsc_lh->cmds->merge_weights(constraint->rsc_lh, rsc->id, rsc->allowed_nodes, constraint->node_attribute, (float)constraint->score / INFINITY, (pe_weights_rollback | pe_weights_positive)); } } gIter = rsc->rsc_tickets; for (; gIter != NULL; gIter = gIter->next) { rsc_ticket_t *rsc_ticket = (rsc_ticket_t *) gIter->data; if (rsc_ticket->role_lh == RSC_ROLE_MASTER && (rsc_ticket->ticket->granted == FALSE || rsc_ticket->ticket->standby)) { resource_location(rsc, NULL, -INFINITY, "__stateful_without_ticket__", data_set); } } dump_node_scores(LOG_DEBUG_3, rsc, "After", rsc->allowed_nodes); /* write them back and sort */ gIter = rsc->children; for (; gIter != NULL; gIter = gIter->next) { resource_t *child = (resource_t *) gIter->data; chosen = child->fns->location(child, NULL, FALSE); if (is_not_set(child->flags, pe_rsc_managed) && child->next_role == RSC_ROLE_MASTER) { child->sort_index = INFINITY; } else if (chosen == NULL || child->sort_index < 0) { pe_rsc_trace(rsc, "%s: %d", child->id, child->sort_index); } else { node = (node_t *) pe_hash_table_lookup(rsc->allowed_nodes, chosen->details->id); CRM_ASSERT(node != NULL); child->sort_index = node->weight; } pe_rsc_trace(rsc, "Set sort index: %s = %d", child->id, child->sort_index); } rsc->children = g_list_sort_with_data(rsc->children, sort_master_instance, data_set); clear_bit(rsc->flags, pe_rsc_merging); } static gboolean filter_anonymous_instance(resource_t * rsc, node_t * node) { GListPtr rIter = NULL; char *key = clone_strip(rsc->id); resource_t *parent = uber_parent(rsc); for (rIter = parent->children; rIter; rIter = rIter->next) { resource_t *child = rIter->data; resource_t *active = parent->fns->find_rsc(child, key, node, pe_find_clone|pe_find_current); /* * Look for an active instance on $node, if there is one, only it receives the master score * Use ->find_rsc() because we might be a cloned group */ if(rsc == active) { pe_rsc_trace(rsc, "Found %s for %s active on %s: done", active->id, key, node->details->uname); free(key); return TRUE; } else if(active) { pe_rsc_trace(rsc, "Found %s for %s on %s: not %s", active->id, key, node->details->uname, rsc->id); free(key); return FALSE; } else { pe_rsc_trace(rsc, "%s on %s: not active", key, node->details->uname); } } for (rIter = parent->children; rIter; rIter = rIter->next) { resource_t *child = rIter->data; /* * We know it's not running, but any score will still count if * the instance has been probed on $node * * Again use ->find_rsc() because we might be a cloned group * and knowing that other members of the group are known here * implies nothing */ rsc = parent->fns->find_rsc(child, key, NULL, pe_find_clone); CRM_LOG_ASSERT(rsc); if(rsc) { pe_rsc_trace(rsc, "Checking %s for %s on %s", rsc->id, key, node->details->uname); if (g_hash_table_lookup(rsc->known_on, node->details->id)) { free(key); return TRUE; } } } free(key); return FALSE; } static int master_score(resource_t * rsc, node_t * node, int not_set_value) { char *attr_name; char *name = rsc->id; const char *attr_value = NULL; int score = not_set_value, len = 0; if (rsc->children) { GListPtr gIter = rsc->children; for (; gIter != NULL; gIter = gIter->next) { resource_t *child = (resource_t *) gIter->data; int c_score = master_score(child, node, not_set_value); if (score == not_set_value) { score = c_score; } else { score += c_score; } } return score; } if (node == NULL) { if (rsc->fns->state(rsc, TRUE) < RSC_ROLE_STARTED) { pe_rsc_trace(rsc, "Ignoring master score for %s: unknown state", rsc->id); return score; } } else { node_t *match = pe_find_node_id(rsc->running_on, node->details->id); node_t *known = pe_hash_table_lookup(rsc->known_on, node->details->id); if (is_not_set(rsc->flags, pe_rsc_unique) && filter_anonymous_instance(rsc, node)) { pe_rsc_trace(rsc, "Anonymous clone %s is allowed on %s", rsc->id, node->details->uname); } else if (match == NULL && known == NULL) { pe_rsc_trace(rsc, "%s (aka. %s) has been filtered on %s - ignoring", rsc->id, rsc->clone_name, node->details->uname); return score; } match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id); if (match == NULL) { return score; } else if (match->weight < 0) { pe_rsc_trace(rsc, "%s on %s has score: %d - ignoring", rsc->id, match->details->uname, match->weight); return score; } } if (rsc->clone_name) { /* Use the name the lrm knows this resource as, * since that's what crm_master would have used too */ name = rsc->clone_name; } len = 8 + strlen(name); attr_name = calloc(1, len); sprintf(attr_name, "master-%s", name); if (node) { attr_value = g_hash_table_lookup(node->details->attrs, attr_name); pe_rsc_trace(rsc, "%s: %s[%s] = %s", rsc->id, attr_name, node->details->uname, crm_str(attr_value)); } if (attr_value != NULL) { score = char2score(attr_value); } free(attr_name); return score; } #define max(a, b) achildren; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); if (clone_data->applied_master_prefs) { /* Make sure we only do this once */ return; } clone_data->applied_master_prefs = TRUE; for (; gIter != NULL; gIter = gIter->next) { GHashTableIter iter; node_t *node = NULL; resource_t *child_rsc = (resource_t *) gIter->data; g_hash_table_iter_init(&iter, child_rsc->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { if (can_run_resources(node) == FALSE) { /* This node will never be promoted to master, * so don't apply the master score as that may * lead to clone shuffling */ continue; } score = master_score(child_rsc, node, 0); if (score > 0) { new_score = merge_weights(node->weight, score); if (new_score != node->weight) { pe_rsc_trace(rsc, "\t%s: Updating preference for %s (%d->%d)", child_rsc->id, node->details->uname, node->weight, new_score); node->weight = new_score; } } new_score = max(child_rsc->priority, score); if (new_score != child_rsc->priority) { pe_rsc_trace(rsc, "\t%s: Updating priority (%d->%d)", child_rsc->id, child_rsc->priority, new_score); child_rsc->priority = new_score; } } } } static void set_role_slave(resource_t * rsc, gboolean current) { GListPtr gIter = rsc->children; if (current) { if (rsc->role == RSC_ROLE_STARTED) { rsc->role = RSC_ROLE_SLAVE; } } else { GListPtr allocated = NULL; rsc->fns->location(rsc, &allocated, FALSE); if (allocated) { rsc->next_role = RSC_ROLE_SLAVE; } else { rsc->next_role = RSC_ROLE_STOPPED; } g_list_free(allocated); } for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; set_role_slave(child_rsc, current); } } static void set_role_master(resource_t * rsc) { GListPtr gIter = rsc->children; if (rsc->next_role == RSC_ROLE_UNKNOWN) { rsc->next_role = RSC_ROLE_MASTER; } for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; set_role_master(child_rsc); } } node_t * master_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) { int promoted = 0; GListPtr gIter = NULL; GListPtr gIter2 = NULL; GHashTableIter iter; node_t *node = NULL; node_t *chosen = NULL; enum rsc_role_e next_role = RSC_ROLE_UNKNOWN; char score[33]; size_t len = sizeof(score); clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); if (is_not_set(rsc->flags, pe_rsc_provisional)) { return NULL; } else if (is_set(rsc->flags, pe_rsc_allocating)) { pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id); return NULL; } apply_master_prefs(rsc); clone_color(rsc, prefer, data_set); set_bit(rsc->flags, pe_rsc_allocating); /* count now tracks the number of masters allocated */ g_hash_table_iter_init(&iter, rsc->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { node->count = 0; } /* * assign priority */ for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { GListPtr list = NULL; resource_t *child_rsc = (resource_t *) gIter->data; pe_rsc_trace(rsc, "Assigning priority for %s: %s", child_rsc->id, role2text(child_rsc->next_role)); if (child_rsc->fns->state(child_rsc, TRUE) == RSC_ROLE_STARTED) { set_role_slave(child_rsc, TRUE); } chosen = child_rsc->fns->location(child_rsc, &list, FALSE); if (g_list_length(list) > 1) { crm_config_err("Cannot promote non-colocated child %s", child_rsc->id); } g_list_free(list); if (chosen == NULL) { continue; } next_role = child_rsc->fns->state(child_rsc, FALSE); switch (next_role) { case RSC_ROLE_STARTED: case RSC_ROLE_UNKNOWN: CRM_CHECK(chosen != NULL, break); /* * Default to -1 if no value is set * * This allows master locations to be specified * based solely on rsc_location constraints, * but prevents anyone from being promoted if * neither a constraint nor a master-score is present */ child_rsc->priority = master_score(child_rsc, chosen, -1); break; case RSC_ROLE_SLAVE: case RSC_ROLE_STOPPED: child_rsc->priority = -INFINITY; break; case RSC_ROLE_MASTER: /* We will arrive here if we're re-creating actions after a stonith */ break; default: CRM_CHECK(FALSE /* unhandled */ , crm_err("Unknown resource role: %d for %s", next_role, child_rsc->id)); } apply_master_location(child_rsc, child_rsc->rsc_location, chosen); apply_master_location(child_rsc, rsc->rsc_location, chosen); for (gIter2 = child_rsc->rsc_cons; gIter2 != NULL; gIter2 = gIter2->next) { rsc_colocation_t *cons = (rsc_colocation_t *) gIter2->data; child_rsc->cmds->rsc_colocation_lh(child_rsc, cons->rsc_rh, cons); } child_rsc->sort_index = child_rsc->priority; pe_rsc_trace(rsc, "Assigning priority for %s: %d", child_rsc->id, child_rsc->priority); if (next_role == RSC_ROLE_MASTER) { child_rsc->sort_index = INFINITY; } } dump_node_scores(LOG_DEBUG_3, rsc, "Pre merge", rsc->allowed_nodes); master_promotion_order(rsc, data_set); /* mark the first N as masters */ for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; score2char_stack(child_rsc->sort_index, score, len); chosen = child_rsc->fns->location(child_rsc, NULL, FALSE); if (show_scores) { fprintf(stdout, "%s promotion score on %s: %s\n", child_rsc->id, chosen ? chosen->details->uname : "none", score); } else { do_crm_log(scores_log_level, "%s promotion score on %s: %s", child_rsc->id, chosen ? chosen->details->uname : "none", score); } chosen = NULL; /* nuke 'chosen' so that we don't promote more than the * required number of instances */ if (child_rsc->sort_index < 0) { pe_rsc_trace(rsc, "Not supposed to promote child: %s", child_rsc->id); } else if (promoted < clone_data->master_max || is_not_set(rsc->flags, pe_rsc_managed)) { chosen = can_be_master(child_rsc); } pe_rsc_debug(rsc, "%s master score: %d", child_rsc->id, child_rsc->priority); if (chosen == NULL) { set_role_slave(child_rsc, FALSE); continue; } else if(child_rsc->role < RSC_ROLE_MASTER && is_set(data_set->flags, pe_flag_have_quorum) == FALSE && data_set->no_quorum_policy == no_quorum_freeze) { crm_notice("Resource %s cannot be elevated from %s to %s: no-quorum-policy=freeze", child_rsc->id, role2text(child_rsc->role), role2text(child_rsc->next_role)); set_role_slave(child_rsc, FALSE); continue; } chosen->count++; pe_rsc_info(rsc, "Promoting %s (%s %s)", child_rsc->id, role2text(child_rsc->role), chosen->details->uname); set_role_master(child_rsc); promoted++; } clone_data->masters_allocated = promoted; pe_rsc_info(rsc, "%s: Promoted %d instances of a possible %d to master", rsc->id, promoted, clone_data->master_max); clear_bit(rsc->flags, pe_rsc_provisional); clear_bit(rsc->flags, pe_rsc_allocating); return NULL; } void master_create_actions(resource_t * rsc, pe_working_set_t * data_set) { action_t *action = NULL; GListPtr gIter = rsc->children; action_t *action_complete = NULL; gboolean any_promoting = FALSE; gboolean any_demoting = FALSE; resource_t *last_promote_rsc = NULL; resource_t *last_demote_rsc = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); pe_rsc_debug(rsc, "Creating actions for %s", rsc->id); /* create actions as normal */ clone_create_actions(rsc, data_set); for (; gIter != NULL; gIter = gIter->next) { gboolean child_promoting = FALSE; gboolean child_demoting = FALSE; resource_t *child_rsc = (resource_t *) gIter->data; pe_rsc_trace(rsc, "Creating actions for %s", child_rsc->id); child_rsc->cmds->create_actions(child_rsc, data_set); master_update_pseudo_status(child_rsc, &child_demoting, &child_promoting); any_demoting = any_demoting || child_demoting; any_promoting = any_promoting || child_promoting; pe_rsc_trace(rsc, "Created actions for %s: %d %d", child_rsc->id, child_promoting, child_demoting); } /* promote */ action = create_pseudo_resource_op(rsc, RSC_PROMOTE, !any_promoting, TRUE, data_set); action_complete = create_pseudo_resource_op(rsc, RSC_PROMOTED, !any_promoting, TRUE, data_set); action_complete->priority = INFINITY; child_promoting_constraints(clone_data, pe_order_optional, rsc, NULL, last_promote_rsc, data_set); if (clone_data->promote_notify == NULL) { clone_data->promote_notify = create_notification_boundaries(rsc, RSC_PROMOTE, action, action_complete, data_set); } /* demote */ action = create_pseudo_resource_op(rsc, RSC_DEMOTE, !any_demoting, TRUE, data_set); action_complete = create_pseudo_resource_op(rsc, RSC_DEMOTED, !any_demoting, TRUE, data_set); action_complete->priority = INFINITY; child_demoting_constraints(clone_data, pe_order_optional, rsc, NULL, last_demote_rsc, data_set); if (clone_data->demote_notify == NULL) { clone_data->demote_notify = create_notification_boundaries(rsc, RSC_DEMOTE, action, action_complete, data_set); if (clone_data->promote_notify) { /* If we ever wanted groups to have notifications we'd need to move this to native_internal_constraints() one day * Requires exposing *_notify */ order_actions(clone_data->stop_notify->post_done, clone_data->promote_notify->pre, pe_order_optional); order_actions(clone_data->start_notify->post_done, clone_data->promote_notify->pre, pe_order_optional); order_actions(clone_data->demote_notify->post_done, clone_data->promote_notify->pre, pe_order_optional); order_actions(clone_data->demote_notify->post_done, clone_data->start_notify->pre, pe_order_optional); order_actions(clone_data->demote_notify->post_done, clone_data->stop_notify->pre, pe_order_optional); } } /* restore the correct priority */ gIter = rsc->children; for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; child_rsc->priority = rsc->priority; } } void master_promotion_constraints(resource_t * rsc, pe_working_set_t * data_set) { /* global stopped before start */ new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_START, pe_order_optional, data_set); /* global stopped before promote */ new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_PROMOTE, pe_order_optional, data_set); /* global demoted before start */ new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_START, pe_order_optional, data_set); /* global started before promote */ new_rsc_order(rsc, RSC_STARTED, rsc, RSC_PROMOTE, pe_order_optional, data_set); /* global demoted before stop */ new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_STOP, pe_order_optional, data_set); /* global demote before demoted */ new_rsc_order(rsc, RSC_DEMOTE, rsc, RSC_DEMOTED, pe_order_optional, data_set); /* global demoted before promote */ new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_PROMOTE, pe_order_optional, data_set); } void master_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) { GListPtr gIter = rsc->children; resource_t *last_rsc = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); clone_internal_constraints(rsc, data_set); master_promotion_constraints(rsc, data_set); for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; /* child demote before promote */ new_rsc_order(child_rsc, RSC_DEMOTE, child_rsc, RSC_PROMOTE, pe_order_optional, data_set); child_promoting_constraints(clone_data, pe_order_optional, rsc, child_rsc, last_rsc, data_set); child_demoting_constraints(clone_data, pe_order_optional, rsc, child_rsc, last_rsc, data_set); last_rsc = child_rsc; } } static void node_hash_update_one(GHashTable * hash, node_t * other, const char *attr, int score) { GHashTableIter iter; node_t *node = NULL; const char *value = NULL; if (other == NULL) { return; } else if (attr == NULL) { attr = "#" XML_ATTR_UNAME; } value = g_hash_table_lookup(other->details->attrs, attr); g_hash_table_iter_init(&iter, hash); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { const char *tmp = g_hash_table_lookup(node->details->attrs, attr); if (safe_str_eq(value, tmp)) { crm_trace("%s: %d + %d", node->details->uname, node->weight, other->weight); node->weight = merge_weights(node->weight, score); } } } void master_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint) { GListPtr gIter = NULL; CRM_CHECK(rsc_rh != NULL, return); if (is_set(rsc_rh->flags, pe_rsc_provisional)) { return; } else if (constraint->role_rh == RSC_ROLE_UNKNOWN) { pe_rsc_trace(rsc_rh, "Handling %s as a clone colocation", constraint->id); clone_rsc_colocation_rh(rsc_lh, rsc_rh, constraint); return; } CRM_CHECK(rsc_lh != NULL, return); CRM_CHECK(rsc_lh->variant == pe_native, return); pe_rsc_trace(rsc_rh, "Processing constraint %s: %d", constraint->id, constraint->score); if (constraint->role_rh == RSC_ROLE_UNKNOWN) { gIter = rsc_rh->children; for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint); } } else if (is_set(rsc_lh->flags, pe_rsc_provisional)) { GListPtr rhs = NULL; gIter = rsc_rh->children; for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE); enum rsc_role_e next_role = child_rsc->fns->state(child_rsc, FALSE); pe_rsc_trace(rsc_rh, "Processing: %s", child_rsc->id); if (chosen != NULL && next_role == constraint->role_rh) { pe_rsc_trace(rsc_rh, "Applying: %s %s %s %d", child_rsc->id, role2text(next_role), chosen->details->uname, constraint->score); if (constraint->score < INFINITY) { node_hash_update_one(rsc_lh->allowed_nodes, chosen, constraint->node_attribute, constraint->score); } rhs = g_list_prepend(rhs, chosen); } } /* Only do this if it's not a master-master colocation * Doing this unconditionally would prevent the slaves from being started */ if (constraint->role_lh != RSC_ROLE_MASTER || constraint->role_rh != RSC_ROLE_MASTER) { if (constraint->score >= INFINITY) { node_list_exclude(rsc_lh->allowed_nodes, rhs, TRUE); } } g_list_free(rhs); } else if (constraint->role_lh == RSC_ROLE_MASTER) { resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, constraint->role_rh, FALSE); if (rh_child == NULL && constraint->score >= INFINITY) { pe_rsc_trace(rsc_lh, "%s can't be promoted %s", rsc_lh->id, constraint->id); rsc_lh->priority = -INFINITY; } else if (rh_child != NULL) { int new_priority = merge_weights(rsc_lh->priority, constraint->score); pe_rsc_debug(rsc_lh, "Applying %s to %s", constraint->id, rsc_lh->id); pe_rsc_debug(rsc_lh, "\t%s: %d->%d", rsc_lh->id, rsc_lh->priority, new_priority); rsc_lh->priority = new_priority; } } return; } void master_append_meta(resource_t * rsc, xmlNode * xml) { char *name = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); clone_append_meta(rsc, xml); name = crm_meta_name(XML_RSC_ATTR_MASTER_MAX); crm_xml_add_int(xml, name, clone_data->master_max); free(name); name = crm_meta_name(XML_RSC_ATTR_MASTER_NODEMAX); crm_xml_add_int(xml, name, clone_data->master_node_max); free(name); } diff --git a/pengine/regression.core.sh.in b/pengine/regression.core.sh.in index b60913d183..731844f66a 100644 --- a/pengine/regression.core.sh.in +++ b/pengine/regression.core.sh.in @@ -1,325 +1,324 @@ # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # verbose=0 num_failed=0 num_tests=0 force_local=0 VALGRIND_CMD="" diff_opts="--ignore-all-space --ignore-blank-lines -u -N" if [ "x$0" = "x-bash" ]; then test_home=@abs_top_builddir@/pengine else test_home=`dirname $0` test_name=`basename $0` fi function info() { printf "$*\n" } function error() { printf " * ERROR: $*\n" } function failed() { printf " * FAILED: $*\n" } function show_test() { name=$1; shift printf " Test %-25s $*\n" "$name:" } info "Test home is:\t$test_home" io_dir=$test_home/test10 test_binary=@abs_top_builddir@/tools/crm_simulate PCMK_schema_directory=@abs_top_builddir@/xml failed=$test_home/.regression.failed.diff single_test= while true ; do case "$1" in -V|--verbose) verbose=1; shift;; -v|--valgrind) export G_SLICE=always-malloc VALGRIND_CMD="valgrind -q --gen-suppressions=all --log-file=%q{valgrind_output} --show-reachable=no --leak-check=full --trace-children=no --time-stamp=yes --num-callers=20 --suppressions=@datadir@/@PACKAGE@/tests/valgrind-pcmk.suppressions" test_binary= shift;; --valgrind-dhat) VALGRIND_CMD="valgrind --log-file=%q{valgrind_output} --show-top-n=100 --num-callers=4 --time-stamp=yes --trace-children=no --tool=exp-dhat" test_binary= shift;; --valgrind-skip-output) VALGRIND_SKIP_OUTPUT=1 shift;; --run) single_test="$2"; shift; shift; break;; -b|--binary) test_binary=$2; PCMK_schema_directory=""; shift; shift;; -i|--io-dir) io_dir=$2; shift; shift;; -?|--help) echo "$0 [--binary name] [--force-local]"; shift; exit 0;; --) shift ; break ;; "") break;; *) echo "unknown option: $1"; exit 1;; esac done if [ "x$test_binary" = "x" ]; then test_binary=@sbindir@/crm_simulate PCMK_schema_directory=@CRM_DTD_DIRECTORY@ elif [ ! -x $test_binary ]; then test_binary=@sbindir@/crm_simulate PCMK_schema_directory=@CRM_DTD_DIRECTORY@ fi export PCMK_schema_directory if [ "x$test_binary" = "x" ]; then info "crm_simulate not installed. Aborting." exit 1 fi info "Test binary is:\t$test_binary" info "Schema home is:\t$PCMK_schema_directory" if [ "x$VALGRIND_CMD" != "x" ]; then info "Activating memory testing with valgrind"; fi info " " test_cmd="$VALGRIND_CMD $test_binary" #echo $test_cmd if [ `whoami` != root ]; then declare -x CIB_shadow_dir=/tmp fi function do_test { did_fail=0 expected_rc=0 num_tests=`expr $num_tests + 1` base=$1; shift name=$1; shift input=$io_dir/${base}.xml output=$io_dir/${base}.out expected=$io_dir/${base}.exp dot_png=$io_dir/${base}.png dot_expected=$io_dir/${base}.dot dot_output=$io_dir/${base}.pe.dot scores=$io_dir/${base}.scores score_output=$io_dir/${base}.scores.pe stderr_expected=$io_dir/${base}.stderr stderr_output=$io_dir/${base}.stderr.pe summary=$io_dir/${base}.summary summary_output=$io_dir/${base}.summary.pe valgrind_output=$io_dir/${base}.valgrind export valgrind_output if [ "x$1" = "x--rc" ]; then expected_rc=$2 shift; shift; fi show_test "$base" "$name" if [ ! -f $input ]; then error "No input"; did_fail=1 num_failed=`expr $num_failed + $did_fail` return; fi if [ "$create_mode" != "true" -a ! -f $expected ]; then error "no stored output"; # return; fi # ../admin/crm_verify -X $input if [ ! -z $single_test ]; then echo CIB_shadow_dir=$io_dir $test_cmd -x $input -D $dot_output -G $output -S $* CIB_shadow_dir=$io_dir $test_cmd -x $input -D $dot_output -G $output -S $* 2>&1 | tee $summary_output else CIB_shadow_dir=$io_dir $test_cmd -x $input -S &> $summary_output fi CIB_shadow_dir=$io_dir $test_cmd -x $input -D $dot_output -G $output -SQ -s $* 2> $stderr_output > $score_output rc=$? if [ $rc != $expected_rc ]; then failed "Test returned: $rc"; did_fail=1 echo "CIB_shadow_dir=$io_dir $test_cmd -x $input -D $dot_output -G $output -SQ -s $*" fi if [ -z "$VALGRIND_SKIP_OUTPUT" ]; then if [ -s ${valgrind_output} ]; then error "Valgrind reported errors"; did_fail=1 cat ${valgrind_output} fi rm -f ${valgrind_output} fi if [ -s core ]; then error "Core-file detected: core.${base}"; did_fail=1 rm -f $test_home/core.$base mv core $test_home/core.$base fi if [ -e $stderr_expected ]; then diff $diff_opts $stderr_expected $stderr_output >/dev/null rc2=$? if [ $rc2 != 0 ]; then failed "stderr changed"; diff $diff_opts $stderr_expected $stderr_output 2>/dev/null >> $failed echo "" >> $failed did_fail=1 fi elif [ -s $stderr_output ]; then error "Output was written to stderr" did_fail=1 cat $stderr_output fi rm -f $stderr_output if [ ! -s $output ]; then error "No graph produced"; did_fail=1 num_failed=`expr $num_failed + $did_fail` rm -f $output return; # else # mv $output $output.sed # cat $output.sed | sed 's/id=.[0-9]*.\ //g' >> $output fi if [ ! -s $dot_output ]; then error "No dot-file summary produced"; did_fail=1 num_failed=`expr $num_failed + $did_fail` rm -f $output return; else echo "digraph \"g\" {" > $dot_output.sort LC_ALL=POSIX sort -u $dot_output | grep -v -e ^}$ -e digraph >> $dot_output.sort echo "}" >> $dot_output.sort mv -f $dot_output.sort $dot_output fi if [ ! -s $score_output ]; then error "No allocation scores produced"; did_fail=1 num_failed=`expr $num_failed + $did_fail` rm $output return; else LC_ALL=POSIX sort $score_output > $score_output.sorted mv -f $score_output.sorted $score_output fi if [ "$create_mode" = "true" ]; then cp "$output" "$expected" cp "$dot_output" "$dot_expected" cp "$score_output" "$scores" cp "$summary_output" "$summary" info " Created expected outputs" fi - sed -i -e '/meta-data/d' $summary_output diff $diff_opts $summary $summary_output >/dev/null rc2=$? if [ $rc2 != 0 ]; then failed "summary changed"; diff $diff_opts $summary $summary_output 2>/dev/null >> $failed echo "" >> $failed did_fail=1 fi diff $diff_opts $dot_expected $dot_output >/dev/null rc=$? if [ $rc != 0 ]; then failed "dot-file summary changed"; diff $diff_opts $dot_expected $dot_output 2>/dev/null >> $failed echo "" >> $failed did_fail=1 else rm -f $dot_output fi sed -i -e 's/crm_feature_set="[^"]*"//' -e 's/batch-limit="[0-9]*"//' $expected $output diff $diff_opts $expected $output >/dev/null rc2=$? if [ $rc2 != 0 ]; then failed "xml-file changed"; diff $diff_opts $expected $output 2>/dev/null >> $failed echo "" >> $failed did_fail=1 fi diff $diff_opts $scores $score_output >/dev/null rc=$? if [ $rc != 0 ]; then failed "scores-file changed"; diff $diff_opts $scores $score_output 2>/dev/null >> $failed echo "" >> $failed did_fail=1 fi rm -f $output $score_output $summary_output num_failed=`expr $num_failed + $did_fail` } function test_results { if [ $num_failed != 0 ]; then if [ -s $failed ]; then if [ "$verbose" = "1" ]; then error "Results of $num_failed failed tests (out of $num_tests)...." cat $failed else error "Results of $num_failed failed tests (out of $num_tests) are in $failed...." error "Use $0 -V to display them automatically." fi else error "$num_failed (of $num_tests) tests failed (no diff results)" rm $failed fi fi exit $num_failed } if [ -z $single_test ]; then # zero out the error log > $failed else failed=.single > $failed do_test $single_test "Single shot" $* cat $failed exit $? fi diff --git a/tools/regression.sh b/tools/regression.sh index 581f571a21..2c48a7a507 100755 --- a/tools/regression.sh +++ b/tools/regression.sh @@ -1,812 +1,811 @@ #!/bin/bash : ${shadow=tools-regression} test_home=`dirname $0` num_errors=0 num_passed=0 GREP_OPTIONS= verbose=0 tests="dates tools acls validity" function test_assert() { target=$1; shift cib=$1; shift app=`echo "$cmd" | sed 's/\ .*//'` printf "* Running: $app - $desc\n" 1>&2 printf "=#=#=#= Begin test: $desc =#=#=#=\n" eval $VALGRIND_CMD $cmd 2>&1 rc=$? if [ x$cib != x0 ]; then printf "=#=#=#= Current cib after: $desc =#=#=#=\n" CIB_user=root cibadmin -Q fi printf "=#=#=#= End test: $desc - `crm_error $rc` ($rc) =#=#=#=\n" if [ $rc -ne $target ]; then num_errors=`expr $num_errors + 1` printf "* Failed (rc=%.3d): %-14s - %s\n" $rc $app "$desc" printf "* Failed (rc=%.3d): %-14s - %s\n" $rc $app "$desc (`which $app`)" 1>&2 return exit 1 else printf "* Passed: %-14s - %s\n" $app "$desc" num_passed=`expr $num_passed + 1` fi } function usage() { echo "Usage: ./regression.sh [-s(ave)] [-x] [-v(erbose)]" exit $1 } done=0 do_save=0 VALGRIND_CMD= while test "$done" = "0"; do case "$1" in -t) tests=$2; shift; shift;; -V|--verbose) verbose=1; shift;; -v|--valgrind) export G_SLICE=always-malloc VALGRIND_CMD="valgrind -q --gen-suppressions=all --show-reachable=no --leak-check=full --trace-children=no --time-stamp=yes --num-callers=20 --suppressions=/usr/share/pacemaker/tests/valgrind-pcmk.suppressions" shift;; -x) set -x; shift;; -s) do_save=1; shift;; -p) PATH="$2:$PATH"; export PATH; shift 1;; -?) usage 0;; -*) echo "unknown option: $1"; usage 1;; *) done=1;; esac done if [ "x$VALGRIND_CMD" = "x" -a -x $test_home/crm_simulate ]; then xml_home=`dirname ${test_home}` echo "Using local binaries from: $test_home, schemas from $xml_home" export PATH="$test_home:$PATH" export PCMK_schema_directory=${xml_home}/xml fi function test_tools() { export CIB_shadow_dir=$test_home $VALGRIND_CMD crm_shadow --batch --force --create-empty $shadow 2>&1 export CIB_shadow=$shadow desc="Validate CIB" cmd="cibadmin -Q" test_assert 0 desc="Configure something before erasing" cmd="crm_attribute -n cluster-delay -v 60s" test_assert 0 desc="Require --force for CIB erasure" cmd="cibadmin -E" test_assert 22 desc="Allow CIB erasure with --force" cmd="cibadmin -E --force" test_assert 0 desc="Query CIB" cmd="cibadmin -Q > /tmp/$$.existing.xml" test_assert 0 desc="Set cluster option" cmd="crm_attribute -n cluster-delay -v 60s" test_assert 0 desc="Query new cluster option" cmd="cibadmin -Q -o crm_config | grep cib-bootstrap-options-cluster-delay" test_assert 0 desc="Query cluster options" cmd="cibadmin -Q -o crm_config > /tmp/$$.opt.xml" test_assert 0 desc="Set no-quorum policy" cmd="crm_attribute -n no-quorum-policy -v ignore" test_assert 0 desc="Delete nvpair" cmd="cibadmin -D -o crm_config --xml-text ''" test_assert 0 desc="Create operaton should fail" cmd="cibadmin -C -o crm_config --xml-file /tmp/$$.opt.xml" test_assert 76 desc="Modify cluster options section" cmd="cibadmin -M -o crm_config --xml-file /tmp/$$.opt.xml" test_assert 0 desc="Query updated cluster option" cmd="cibadmin -Q -o crm_config | grep cib-bootstrap-options-cluster-delay" test_assert 0 desc="Set duplicate cluster option" cmd="crm_attribute -n cluster-delay -v 40s -s duplicate" test_assert 0 desc="Setting multiply defined cluster option should fail" cmd="crm_attribute -n cluster-delay -v 30s" test_assert 76 desc="Set cluster option with -s" cmd="crm_attribute -n cluster-delay -v 30s -s duplicate" test_assert 0 desc="Delete cluster option with -i" cmd="crm_attribute -n cluster-delay -D -i cib-bootstrap-options-cluster-delay" test_assert 0 desc="Create node1 and bring it online" cmd="crm_simulate --live-check --in-place --node-up=node1" test_assert 0 desc="Create node attribute" cmd="crm_attribute -n ram -v 1024M -U node1 -t nodes" test_assert 0 desc="Query new node attribute" cmd="cibadmin -Q -o nodes | grep node1-ram" test_assert 0 desc="Set a transient (fail-count) node attribute" cmd="crm_attribute -n fail-count-foo -v 3 -N node1 -t status" test_assert 0 desc="Query a fail count" cmd="crm_failcount --query -r foo -N node1" test_assert 0 desc="Delete a transient (fail-count) node attribute" cmd="crm_attribute -n fail-count-foo -D -N node1 -t status" test_assert 0 desc="Digest calculation" cmd="cibadmin -Q | cibadmin -5 -p 2>&1 > /dev/null" test_assert 0 # This update will fail because it has version numbers desc="Replace operation should fail" cmd="cibadmin -R --xml-file /tmp/$$.existing.xml" test_assert 205 desc="Default standby value" cmd="crm_standby -N node1 -G" test_assert 0 desc="Set standby status" cmd="crm_standby -N node1 -v true" test_assert 0 desc="Query standby value" cmd="crm_standby -N node1 -G" test_assert 0 desc="Delete standby value" cmd="crm_standby -N node1 -D" test_assert 0 desc="Create a resource" cmd="cibadmin -C -o resources --xml-text ''" test_assert 0 desc="Create a resource meta attribute" cmd="crm_resource -r dummy --meta -p is-managed -v false" test_assert 0 desc="Query a resource meta attribute" cmd="crm_resource -r dummy --meta -g is-managed" test_assert 0 desc="Remove a resource meta attribute" cmd="crm_resource -r dummy --meta -d is-managed" test_assert 0 desc="Create a resource attribute" cmd="crm_resource -r dummy -p delay -v 10s" test_assert 0 desc="List the configured resources" cmd="crm_resource -L" test_assert 0 desc="Require a destination when migrating a resource that is stopped" cmd="crm_resource -r dummy -M" test_assert 22 desc="Don't support migration to non-existent locations" cmd="crm_resource -r dummy -M -N i.dont.exist" test_assert 6 desc="Create a fencing resource" cmd="cibadmin -C -o resources --xml-text ''" test_assert 0 desc="Bring resources online" cmd="crm_simulate --live-check --in-place -S" test_assert 0 desc="Try to move a resource to its existing location" cmd="crm_resource -r dummy --move --host node1" test_assert 22 desc="Move a resource from its existing location" cmd="crm_resource -r dummy --move" test_assert 0 desc="Clear out constraints generated by --move" cmd="crm_resource -r dummy --clear" test_assert 0 desc="Default ticket granted state" cmd="crm_ticket -t ticketA -G granted -d false" test_assert 0 desc="Set ticket granted state" cmd="crm_ticket -t ticketA -r --force" test_assert 0 desc="Query ticket granted state" cmd="crm_ticket -t ticketA -G granted" test_assert 0 desc="Delete ticket granted state" cmd="crm_ticket -t ticketA -D granted --force" test_assert 0 desc="Make a ticket standby" cmd="crm_ticket -t ticketA -s" test_assert 0 desc="Query ticket standby state" cmd="crm_ticket -t ticketA -G standby" test_assert 0 desc="Activate a ticket" cmd="crm_ticket -t ticketA -a" test_assert 0 desc="Delete ticket standby state" cmd="crm_ticket -t ticketA -D standby" test_assert 0 desc="Ban a resource on unknown node" cmd="crm_resource -r dummy -B -N host1" test_assert 6 desc="Create two more nodes and bring them online" cmd="crm_simulate --live-check --in-place --node-up=node2 --node-up=node3" test_assert 0 desc="Ban dummy from node1" cmd="crm_resource -r dummy -B -N node1" test_assert 0 desc="Ban dummy from node2" cmd="crm_resource -r dummy -B -N node2" test_assert 0 desc="Relocate resources due to ban" cmd="crm_simulate --live-check --in-place -S" test_assert 0 desc="Move dummy to node1" cmd="crm_resource -r dummy -M -N node1" test_assert 0 desc="Clear implicit constraints for dummy on node2" cmd="crm_resource -r dummy -U -N node2" test_assert 0 desc="Drop the status section" cmd="cibadmin -R -o status --xml-text ''" test_assert 0 0 desc="Create a clone" cmd="cibadmin -C -o resources --xml-text ''" test_assert 0 0 desc="Create a resource meta attribute" cmd="crm_resource -r test-primitive --meta -p is-managed -v false" test_assert 0 desc="Create a resource meta attribute in the primitive" cmd="crm_resource -r test-primitive --meta -p is-managed -v false --force" test_assert 0 desc="Update resource meta attribute with duplicates" cmd="crm_resource -r test-clone --meta -p is-managed -v true" test_assert 0 desc="Update resource meta attribute with duplicates (force clone)" cmd="crm_resource -r test-clone --meta -p is-managed -v true --force" test_assert 0 desc="Update child resource meta attribute with duplicates" cmd="crm_resource -r test-primitive --meta -p is-managed -v false" test_assert 0 desc="Delete resource meta attribute with duplicates" cmd="crm_resource -r test-clone --meta -d is-managed" test_assert 0 desc="Delete resource meta attribute in parent" cmd="crm_resource -r test-primitive --meta -d is-managed" test_assert 0 desc="Create a resource meta attribute in the primitive" cmd="crm_resource -r test-primitive --meta -p is-managed -v false --force" test_assert 0 desc="Update existing resource meta attribute" cmd="crm_resource -r test-clone --meta -p is-managed -v true" test_assert 0 desc="Create a resource meta attribute in the parent" cmd="crm_resource -r test-clone --meta -p is-managed -v true --force" test_assert 0 desc="Copy resources" cmd="cibadmin -Q -o resources > /tmp/$$.resources.xml" test_assert 0 0 desc="Delete resource paremt meta attribute (force)" cmd="crm_resource -r test-clone --meta -d is-managed --force" test_assert 0 desc="Restore duplicates" cmd="cibadmin -R -o resources --xml-file /tmp/$$.resources.xml" test_assert 0 desc="Delete resource child meta attribute" cmd="crm_resource -r test-primitive --meta -d is-managed" test_assert 0 rm -f /tmp/$$.existing.xml /tmp/$$.resources.xml } function test_dates() { desc="2014-01-01 00:30:00 - 1 Hour" cmd="iso8601 -d '2014-01-01 00:30:00Z' -D P-1H -E '2013-12-31 23:30:00Z'" test_assert 0 0 for y in 06 07 08 09 10 11 12 13 14 15 16 17 18; do desc="20$y-W01-7" cmd="iso8601 -d '20$y-W01-7 00Z'" test_assert 0 0 desc="20$y-W01-7 - round-trip" cmd="iso8601 -d '20$y-W01-7 00Z' -W -E '20$y-W01-7 00:00:00Z'" test_assert 0 0 desc="20$y-W01-1" cmd="iso8601 -d '20$y-W01-1 00Z'" test_assert 0 0 desc="20$y-W01-1 - round-trip" cmd="iso8601 -d '20$y-W01-1 00Z' -W -E '20$y-W01-1 00:00:00Z'" test_assert 0 0 done desc="2009-W53-07" cmd="iso8601 -d '2009-W53-7 00:00:00Z' -W -E '2009-W53-7 00:00:00Z'" test_assert 0 0 desc="2009-01-31 + 1 Month" cmd="iso8601 -d '2009-01-31 00:00:00Z' -D P1M -E '2009-02-28 00:00:00Z'" test_assert 0 0 desc="2009-01-31 + 2 Months" cmd="iso8601 -d '2009-01-31 00:00:00Z' -D P2M -E '2009-03-31 00:00:00Z'" test_assert 0 0 desc="2009-01-31 + 3 Months" cmd="iso8601 -d '2009-01-31 00:00:00Z' -D P3M -E '2009-04-30 00:00:00Z'" test_assert 0 0 desc="2009-03-31 - 1 Month" cmd="iso8601 -d '2009-03-31 00:00:00Z' -D P-1M -E '2009-02-28 00:00:00Z'" test_assert 0 0 } function get_epoch() { CIB_user=root CIB_file=$1 CIB_shadow="" cibadmin -Q | head -n 1 | sed -e 's/.* epoch=\"\([0-9]*\).*/\1/' } function restore_epoch() { infile=$1; shift old=$1; shift new=$(get_epoch $infile) sed -i 's/epoch=.$old/epoch=\"$new/g' $infile } function test_acl_loop() { # Make sure we're rejecting things for the right reasons export PCMK_trace_functions=__xml_acl_check,__xml_acl_post_process export PCMK_stderr=1 CIB_user=root cibadmin --replace --xml-text '' export CIB_user=unknownguy desc="$CIB_user: Query configuration" cmd="cibadmin -Q" test_assert 13 0 desc="$CIB_user: Set enable-acl" cmd="crm_attribute -n enable-acl -v false" test_assert 13 0 desc="$CIB_user: Set stonith-enabled" cmd="crm_attribute -n stonith-enabled -v false" test_assert 13 0 desc="$CIB_user: Create a resource" cmd="cibadmin -C -o resources --xml-text ''" test_assert 13 0 export CIB_user=l33t-haxor desc="$CIB_user: Query configuration" cmd="cibadmin -Q" test_assert 13 0 desc="$CIB_user: Set enable-acl" cmd="crm_attribute -n enable-acl -v false" test_assert 13 0 desc="$CIB_user: Set stonith-enabled" cmd="crm_attribute -n stonith-enabled -v false" test_assert 13 0 desc="$CIB_user: Create a resource" cmd="cibadmin -C -o resources --xml-text ''" test_assert 13 0 export CIB_user=niceguy desc="$CIB_user: Query configuration" cmd="cibadmin -Q" test_assert 0 0 desc="$CIB_user: Set enable-acl" cmd="crm_attribute -n enable-acl -v false" test_assert 13 0 desc="$CIB_user: Set stonith-enabled" cmd="crm_attribute -n stonith-enabled -v false" test_assert 0 desc="$CIB_user: Create a resource" cmd="cibadmin -C -o resources --xml-text ''" test_assert 13 0 export CIB_user=root desc="$CIB_user: Query configuration" cmd="cibadmin -Q" test_assert 0 0 desc="$CIB_user: Set stonith-enabled" cmd="crm_attribute -n stonith-enabled -v true" test_assert 0 desc="$CIB_user: Create a resource" cmd="cibadmin -C -o resources --xml-text ''" test_assert 0 export CIB_user=l33t-haxor desc="$CIB_user: Create a resource meta attribute" cmd="crm_resource -r dummy --meta -p target-role -v Stopped" test_assert 13 0 desc="$CIB_user: Query a resource meta attribute" cmd="crm_resource -r dummy --meta -g target-role" test_assert 13 0 desc="$CIB_user: Remove a resource meta attribute" cmd="crm_resource -r dummy --meta -d target-role" test_assert 13 0 export CIB_user=niceguy desc="$CIB_user: Create a resource meta attribute" cmd="crm_resource -r dummy --meta -p target-role -v Stopped" test_assert 0 desc="$CIB_user: Query a resource meta attribute" cmd="crm_resource -r dummy --meta -g target-role" test_assert 0 desc="$CIB_user: Remove a resource meta attribute" cmd="crm_resource -r dummy --meta -d target-role" test_assert 0 desc="$CIB_user: Create a resource meta attribute" cmd="crm_resource -r dummy --meta -p target-role -v Started" test_assert 0 export CIB_user=badidea desc="$CIB_user: Query configuration - implied deny" cmd="cibadmin -Q" test_assert 0 0 export CIB_user=betteridea desc="$CIB_user: Query configuration - explicit deny" cmd="cibadmin -Q" test_assert 0 0 CIB_user=root cibadmin -Q > /tmp/$$.haxor.xml CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin --delete --xml-text '' CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -Ql export CIB_user=niceguy desc="$CIB_user: Replace - remove acls" cmd="cibadmin --replace --xml-file /tmp/$$.haxor.xml" test_assert 13 0 CIB_user=root cibadmin -Q > /tmp/$$.haxor.xml CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -C -o resources --xml-text '' CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -Ql desc="$CIB_user: Replace - create resource" cmd="cibadmin --replace --xml-file /tmp/$$.haxor.xml" test_assert 13 0 CIB_user=root cibadmin -Q > /tmp/$$.haxor.xml CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" crm_attribute -n enable-acl -v false CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -Ql desc="$CIB_user: Replace - modify attribute (deny)" cmd="cibadmin --replace --xml-file /tmp/$$.haxor.xml" test_assert 13 0 CIB_user=root cibadmin -Q > /tmp/$$.haxor.xml CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin --replace --xml-text '' CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -Ql desc="$CIB_user: Replace - delete attribute (deny)" cmd="cibadmin --replace --xml-file /tmp/$$.haxor.xml" test_assert 13 0 CIB_user=root cibadmin -Q > /tmp/$$.haxor.xml CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin --modify --xml-text '' CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -Ql desc="$CIB_user: Replace - create attribute (deny)" cmd="cibadmin --replace --xml-file /tmp/$$.haxor.xml" test_assert 13 0 rm -rf /tmp/$$.haxor.xml CIB_user=bob CIB_user=root cibadmin -Q > /tmp/$$.haxor.xml CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin --modify --xml-text '' CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -Ql desc="$CIB_user: Replace - create attribute (allow)" cmd="cibadmin --replace -o resources --xml-file /tmp/$$.haxor.xml" test_assert 0 0 CIB_user=root cibadmin -Q > /tmp/$$.haxor.xml CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin --modify --xml-text '' CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -Ql desc="$CIB_user: Replace - modify attribute (allow)" cmd="cibadmin --replace -o resources --xml-file /tmp/$$.haxor.xml" test_assert 0 0 CIB_user=root cibadmin -Q > /tmp/$$.haxor.xml CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin --replace -o resources --xml-text '' CIB_user=root CIB_file=/tmp/$$.haxor.xml CIB_shadow="" cibadmin -Ql desc="$CIB_user: Replace - delete attribute (allow)" cmd="cibadmin --replace -o resources --xml-file /tmp/$$.haxor.xml" test_assert 0 0 } function test_acls() { export CIB_shadow_dir=$test_home $VALGRIND_CMD crm_shadow --batch --force --create-empty $shadow --validate-with pacemaker-1.3 2>&1 export CIB_shadow=$shadow cat </tmp/$$.acls.xml EOF desc="Configure some ACLs" cmd="cibadmin -M -o acls --xml-file /tmp/$$.acls.xml" test_assert 0 desc="Enable ACLs" cmd="crm_attribute -n enable-acl -v true" test_assert 0 desc="Set cluster option" cmd="crm_attribute -n no-quorum-policy -v ignore" test_assert 0 desc="New ACL" cmd="cibadmin --create -o acls --xml-text ''" test_assert 0 desc="Another ACL" cmd="cibadmin --create -o acls --xml-text ''" test_assert 0 desc="Updated ACL" cmd="cibadmin --replace -o acls --xml-text ''" test_assert 0 test_acl_loop printf "\n\n !#!#!#!#! Upgrading to pacemaker-2.0 and retesting !#!#!#!#!\n" printf "\nUpgrading to pacemaker-2.0 and re-testing\n" 1>&2 export CIB_user=root desc="$CIB_user: Upgrade to pacemaker-2.0" cmd="cibadmin --upgrade --force -V" test_assert 0 sed -i 's/epoch=.2/epoch=\"6/g' $CIB_shadow_dir/shadow.$CIB_shadow sed -i 's/admin_epoch=.1/admin_epoch=\"0/g' $CIB_shadow_dir/shadow.$CIB_shadow test_acl_loop } function test_validity() { export CIB_shadow_dir=$test_home $VALGRIND_CMD crm_shadow --batch --force --create-empty $shadow --validate-with pacemaker-1.2 2>&1 export CIB_shadow=$shadow export PCMK_trace_functions=update_validation,cli_config_update export PCMK_stderr=1 cibadmin -C -o resources --xml-text '' cibadmin -C -o resources --xml-text '' cibadmin -C -o constraints --xml-text '' cibadmin -Q > /tmp/$$.good-1.2.xml desc="Try to make resulting CIB invalid (enum violation)" cmd="cibadmin -M -o constraints --xml-text ''" test_assert 203 sed 's|"start"|"break"|' /tmp/$$.good-1.2.xml > /tmp/$$.bad-1.2.xml desc="Run crm_simulate with invalid CIB (enum violation)" cmd="crm_simulate -x /tmp/$$.bad-1.2.xml -S" test_assert 126 0 desc="Try to make resulting CIB invalid (unrecognized validate-with)" cmd="cibadmin -M --xml-text ''" test_assert 203 sed 's|"pacemaker-1.2"|"pacemaker-9999.0"|' /tmp/$$.good-1.2.xml > /tmp/$$.bad-1.2.xml desc="Run crm_simulate with invalid CIB (unrecognized validate-with)" cmd="crm_simulate -x /tmp/$$.bad-1.2.xml -S" test_assert 126 0 desc="Try to make resulting CIB invalid, but possibly recoverable (valid with X.Y+1)" cmd="cibadmin -C -o configuration --xml-text ''" test_assert 203 sed 's||\0|' /tmp/$$.good-1.2.xml > /tmp/$$.bad-1.2.xml desc="Run crm_simulate with invalid, but possibly recoverable CIB (valid with X.Y+1)" cmd="crm_simulate -x /tmp/$$.bad-1.2.xml -S" test_assert 0 0 sed 's|\s\s*validate-with="[^"]*"||' /tmp/$$.good-1.2.xml > /tmp/$$.bad-1.2.xml desc="Make resulting CIB valid, although without validate-with attribute" cmd="cibadmin -R --xml-file /tmp/$$.bad-1.2.xml" test_assert 0 desc="Run crm_simulate with valid CIB, but without validate-with attribute" cmd="crm_simulate -x /tmp/$$.bad-1.2.xml -S" test_assert 0 0 # this will just disable validation and accept the config, outputting # validation errors sed -e 's|\s\s*validate-with="[^"]*"||' \ -e 's|\(\s\s*epoch="[^"]*\)"|\10"|' -e 's|"start"|"break"|' \ /tmp/$$.good-1.2.xml > /tmp/$$.bad-1.2.xml desc="Make resulting CIB invalid, and without validate-with attribute" cmd="cibadmin -R --xml-file /tmp/$$.bad-1.2.xml" test_assert 0 desc="Run crm_simulate with invalid CIB, also without validate-with attribute" cmd="crm_simulate -x /tmp/$$.bad-1.2.xml -S" test_assert 0 0 rm -f /tmp/$$.good-1.2.xml /tmp/$$.bad-1.2.xml } for t in $tests; do echo "Testing $t" test_$t > $test_home/regression.$t.out sed -i -e 's/cib-last-written.*>/>/'\ -e 's/ last-run=\"[0-9]*\"//'\ -e 's/crm_feature_set="[^"]*" //'\ -e 's/validate-with="[^"]*" //'\ -e 's/Created new pacemaker-.* configuration/Created new pacemaker configuration/'\ -e 's/.*__xml_acl_check/__xml_acl_check/g'\ -e 's/.*__xml_acl_post_process/__xml_acl_post_process/g'\ -e 's/.*error: unpack_resources:/error: unpack_resources:/g'\ -e 's/ last-rc-change=\"[0-9]*\"//'\ -e 's|^/tmp/[0-9][0-9]*\.||'\ -e 's/^Entity: line [0-9][0-9]*: //'\ -e 's/schemas\.c:\([0-9][0-9]*\)/schemas.c:NNN/' \ -e 's/\(validation ([0-9][0-9]* of \)[0-9][0-9]*\().*\)/\1X\2/' \ - -e '/meta-data/d' \ $test_home/regression.$t.out if [ $do_save = 1 ]; then cp $test_home/regression.$t.out $test_home/regression.$t.exp fi done failed=0 echo -e "\n\nResults" for t in $tests; do if [ $do_save = 1 ]; then cp $test_home/regression.$t.out $test_home/regression.$t.exp fi if [ $verbose = 1 ]; then diff -wu $test_home/regression.$t.exp $test_home/regression.$t.out else diff -wu $test_home/regression.$t.exp $test_home/regression.$t.out fi if [ $? != 0 ]; then failed=1 fi done echo -e "\n\nSummary" for t in $tests; do grep -e "^*" $test_home/regression.$t.out done if [ $num_errors != 0 ]; then echo $num_errors tests failed exit 1 elif [ $failed = 1 ]; then echo $num_passed tests passed but diff failed exit 2 else echo $num_passed tests passed for t in $tests; do rm -f "$test_home/regression.$t.out" done crm_shadow --force --delete $shadow >/dev/null 2>&1 exit 0 fi diff --git a/tools/report.collector b/tools/report.collector index 0130a2b40e..d7c7c51e62 100644 --- a/tools/report.collector +++ b/tools/report.collector @@ -1,863 +1,906 @@ # Copyright (C) 2007 Dejan Muhamedagic # Almost everything as part of hb_report # Copyright (C) 2010 Andrew Beekhof # Cleanups, refactoring, extensions # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # if echo $REPORT_HOME | grep -qs '^/' then debug "Using full path to working directory: $REPORT_HOME" else REPORT_HOME="$HOME/$REPORT_HOME" debug "Canonicalizing working directory path: $REPORT_HOME" fi detect_host findlogdcf() { for f in \ `test -x $CRM_DAEMON_DIR/ha_logd && which strings > /dev/null 2>&1 && strings $CRM_DAEMON_DIR/ha_logd | grep 'logd\.cf'` \ `for d; do echo $d/logd.cf $d/ha_logd.cf; done` do if [ -f "$f" ]; then echo $f debug "Located logd.cf at: $f" return 0 fi done debug "Could not determine logd.cf location" return 1 } # # find files newer than a and older than b # isnumber() { echo "$*" | grep -qs '^[0-9][0-9]*$' } touchfile() { t=`mktemp` && perl -e "\$file=\"$t\"; \$tm=$1;" -e 'utime $tm, $tm, $file;' && echo $t } find_files_clean() { [ -z "$from_stamp" ] || rm -f "$from_stamp" [ -z "$to_stamp" ] || rm -f "$to_stamp" from_stamp="" to_stamp="" } find_files() { dirs= from_time=$2 to_time=$3 for d in $1; do if [ -d $d ]; then dirs="$dirs $d" fi done if [ x"$dirs" = x ]; then return fi isnumber "$from_time" && [ "$from_time" -gt 0 ] || { warning "sorry, can't find files in [ $1 ] based on time if you don't supply time" return } trap find_files_clean 0 if ! from_stamp=`touchfile $from_time`; then warning "sorry, can't create temporary file for find_files" return fi findexp="-newer $from_stamp" if isnumber "$to_time" && [ "$to_time" -gt 0 ]; then if ! to_stamp=`touchfile $to_time`; then warning "sorry, can't create temporary file for find_files" find_files_clean return fi findexp="$findexp ! -newer $to_stamp" fi find $dirs -type f $findexp find_files_clean trap "" 0 } # # check permissions of files/dirs # pl_checkperms() { perl -e ' # check permissions and ownership # uid and gid are numeric # everything must match exactly # no error checking! (file should exist, etc) ($filename, $perms, $in_uid, $in_gid) = @ARGV; ($mode,$uid,$gid) = (stat($filename))[2,4,5]; $p=sprintf("%04o", $mode & 07777); $p ne $perms and exit(1); $uid ne $in_uid and exit(1); $gid ne $in_gid and exit(1); ' $* } num_id() { getent $1 $2 | awk -F: '{print $3}' } chk_id() { [ "$2" ] && return 0 echo "$1: id not found" return 1 } check_perms() { while read type f p uid gid; do if [ ! -e "$f" ]; then echo "$f doesn't exist" continue elif [ ! -$type "$f" ]; then echo "$f has wrong type" continue fi n_uid=`num_id passwd $uid` chk_id "$uid" "$n_uid" || continue n_gid=`num_id group $gid` chk_id "$gid" "$n_gid" || continue pl_checkperms $f $p $n_uid $n_gid || { echo "wrong permissions or ownership for $f:" ls -ld $f } done } # # coredumps # findbinary() { random_binary=`which cat 2>/dev/null` # suppose we are lucky binary=`gdb $random_binary $1 < /dev/null 2>/dev/null | grep 'Core was generated' | awk '{print $5}' | sed "s/^.//;s/[.':]*$//"` if [ x = x"$binary" ]; then debug "Could not detect the program name for core $1 from the gdb output; will try with file(1)" binary=$(file $1 | awk '/from/{ for( i=1; i<=NF; i++ ) if( $i == "from" ) { print $(i+1) break } }') binary=`echo $binary | tr -d "'"` binary=$(echo $binary | tr -d '`') if [ "$binary" ]; then binary=`which $binary 2>/dev/null` fi fi if [ x = x"$binary" ]; then warning "Could not find the program path for core $1" return fi fullpath=`which $binary 2>/dev/null` if [ x = x"$fullpath" ]; then if [ -x $CRM_DAEMON_DIR/$binary ]; then echo $CRM_DAEMON_DIR/$binary debug "Found the program at $CRM_DAEMON_DIR/$binary for core $1" else warning "Could not find the program path for core $1" fi else echo $fullpath debug "Found the program at $fullpath for core $1" fi } getbt() { which gdb > /dev/null 2>&1 || { warning "Please install gdb to get backtraces" return } for corefile; do absbinpath=`findbinary $corefile` [ x = x"$absbinpath" ] && continue echo "====================== start backtrace ======================" ls -l $corefile # Summary first... gdb -batch -n -quiet -ex ${BT_OPTS:-"thread apply all bt"} -ex quit \ $absbinpath $corefile 2>/dev/null echo "====================== start detail ======================" # Now the unreadable details... gdb -batch -n -quiet -ex ${BT_OPTS:-"thread apply all bt full"} -ex quit \ $absbinpath $corefile 2>/dev/null echo "======================= end backtrace =======================" done } dump_status_and_config() { crm_mon -1 2>&1 | grep -v '^Last upd' > $target/$CRM_MON_F cibadmin -Ql 2>/dev/null > $target/${CIB_F}.live } getconfig() { cluster=$1; shift; target=$1; shift; for cf in $*; do if [ -e "$cf" ]; then cp -a "$cf" $target/ fi done if which crm_uuid >/dev/null 2>&1; then crm_uuid -r > $target/$HB_UUID_F 2>&1 fi if is_running crmd; then dump_status_and_config case $cluster in cman) crm_node -p --cman > $target/$MEMBERSHIP_F 2>&1;; corosync|openais) crm_node -p --openais > $target/$MEMBERSHIP_F 2>&1;; heartbeat) crm_node -p --heartbeat > $target/$MEMBERSHIP_F 2>&1;; *) crm_node -p > $target/$MEMBERSHIP_F 2>&1;; esac echo "$host" > $target/RUNNING elif is_running pacemaker_remoted; then dump_status_and_config echo "$host" > $target/RUNNING else echo "$host" > $target/STOPPED fi } get_readable_cib() { target="$1"; shift; if [ -f "$target/$CIB_F" ]; then crm_verify -V -x "$target/$CIB_F" >"$target/$CRM_VERIFY_F" 2>&1 if which crm >/dev/null 2>&1 ; then CIB_file="$target/$CIB_F" crm configure show >"$target/$CIB_TXT_F" 2>&1 elif which pcs >/dev/null 2>&1 ; then pcs config -f "$target/$CIB_F" >"$target/$CIB_TXT_F" 2>&1 fi fi } # # remove values of sensitive attributes # # this is not proper xml parsing, but it will work under the # circumstances sanitize_xml_attrs() { sed $( for patt in $SANITIZE; do echo "-e /name=\"$patt\"/s/value=\"[^\"]*\"/value=\"****\"/" done ) } sanitize_hacf() { awk ' $1=="stonith_host"{ for( i=5; i<=NF; i++ ) $i="****"; } {print} ' } sanitize_one_clean() { [ -z "$tmp" ] || rm -f "$tmp" tmp="" [ -z "$ref" ] || rm -f "$ref" ref="" } sanitize() { file=$1 compress="" if [ -z "$SANITIZE" ]; then return fi echo $file | grep -qs 'gz$' && compress=gzip echo $file | grep -qs 'bz2$' && compress=bzip2 if [ "$compress" ]; then decompress="$compress -dc" else compress=cat decompress=cat fi trap sanitize_one_clean 0 tmp=`mktemp` ref=`mktemp` if [ -z "$tmp" -o -z "$ref" ]; then sanitize_one_clean fatal "cannot create temporary files" fi touch -r $file $ref # save the mtime if [ "`basename $file`" = ha.cf ]; then sanitize_hacf else $decompress | sanitize_xml_attrs | $compress fi < $file > $tmp mv $tmp $file # note: cleaning $tmp up is still needed even after it's renamed # because its temp directory is still there. touch -r $ref $file sanitize_one_clean trap "" 0 } # # get some system info # distro() { if which lsb_release >/dev/null 2>&1 then lsb_release -d | sed -e 's/^Description:\s*//' debug "Using lsb_release for distribution info" return fi relf=`ls /etc/debian_version 2>/dev/null` || relf=`ls /etc/slackware-version 2>/dev/null` || relf=`ls -d /etc/*-release 2>/dev/null` && { for f in $relf; do test -f $f && { echo "`ls $f` `cat $f`" debug "Found `echo $relf | tr '\n' ' '` distribution release file(s)" return } done } warning "No lsb_release, no /etc/*-release, no /etc/debian_version: no distro information" } pkg_ver() { if which dpkg >/dev/null 2>&1 ; then pkg_mgr="deb" elif which rpm >/dev/null 2>&1 ; then pkg_mgr="rpm" elif which pkg_info >/dev/null 2>&1 ; then pkg_mgr="pkg_info" elif which pkginfo >/dev/null 2>&1 ; then pkg_mgr="pkginfo" else warning "Unknown package manager" return fi debug "The package manager is: $pkg_mgr" echo "The package manager is: $pkg_mgr" echo "Installed packages:" case $pkg_mgr in deb) dpkg-query -f '${Package} ${Version} ${Architecture}\n' -W | sort echo for pkg in $*; do if dpkg-query -W $pkg 2>/dev/null ; then debug "Verifying installation of: $pkg" echo "Verifying installation of: $pkg" debsums -s $pkg 2>/dev/null fi done ;; rpm) rpm -qa --qf '%{name} %{version}-%{release} - %{distribution} %{arch}\n' | sort echo for pkg in $*; do if rpm -q $pkg >/dev/null 2>&1 ; then debug "Verifying installation of: $pkg" echo "Verifying installation of: $pkg" rpm --verify $pkg 2>&1 fi done ;; pkg_info) pkg_info ;; pkginfo) pkginfo | awk '{print $3}' # format? ;; esac } getbacktraces() { debug "Looking for backtraces: $*" flist=$( for f in `find_files "$CRM_CORE_DIRS" $1 $2`; do bf=`basename $f` test `expr match $bf core` -gt 0 && echo $f done) if [ "$flist" ]; then for core in $flist; do log "Found core file: `ls -al $core`" done # Make a copy of them in case we need more data later # Luckily they compress well mkdir cores >/dev/null 2>&1 cp -a $flist cores/ shrink cores rm -rf cores # Now get as much as we can from them automagically for f in $flist; do getbt $f done fi } getpeinputs() { if [ -n "$PE_STATE_DIR" ]; then flist=$( find_files "$PE_STATE_DIR" "$1" "$2" | sed "s,`dirname $PE_STATE_DIR`/,,g" ) if [ "$flist" ]; then (cd $(dirname "$PE_STATE_DIR") && tar cf - $flist) | (cd "$3" && tar xf -) debug "found `echo $flist | wc -w` pengine input files in $PE_STATE_DIR" fi fi } getblackboxes() { flist=$( find_files $BLACKBOX_DIR $1 $2 ) for bb in $flist; do bb_short=`basename $bb` qb-blackbox $bb > $3/${bb_short}.blackbox 2>&1 info "Extracting contents of blackbox: $bb_short" done } # # some basic system info and stats # sys_info() { cluster=$1; shift echo "Platform: `uname`" echo "Kernel release: `uname -r`" echo "Architecture: `uname -m`" if [ `uname` = Linux ]; then echo "Distribution: `distro`" fi echo cibadmin --version 2>&1 | head -1 cibadmin -! 2>&1 case $cluster in openais) echo openais # version: how? ;; cman) cman_tool -V 2>&1 | head -1 /usr/sbin/corosync -v 2>&1 | head -1 ;; corosync) /usr/sbin/corosync -v 2>&1 | head -1 ;; heartbeat) echo heartbeat $($CRM_DAEMON_DIR/heartbeat -V 2>&1) ;; esac # Cluster glue version hash (if available) stonith -V 2>/dev/null # Resource agents version hash echo "resource-agents: `grep 'Build version:' /usr/lib/ocf/resource.d/heartbeat/.ocf-shellfuncs`" echo pkg_ver $* } sys_stats() { set -x uname -n uptime ps axf ps auxw top -b -n 1 ifconfig -a ip addr list netstat -i arp -an test -d /proc && { cat /proc/cpuinfo } lsscsi lspci mount df set +x } dlm_dump() { if which dlm_tool >/dev/null 2>&1 ; then if is_running dlm_controld; then echo "--- Lockspace overview:" dlm_tool ls -n echo "---Lockspace history:" dlm_tool dump echo "---Lockspace status:" dlm_tool status dlm_tool status -v echo "---Lockspace config:" dlm_tool dump_config dlm_tool log_plock dlm_tool ls | grep name | while read X N ; do echo "--- Lockspace $N:" dlm_tool lockdump "$N" dlm_tool lockdebug -svw "$N" done fi fi } drbd_info() { test -f /proc/drbd && { echo "--- /proc/drbd:" cat /proc/drbd 2>&1 echo } if which drbd-overview >/dev/null 2>&1; then echo "--- drbd-overview:" drbd-overview 2>&1 echo fi if which drbdsetup >/dev/null 2>&1; then echo "--- drbdsetup status:" drbdsetup status --verbose --statistics 2>&1 echo echo "--- drbdsetup events2:" drbdsetup events2 --timestamps --statistics --now 2>&1 echo fi if which drbdadm >/dev/null 2>&1; then echo "--- drbdadm show-gi:" for res in $(drbdsetup status | grep -e ^\\S | awk '{ print $1 }'); do echo "$res:" drbdadm show-gi $res 2>&1 echo done fi } iscfvarset() { test "`getcfvar $1 $2`" } iscfvartrue() { getcfvar $1 $2 $3 | egrep -qsi "^(true|y|yes|on|1)" } uselogd() { cf_file=$2 case $1 in heartbeat) iscfvartrue $1 use_logd $cf_file && return 0 # if use_logd true iscfvarset $1 logfacility $cf_file || iscfvarset $1 logfile $cf_file || iscfvarset $1 debugfile $cf_file || return 0 # or none of the log options set false ;; *) iscfvartrue $1 use_logd $cf_file ;; esac } get_logfiles() { cf_type=$1 cf_file="$2" cf_logd="$3" facility_var="logfacility" if [ -f "$cf_logd" ]; then # TODO: this call is broken, it expects args if uselogd; then cf_file="$cf_logd" cf_type="logd" fi fi case $cf_type in cman|openais|corosync) if [ -f "$cf_file" ]; then debug "Reading $cf_type log settings from $cf_file" if iscfvartrue $cf_type to_syslog "$cf_file"; then facility_var=syslog_facility fi if iscfvartrue $cf_type to_logfile "$cf_file"; then logfile=$(getcfvar $cf_type logfile "$cf_file") fi fi ;; heartbeat|logd) if [ -f "$cf_file" ]; then debug "Reading $cf_type log settings from $cf_file" if iscfvartrue $cf_type debug "$cf_file"; then logfile=$(getcfvar $cf_type debugfile "$cf_file") else logfile=$(getcfvar $cf_type logfile "$cf_file") fi fi ;; esac if [ -z "$logfile" ]; then logfile="/var/log/pacemaker.log" debug "Log settings not found for cluster type $cf_type, assuming $logfile" fi if [ -f "$logfile" ]; then echo $logfile fi if [ "x$facility" = x ]; then facility=`getcfvar $cf_type $facility_var $cf_file` [ "" = "$facility" ] && facility="daemon" fi # Always include system logs (if we can find them) msg="Mark:pcmk:`perl -e 'print time()'`" logger -p $facility.info $msg >/dev/null 2>&1 sleep 2 # Give syslog time to catch up in case it's busy findmsg 1 "$msg" # Look for detail logs: # - initial pacemakerd logs and tracing might go to a different file pattern="Starting Pacemaker" # - make sure we get something from the Policy Engine pattern="$pattern\\|Calculated Transition" # - cib and lrmd updates (helpful on non-DC nodes or when the cluster has been up for a long time) pattern="$pattern\\|cib_perform_op\\|process_lrm_event" # - pacemaker_remote might use a different file pattern="$pattern\\|pacemaker_remoted:" findmsg 3 "$pattern" } essential_files() { cat< /dev/null 2>&1 + if [ $? -eq 0 ]; then + cl_have_journald=1 + else + cl_have_journald=0 + fi + + cl_lognames="$CL_LOGFILES" + if [ $cl_have_journald -eq 1 ]; then + cl_lognames="$cl_lognames journalctl" + fi + cl_lognames=$(trim "$cl_lognames") + if [ -z "$cl_lognames" ]; then + return + fi + + # YYYY-MM-DD HH:MM:SS + cl_start_ymd=$(date -d @${CL_START} +"%F %T") + cl_end_ymd=$(date -d @${CL_END} +"%F %T") + + debug "Gathering logs from $cl_start_ymd to $cl_end_ymd:" + debug " $cl_lognames" + + # Remove our temporary file if we get interrupted here + trap '[ -z "$cl_pattfile" ] || rm -f "$cl_pattfile"' 0 + + # Create a temporary file with patterns to grep for + cl_pattfile=$(mktemp) || fatal "cannot create temporary files" + for cl_pattern in $LOG_PATTERNS; do + echo "$cl_pattern" + done > $cl_pattfile + + echo "Log pattern matches from $REPORT_TARGET:" > $ANALYSIS_F + if [ -n "$CL_LOGFILES" ]; then + for cl_logfile in $CL_LOGFILES; do + cl_extract="$(basename $cl_logfile).extract.txt" + + if [ ! -f "$cl_logfile" ]; then + # Not a file + continue + + elif [ -f "$cl_extract" ]; then + # We already have it + continue + fi + + dumplogset "$cl_logfile" $LOG_START $LOG_END > "$cl_extract" + sanitize "$cl_extract" + + grep -f "$cl_pattfile" "$cl_extract" >> $ANALYSIS_F + done + fi + + # Collect systemd logs if present + if [ $cl_have_journald -eq 1 ]; then + journalctl --since "$cl_start_ymd" --until "$cl_end_ymd" > journal.log + grep -f "$cl_pattfile" journal.log >> $ANALYSIS_F + fi + + rm -f $cl_pattfile + trap "" 0 +} + debug "Initializing $REPORT_TARGET subdir" if [ "$REPORT_MASTER" != "$REPORT_TARGET" ]; then if [ -e $REPORT_HOME/$REPORT_TARGET ]; then warning "Directory $REPORT_HOME/$REPORT_TARGET already exists, using /tmp/$$/$REPORT_TARGET instead" REPORT_HOME=/tmp/$$ fi fi mkdir -p $REPORT_HOME/$REPORT_TARGET cd $REPORT_HOME/$REPORT_TARGET case $CLUSTER in any) cluster=`get_cluster_type`;; *) cluster=$CLUSTER;; esac logd_cf=`findlogdcf` cluster_cf=`find_cluster_cf $cluster` # If cluster stack is still "any", this might be a Pacemaker Remote node, # so don't complain in that case. if [ -z "$cluster_cf" ] && [ $cluster != "any" ]; then warning "Could not determine the location of your cluster configuration" fi -if [ $SEARCH_LOGS = 1 ]; then - logfiles=`get_logfiles $cluster "$cluster_cf" "$logd_cf" | sort -u` - if [ -z "$logfiles" ]; then - fatal "Logfile discovery disabled, try specifying --logfile /some/path" - fi - -elif [ -z "$EXTRA_LOGS" ]; then - fatal "Could not determine the location of your cluster logs, try specifying --logfile /some/path" +if [ "$SEARCH_LOGS" = "1" ]; then + logfiles=$(get_logfiles "$cluster" "$cluster_cf" "$logd_cf" | sort -u) +fi +logfiles="$(trim "$logfiles $EXTRA_LOGS")" -else - logfiles="$EXTRA_LOGS" +if [ -z "$logfiles" ]; then + which journalctl > /dev/null 2>&1 + if [ $? -eq 0 ]; then + info "Systemd journal will be only log collected" + else + info "No logs will be collected" + fi + info "No log files found or specified with --logfile /some/path" fi debug "Config: $cluster ($cluster_cf $logd_cf) $logfiles" sys_info $cluster $PACKAGES > $SYSINFO_F essential_files $cluster | check_perms > $PERMISSIONS_F 2>&1 getconfig $cluster "$REPORT_HOME/$REPORT_TARGET" "$cluster_cf" "$logd_cf" "$CRM_CONFIG_DIR/$CIB_F" "$HA_STATE_DIR/hostcache" "/etc/drbd.conf" "/etc/drbd.d" "/etc/booth" getpeinputs $LOG_START $LOG_END $REPORT_HOME/$REPORT_TARGET getbacktraces $LOG_START $LOG_END > $REPORT_HOME/$REPORT_TARGET/$BT_F getblackboxes $LOG_START $LOG_END $REPORT_HOME/$REPORT_TARGET case $cluster in cman|corosync) if is_running corosync; then corosync-blackbox >corosync-blackbox-live.txt 2>&1 # corosync-fplay > corosync-blackbox.txt tool=`pickfirst corosync-objctl corosync-cmapctl` case $tool in *objctl) $tool -a > corosync.dump 2>/dev/null;; *cmapctl) $tool > corosync.dump 2>/dev/null;; esac corosync-quorumtool -s -i > corosync.quorum 2>&1 fi ;; esac dc=`crm_mon -1 2>/dev/null | awk '/Current DC/ {print $3}'` if [ "$REPORT_TARGET" = "$dc" ]; then echo "$REPORT_TARGET" > DC fi dlm_dump > $DLM_DUMP_F 2>&1 sys_stats > $SYSSTATS_F 2>&1 drbd_info > $DRBD_INFO_F 2>&1 debug "Sanitizing files: $SANITIZE" # # replace sensitive info with '****' # cf="" if [ ! -z "$cluster_cf" ]; then cf=`basename $cluster_cf` fi for f in "$cf" "$CIB_F" "$CIB_F.live" pengine/*; do if [ -f "$f" ]; then sanitize "$f" fi done # For convenience, generate human-readable version of CIB and any XML errors # in it (AFTER sanitizing, so we don't need to sanitize this output) get_readable_cib "$REPORT_HOME/$REPORT_TARGET" -# Grab logs -start=`date -d @${LOG_START} +"%F %T"` -end=`date -d @${LOG_END} +"%F %T"` - -debug "Gathering logs from $start to $end: $logfiles $EXTRA_LOGS" -trap '[ -z "$pattfile" ] || rm -f "$pattfile"' 0 -pattfile=`mktemp` || fatal "cannot create temporary files" -for p in $LOG_PATTERNS; do - echo "$p" -done > $pattfile - -for l in $logfiles $EXTRA_LOGS; do - b="$(basename $l).extract.txt" - - if [ ! -f "$l" ]; then - # Not a file - continue - - elif [ -f "$b" ]; then - # We already have it - continue - fi - - dumplogset "$l" $LOG_START $LOG_END > "$b" - sanitize "$b" - - echo "Log patterns $REPORT_TARGET:" > $ANALYSIS_F - grep -f "$pattfile" "$b" >> $ANALYSIS_F -done - -which journalctl > /dev/null 2>&1 -if [ $? = 0 ]; then - log "Including segment [$LOG_START-$LOG_END] from journald" - journalctl --since "$start" --until "$end" > journal.log - cat journal.log | grep -f $pattfile >> $ANALYSIS_F -fi - -rm -f $pattfile -trap "" 0 +collect_logs "$LOG_START" "$LOG_END" $logfiles # Purge files containing no information for f in `ls -1`; do if [ -d "$f" ]; then continue elif [ ! -s "$f" ]; then case $f in *core*) log "Detected empty core file: $f";; *) debug "Removing empty file: `ls -al $f`" rm -f $f ;; esac fi done # Parse for events -for l in $logfiles $EXTRA_LOGS; do +for l in $logfiles; do b="$(basename $l).extract.txt" node_events "$b" > $EVENTS_F # Link the first logfile to a standard name if it doesn't yet exist if [ -e "$b" -a ! -e "$HALOG_F" ]; then ln -s "$b" "$HALOG_F" fi done if [ -e $REPORT_HOME/.env ]; then debug "Localhost: $REPORT_MASTER $REPORT_TARGET" elif [ "$REPORT_MASTER" != "$REPORT_TARGET" ]; then debug "Streaming report back to $REPORT_MASTER" (cd $REPORT_HOME && tar cf - $REPORT_TARGET) if [ "$REMOVE" = "1" ]; then cd rm -rf $REPORT_HOME fi fi # vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4 textwidth=80: