diff --git a/include/crm/common/action_relation_internal.h b/include/crm/common/action_relation_internal.h index fcd43ba953..658432e1d6 100644 --- a/include/crm/common/action_relation_internal.h +++ b/include/crm/common/action_relation_internal.h @@ -1,124 +1,127 @@ /* * Copyright 2023 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #ifndef PCMK__CRM_COMMON_ACTION_RELATION_INTERNAL__H # define PCMK__CRM_COMMON_ACTION_RELATION_INTERNAL__H /*! * Flags to indicate the relationship between two actions * * @COMPAT The values and semantics of these flags should not be changed until * the deprecated enum pe_ordering is dropped from the public API. */ enum pcmk__action_relation_flags { //! No relation (compare with equality rather than bit set) pcmk__ar_none = 0U, //! Actions are ordered (optionally, if no other flags are set) pcmk__ar_ordered = (1U << 0), //! Relation applies only if 'first' cannot be part of a live migration pcmk__ar_if_first_unmigratable = (1U << 1), /*! * If 'then' is required, 'first' becomes required (and becomes unmigratable * if 'then' is); also, if 'first' is a stop of a blocked resource, 'then' * becomes unrunnable */ pcmk__ar_then_implies_first = (1U << 4), /*! * If 'first' is required, 'then' becomes required; if 'first' is a stop of * a blocked resource, 'then' becomes unrunnable */ pcmk__ar_first_implies_then = (1U << 5), /*! * If 'then' is required and for a promoted instance, 'first' becomes * required (and becomes unmigratable if 'then' is) */ pcmk__ar_promoted_then_implies_first = (1U << 6), /*! * 'first' is runnable only if 'then' is both runnable and migratable, * and 'first' becomes required if 'then' is */ pcmk__ar_unmigratable_then_blocks = (1U << 7), //! 'then' is runnable (and migratable) only if 'first' is runnable pcmk__ar_unrunnable_first_blocks = (1U << 8), //! If 'first' is unrunnable, 'then' becomes a real, unmigratable action pcmk__ar_first_else_then = (1U << 9), //! If 'first' is required, 'then' action for instance on same node is pcmk__ar_first_implies_same_node_then = (1U << 10), /*! * Disable relation if 'first' is unrunnable and for an active resource, * otherwise order actions and make 'then' unrunnable if 'first' is. * * This is used to order a bundle replica's start of its container before a * probe of its remote connection resource, in case the connection uses the * REMOTE_CONTAINER_HACK to replace the connection address with where the * container is running. */ pcmk__ar_nested_remote_probe = (1U << 11), /*! * If 'first' is for a blocked resource, make 'then' unrunnable. * * If 'then' is required, make 'first' required, make 'first' unmigratable * if 'then' is unmigratable, and make 'then' unrunnable if 'first' is * unrunnable. * * If 'then' is unrunnable and for the same resource as 'first', make * 'first' required if it is runnable, and make 'first' unmigratable if * 'then' is unmigratable. * * This is used for "stop then start primitive" (restarts) and * "stop group member then stop previous member". */ pcmk__ar_intermediate_stop = (1U << 12), /*! * The actions must be serialized if in the same transition but can be in * either order. (In practice, we always arrange them as 'first' then * 'then', so they end up being essentially the same as optional orderings.) * * @TODO Handle more intelligently -- for example, we could schedule the * action with the fewest inputs first, so we're more likely to execute at * least one if there is a failure during the transition. Or, we could * prefer certain action types over others, or base it on resource priority. */ pcmk__ar_serialize = (1U << 14), //! Relation applies only if actions are on same node pcmk__ar_if_on_same_node = (1U << 15), //! If 'then' is required, 'first' must be added to the transition graph pcmk__ar_then_implies_first_graphed = (1U << 16), //! If 'first' is required and runnable, 'then' must be in graph pcmk__ar_first_implies_then_graphed = (1U << 17), //! User-configured asymmetric ordering pcmk__ar_asymmetric = (1U << 20), //! Actions are ordered if on same node (or migration target for migrate_to) pcmk__ar_if_on_same_node_or_target = (1U << 21), //! 'then' action is runnable if certain number of 'first' instances are pcmk__ar_min_runnable = (1U << 22), //! Ordering applies only if 'first' is required and on same node as 'then' pcmk__ar_if_required_on_same_node = (1U << 23), + + //! Ordering applies even if 'first' runs on guest node created by 'then' + pcmk__ar_guest_allowed = (1U << 24), }; #endif // PCMK__CRM_COMMON_ACTION_RELATION_INTERNAL__H diff --git a/lib/pacemaker/pcmk_sched_bundle.c b/lib/pacemaker/pcmk_sched_bundle.c index 2d4376dd77..4ff624ce3a 100644 --- a/lib/pacemaker/pcmk_sched_bundle.c +++ b/lib/pacemaker/pcmk_sched_bundle.c @@ -1,1055 +1,1056 @@ /* * Copyright 2004-2023 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include "libpacemaker_private.h" struct assign_data { const pe_node_t *prefer; bool stop_if_fail; }; /*! * \internal * \brief Assign a single bundle replica's resources (other than container) * * \param[in,out] replica Replica to assign * \param[in] user_data Preferred node, if any * * \return true (to indicate that any further replicas should be processed) */ static bool assign_replica(pe__bundle_replica_t *replica, void *user_data) { pe_node_t *container_host = NULL; struct assign_data *assign_data = user_data; const pe_node_t *prefer = assign_data->prefer; bool stop_if_fail = assign_data->stop_if_fail; const pe_resource_t *bundle = pe__const_top_resource(replica->container, true); if (replica->ip != NULL) { pe_rsc_trace(bundle, "Assigning bundle %s IP %s", bundle->id, replica->ip->id); replica->ip->cmds->assign(replica->ip, prefer, stop_if_fail); } container_host = replica->container->allocated_to; if (replica->remote != NULL) { if (pe__is_guest_or_remote_node(container_host)) { /* REMOTE_CONTAINER_HACK: "Nested" connection resources must be on * the same host because Pacemaker Remote only supports a single * active connection. */ pcmk__new_colocation("#replica-remote-with-host-remote", NULL, INFINITY, replica->remote, container_host->details->remote_rsc, NULL, NULL, pcmk__coloc_influence); } pe_rsc_trace(bundle, "Assigning bundle %s connection %s", bundle->id, replica->remote->id); replica->remote->cmds->assign(replica->remote, prefer, stop_if_fail); } if (replica->child != NULL) { pe_node_t *node = NULL; GHashTableIter iter; g_hash_table_iter_init(&iter, replica->child->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) { if (!pe__same_node(node, replica->node)) { node->weight = -INFINITY; } else if (!pcmk__threshold_reached(replica->child, node, NULL)) { node->weight = INFINITY; } } pe__set_resource_flags(replica->child->parent, pcmk_rsc_assigning); pe_rsc_trace(bundle, "Assigning bundle %s replica child %s", bundle->id, replica->child->id); replica->child->cmds->assign(replica->child, replica->node, stop_if_fail); pe__clear_resource_flags(replica->child->parent, pcmk_rsc_assigning); } return true; } /*! * \internal * \brief Assign a bundle resource to a node * * \param[in,out] rsc Resource to assign to a node * \param[in] prefer Node to prefer, if all else is equal * \param[in] stop_if_fail If \c true and a primitive descendant of \p rsc * can't be assigned to a node, set the * descendant's next role to stopped and update * existing actions * * \return Node that \p rsc is assigned to, if assigned entirely to one node * * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can * completely undo the assignment. A successful assignment can be either * undone or left alone as final. A failed assignment has the same effect * as calling pcmk__unassign_resource(); there are no side effects on * roles or actions. */ pe_node_t * pcmk__bundle_assign(pe_resource_t *rsc, const pe_node_t *prefer, bool stop_if_fail) { GList *containers = NULL; pe_resource_t *bundled_resource = NULL; struct assign_data assign_data = { prefer, stop_if_fail }; CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle)); pe_rsc_trace(rsc, "Assigning bundle %s", rsc->id); pe__set_resource_flags(rsc, pcmk_rsc_assigning); pe__show_node_scores(!pcmk_is_set(rsc->cluster->flags, pcmk_sched_output_scores), rsc, __func__, rsc->allowed_nodes, rsc->cluster); // Assign all containers first, so we know what nodes the bundle will be on containers = g_list_sort(pe__bundle_containers(rsc), pcmk__cmp_instance); pcmk__assign_instances(rsc, containers, pe__bundle_max(rsc), rsc->fns->max_per_node(rsc)); g_list_free(containers); // Then assign remaining replica resources pe__foreach_bundle_replica(rsc, assign_replica, (void *) &assign_data); // Finally, assign the bundled resources to each bundle node bundled_resource = pe__bundled_resource(rsc); if (bundled_resource != NULL) { pe_node_t *node = NULL; GHashTableIter iter; g_hash_table_iter_init(&iter, bundled_resource->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) { if (pe__node_is_bundle_instance(rsc, node)) { node->weight = 0; } else { node->weight = -INFINITY; } } bundled_resource->cmds->assign(bundled_resource, prefer, stop_if_fail); } pe__clear_resource_flags(rsc, pcmk_rsc_assigning|pcmk_rsc_unassigned); return NULL; } /*! * \internal * \brief Create actions for a bundle replica's resources (other than child) * * \param[in,out] replica Replica to create actions for * \param[in] user_data Unused * * \return true (to indicate that any further replicas should be processed) */ static bool create_replica_actions(pe__bundle_replica_t *replica, void *user_data) { if (replica->ip != NULL) { replica->ip->cmds->create_actions(replica->ip); } if (replica->container != NULL) { replica->container->cmds->create_actions(replica->container); } if (replica->remote != NULL) { replica->remote->cmds->create_actions(replica->remote); } return true; } /*! * \internal * \brief Create all actions needed for a given bundle resource * * \param[in,out] rsc Bundle resource to create actions for */ void pcmk__bundle_create_actions(pe_resource_t *rsc) { pe_action_t *action = NULL; GList *containers = NULL; pe_resource_t *bundled_resource = NULL; CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle)); pe__foreach_bundle_replica(rsc, create_replica_actions, NULL); containers = pe__bundle_containers(rsc); pcmk__create_instance_actions(rsc, containers); g_list_free(containers); bundled_resource = pe__bundled_resource(rsc); if (bundled_resource != NULL) { bundled_resource->cmds->create_actions(bundled_resource); if (pcmk_is_set(bundled_resource->flags, pcmk_rsc_promotable)) { pe__new_rsc_pseudo_action(rsc, PCMK_ACTION_PROMOTE, true, true); action = pe__new_rsc_pseudo_action(rsc, PCMK_ACTION_PROMOTED, true, true); action->priority = INFINITY; pe__new_rsc_pseudo_action(rsc, PCMK_ACTION_DEMOTE, true, true); action = pe__new_rsc_pseudo_action(rsc, PCMK_ACTION_DEMOTED, true, true); action->priority = INFINITY; } } } /*! * \internal * \brief Create internal constraints for a bundle replica's resources * * \param[in,out] replica Replica to create internal constraints for * \param[in,out] user_data Replica's parent bundle * * \return true (to indicate that any further replicas should be processed) */ static bool replica_internal_constraints(pe__bundle_replica_t *replica, void *user_data) { pe_resource_t *bundle = user_data; replica->container->cmds->internal_constraints(replica->container); // Start bundle -> start replica container pcmk__order_starts(bundle, replica->container, pcmk__ar_unrunnable_first_blocks |pcmk__ar_then_implies_first_graphed); // Stop bundle -> stop replica child and container if (replica->child != NULL) { pcmk__order_stops(bundle, replica->child, pcmk__ar_then_implies_first_graphed); } pcmk__order_stops(bundle, replica->container, pcmk__ar_then_implies_first_graphed); // Start replica container -> bundle is started pcmk__order_resource_actions(replica->container, PCMK_ACTION_START, bundle, PCMK_ACTION_RUNNING, pcmk__ar_first_implies_then_graphed); // Stop replica container -> bundle is stopped pcmk__order_resource_actions(replica->container, PCMK_ACTION_STOP, bundle, PCMK_ACTION_STOPPED, pcmk__ar_first_implies_then_graphed); if (replica->ip != NULL) { replica->ip->cmds->internal_constraints(replica->ip); // Replica IP address -> replica container (symmetric) pcmk__order_starts(replica->ip, replica->container, - pcmk__ar_unrunnable_first_blocks|pe_order_preserve); + pcmk__ar_unrunnable_first_blocks + |pcmk__ar_guest_allowed); pcmk__order_stops(replica->container, replica->ip, - pcmk__ar_then_implies_first|pe_order_preserve); + pcmk__ar_then_implies_first|pcmk__ar_guest_allowed); pcmk__new_colocation("#ip-with-container", NULL, INFINITY, replica->ip, replica->container, NULL, NULL, pcmk__coloc_influence); } if (replica->remote != NULL) { /* This handles ordering and colocating remote relative to container * (via "#resource-with-container"). Since IP is also ordered and * colocated relative to the container, we don't need to do anything * explicit here with IP. */ replica->remote->cmds->internal_constraints(replica->remote); } if (replica->child != NULL) { CRM_ASSERT(replica->remote != NULL); // "Start remote then child" is implicit in scheduler's remote logic } return true; } /*! * \internal * \brief Create implicit constraints needed for a bundle resource * * \param[in,out] rsc Bundle resource to create implicit constraints for */ void pcmk__bundle_internal_constraints(pe_resource_t *rsc) { pe_resource_t *bundled_resource = NULL; CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle)); pe__foreach_bundle_replica(rsc, replica_internal_constraints, rsc); bundled_resource = pe__bundled_resource(rsc); if (bundled_resource == NULL) { return; } // Start bundle -> start bundled clone pcmk__order_resource_actions(rsc, PCMK_ACTION_START, bundled_resource, PCMK_ACTION_START, pcmk__ar_then_implies_first_graphed); // Bundled clone is started -> bundle is started pcmk__order_resource_actions(bundled_resource, PCMK_ACTION_RUNNING, rsc, PCMK_ACTION_RUNNING, pcmk__ar_first_implies_then_graphed); // Stop bundle -> stop bundled clone pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP, bundled_resource, PCMK_ACTION_STOP, pcmk__ar_then_implies_first_graphed); // Bundled clone is stopped -> bundle is stopped pcmk__order_resource_actions(bundled_resource, PCMK_ACTION_STOPPED, rsc, PCMK_ACTION_STOPPED, pcmk__ar_first_implies_then_graphed); bundled_resource->cmds->internal_constraints(bundled_resource); if (!pcmk_is_set(bundled_resource->flags, pcmk_rsc_promotable)) { return; } pcmk__promotable_restart_ordering(rsc); // Demote bundle -> demote bundled clone pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE, bundled_resource, PCMK_ACTION_DEMOTE, pcmk__ar_then_implies_first_graphed); // Bundled clone is demoted -> bundle is demoted pcmk__order_resource_actions(bundled_resource, PCMK_ACTION_DEMOTED, rsc, PCMK_ACTION_DEMOTED, pcmk__ar_first_implies_then_graphed); // Promote bundle -> promote bundled clone pcmk__order_resource_actions(rsc, PCMK_ACTION_PROMOTE, bundled_resource, PCMK_ACTION_PROMOTE, pcmk__ar_then_implies_first_graphed); // Bundled clone is promoted -> bundle is promoted pcmk__order_resource_actions(bundled_resource, PCMK_ACTION_PROMOTED, rsc, PCMK_ACTION_PROMOTED, pcmk__ar_first_implies_then_graphed); } struct match_data { const pe_node_t *node; // Node to compare against replica pe_resource_t *container; // Replica container corresponding to node }; /*! * \internal * \brief Check whether a replica container is assigned to a given node * * \param[in] replica Replica to check * \param[in,out] user_data struct match_data with node to compare against * * \return true if the replica does not match (to indicate further replicas * should be processed), otherwise false */ static bool match_replica_container(const pe__bundle_replica_t *replica, void *user_data) { struct match_data *match_data = user_data; if (pcmk__instance_matches(replica->container, match_data->node, pcmk_role_unknown, false)) { match_data->container = replica->container; return false; // Match found, don't bother searching further replicas } return true; // No match, keep searching } /*! * \internal * \brief Get the host to which a bundle node is assigned * * \param[in] node Possible bundle node to check * * \return Node to which the container for \p node is assigned if \p node is a * bundle node, otherwise \p node itself */ static const pe_node_t * get_bundle_node_host(const pe_node_t *node) { if (pe__is_bundle_node(node)) { const pe_resource_t *container = node->details->remote_rsc->container; return container->fns->location(container, NULL, 0); } return node; } /*! * \internal * \brief Find a bundle container compatible with a dependent resource * * \param[in] dependent Dependent resource in colocation with bundle * \param[in] bundle Bundle that \p dependent is colocated with * * \return A container from \p bundle assigned to the same node as \p dependent * if assigned, otherwise assigned to any of dependent's allowed nodes, * otherwise NULL. */ static pe_resource_t * compatible_container(const pe_resource_t *dependent, const pe_resource_t *bundle) { GList *scratch = NULL; struct match_data match_data = { NULL, NULL }; // If dependent is assigned, only check there match_data.node = dependent->fns->location(dependent, NULL, 0); match_data.node = get_bundle_node_host(match_data.node); if (match_data.node != NULL) { pe__foreach_const_bundle_replica(bundle, match_replica_container, &match_data); return match_data.container; } // Otherwise, check for any of the dependent's allowed nodes scratch = g_hash_table_get_values(dependent->allowed_nodes); scratch = pcmk__sort_nodes(scratch, NULL); for (const GList *iter = scratch; iter != NULL; iter = iter->next) { match_data.node = iter->data; match_data.node = get_bundle_node_host(match_data.node); if (match_data.node == NULL) { continue; } pe__foreach_const_bundle_replica(bundle, match_replica_container, &match_data); if (match_data.container != NULL) { break; } } g_list_free(scratch); return match_data.container; } struct coloc_data { const pcmk__colocation_t *colocation; pe_resource_t *dependent; GList *container_hosts; }; /*! * \internal * \brief Apply a colocation score to replica node scores or resource priority * * \param[in] replica Replica of primary bundle resource in colocation * \param[in,out] user_data struct coloc_data for colocation being applied * * \return true (to indicate that any further replicas should be processed) */ static bool replica_apply_coloc_score(const pe__bundle_replica_t *replica, void *user_data) { struct coloc_data *coloc_data = user_data; pe_node_t *chosen = NULL; if (coloc_data->colocation->score < INFINITY) { replica->container->cmds->apply_coloc_score(coloc_data->dependent, replica->container, coloc_data->colocation, false); return true; } chosen = replica->container->fns->location(replica->container, NULL, 0); if ((chosen == NULL) || is_set_recursive(replica->container, pcmk_rsc_blocked, true)) { return true; } if ((coloc_data->colocation->primary_role >= pcmk_role_promoted) && ((replica->child == NULL) || (replica->child->next_role < pcmk_role_promoted))) { return true; } pe_rsc_trace(pe__const_top_resource(replica->container, true), "Allowing mandatory colocation %s using %s @%d", coloc_data->colocation->id, pe__node_name(chosen), chosen->weight); coloc_data->container_hosts = g_list_prepend(coloc_data->container_hosts, chosen); return true; } /*! * \internal * \brief Apply a colocation's score to node scores or resource priority * * Given a colocation constraint, apply its score to the dependent's * allowed node scores (if we are still placing resources) or priority (if * we are choosing promotable clone instance roles). * * \param[in,out] dependent Dependent resource in colocation * \param[in] primary Primary resource in colocation * \param[in] colocation Colocation constraint to apply * \param[in] for_dependent true if called on behalf of dependent */ void pcmk__bundle_apply_coloc_score(pe_resource_t *dependent, const pe_resource_t *primary, const pcmk__colocation_t *colocation, bool for_dependent) { struct coloc_data coloc_data = { colocation, dependent, NULL }; /* This should never be called for the bundle itself as a dependent. * Instead, we add its colocation constraints to its containers and bundled * primitive and call the apply_coloc_score() method for them as dependents. */ CRM_ASSERT((primary != NULL) && (primary->variant == pcmk_rsc_variant_bundle) && (dependent != NULL) && (dependent->variant == pcmk_rsc_variant_primitive) && (colocation != NULL) && !for_dependent); if (pcmk_is_set(primary->flags, pcmk_rsc_unassigned)) { pe_rsc_trace(primary, "Skipping applying colocation %s " "because %s is still provisional", colocation->id, primary->id); return; } pe_rsc_trace(primary, "Applying colocation %s (%s with %s at %s)", colocation->id, dependent->id, primary->id, pcmk_readable_score(colocation->score)); /* If the constraint dependent is a clone or bundle, "dependent" here is one * of its instances. Look for a compatible instance of this bundle. */ if (colocation->dependent->variant > pcmk_rsc_variant_group) { const pe_resource_t *primary_container = compatible_container(dependent, primary); if (primary_container != NULL) { // Success, we found one pe_rsc_debug(primary, "Pairing %s with %s", dependent->id, primary_container->id); dependent->cmds->apply_coloc_score(dependent, primary_container, colocation, true); } else if (colocation->score >= INFINITY) { // Failure, and it's fatal crm_notice("%s cannot run because there is no compatible " "instance of %s to colocate with", dependent->id, primary->id); pcmk__assign_resource(dependent, NULL, true, true); } else { // Failure, but we can ignore it pe_rsc_debug(primary, "%s cannot be colocated with any instance of %s", dependent->id, primary->id); } return; } pe__foreach_const_bundle_replica(primary, replica_apply_coloc_score, &coloc_data); if (colocation->score >= INFINITY) { pcmk__colocation_intersect_nodes(dependent, primary, colocation, coloc_data.container_hosts, false); } g_list_free(coloc_data.container_hosts); } // Bundle implementation of resource_alloc_functions_t:with_this_colocations() void pcmk__with_bundle_colocations(const pe_resource_t *rsc, const pe_resource_t *orig_rsc, GList **list) { const pe_resource_t *bundled_rsc = NULL; CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle) && (orig_rsc != NULL) && (list != NULL)); // The bundle itself and its containers always get its colocations if ((orig_rsc == rsc) || pcmk_is_set(orig_rsc->flags, pcmk_rsc_replica_container)) { pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc); return; } /* The bundled resource gets the colocations if it's promotable and we've * begun choosing roles */ bundled_rsc = pe__bundled_resource(rsc); if ((bundled_rsc == NULL) || !pcmk_is_set(bundled_rsc->flags, pcmk_rsc_promotable) || (pe__const_top_resource(orig_rsc, false) != bundled_rsc)) { return; } if (orig_rsc == bundled_rsc) { if (pe__clone_flag_is_set(orig_rsc, pe__clone_promotion_constrained)) { /* orig_rsc is the clone and we're setting roles (or have already * done so) */ pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc); } } else if (!pcmk_is_set(orig_rsc->flags, pcmk_rsc_unassigned)) { /* orig_rsc is an instance and is already assigned. If something * requests colocations for orig_rsc now, it's for setting roles. */ pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc); } } // Bundle implementation of resource_alloc_functions_t:this_with_colocations() void pcmk__bundle_with_colocations(const pe_resource_t *rsc, const pe_resource_t *orig_rsc, GList **list) { const pe_resource_t *bundled_rsc = NULL; CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle) && (orig_rsc != NULL) && (list != NULL)); // The bundle itself and its containers always get its colocations if ((orig_rsc == rsc) || pcmk_is_set(orig_rsc->flags, pcmk_rsc_replica_container)) { pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc); return; } /* The bundled resource gets the colocations if it's promotable and we've * begun choosing roles */ bundled_rsc = pe__bundled_resource(rsc); if ((bundled_rsc == NULL) || !pcmk_is_set(bundled_rsc->flags, pcmk_rsc_promotable) || (pe__const_top_resource(orig_rsc, false) != bundled_rsc)) { return; } if (orig_rsc == bundled_rsc) { if (pe__clone_flag_is_set(orig_rsc, pe__clone_promotion_constrained)) { /* orig_rsc is the clone and we're setting roles (or have already * done so) */ pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc); } } else if (!pcmk_is_set(orig_rsc->flags, pcmk_rsc_unassigned)) { /* orig_rsc is an instance and is already assigned. If something * requests colocations for orig_rsc now, it's for setting roles. */ pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc); } } /*! * \internal * \brief Return action flags for a given bundle resource action * * \param[in,out] action Bundle resource action to get flags for * \param[in] node If not NULL, limit effects to this node * * \return Flags appropriate to \p action on \p node */ uint32_t pcmk__bundle_action_flags(pe_action_t *action, const pe_node_t *node) { GList *containers = NULL; uint32_t flags = 0; pe_resource_t *bundled_resource = NULL; CRM_ASSERT((action != NULL) && (action->rsc != NULL) && (action->rsc->variant == pcmk_rsc_variant_bundle)); bundled_resource = pe__bundled_resource(action->rsc); if (bundled_resource != NULL) { // Clone actions are done on the bundled clone resource, not container switch (get_complex_task(bundled_resource, action->task)) { case pcmk_action_unspecified: case pcmk_action_notify: case pcmk_action_notified: case pcmk_action_promote: case pcmk_action_promoted: case pcmk_action_demote: case pcmk_action_demoted: return pcmk__collective_action_flags(action, bundled_resource->children, node); default: break; } } containers = pe__bundle_containers(action->rsc); flags = pcmk__collective_action_flags(action, containers, node); g_list_free(containers); return flags; } /*! * \internal * \brief Apply a location constraint to a bundle replica * * \param[in,out] replica Replica to apply constraint to * \param[in,out] user_data Location constraint to apply * * \return true (to indicate that any further replicas should be processed) */ static bool apply_location_to_replica(pe__bundle_replica_t *replica, void *user_data) { pe__location_t *location = user_data; if (replica->container != NULL) { replica->container->cmds->apply_location(replica->container, location); } if (replica->ip != NULL) { replica->ip->cmds->apply_location(replica->ip, location); } return true; } /*! * \internal * \brief Apply a location constraint to a bundle resource's allowed node scores * * \param[in,out] rsc Bundle resource to apply constraint to * \param[in,out] location Location constraint to apply */ void pcmk__bundle_apply_location(pe_resource_t *rsc, pe__location_t *location) { pe_resource_t *bundled_resource = NULL; CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle) && (location != NULL)); pcmk__apply_location(rsc, location); pe__foreach_bundle_replica(rsc, apply_location_to_replica, location); bundled_resource = pe__bundled_resource(rsc); if ((bundled_resource != NULL) && ((location->role_filter == pcmk_role_unpromoted) || (location->role_filter == pcmk_role_promoted))) { bundled_resource->cmds->apply_location(bundled_resource, location); bundled_resource->rsc_location = g_list_prepend( bundled_resource->rsc_location, location); } } #define XPATH_REMOTE "//nvpair[@name='" XML_RSC_ATTR_REMOTE_RA_ADDR "']" /*! * \internal * \brief Add a bundle replica's actions to transition graph * * \param[in,out] replica Replica to add to graph * \param[in] user_data Bundle that replica belongs to (for logging only) * * \return true (to indicate that any further replicas should be processed) */ static bool add_replica_actions_to_graph(pe__bundle_replica_t *replica, void *user_data) { if ((replica->remote != NULL) && (replica->container != NULL) && pe__bundle_needs_remote_name(replica->remote)) { /* REMOTE_CONTAINER_HACK: Allow remote nodes to run containers that * run pacemaker-remoted inside, without needing a separate IP for * the container. This is done by configuring the inner remote's * connection host as the magic string "#uname", then * replacing it with the underlying host when needed. */ xmlNode *nvpair = get_xpath_object(XPATH_REMOTE, replica->remote->xml, LOG_ERR); const char *calculated_addr = NULL; // Replace the value in replica->remote->xml (if appropriate) calculated_addr = pe__add_bundle_remote_name(replica->remote, replica->remote->cluster, nvpair, "value"); if (calculated_addr != NULL) { /* Since this is for the bundle as a resource, and not any * particular action, replace the value in the default * parameters (not evaluated for node). create_graph_action() * will grab it from there to replace it in node-evaluated * parameters. */ GHashTable *params = pe_rsc_params(replica->remote, NULL, replica->remote->cluster); g_hash_table_replace(params, strdup(XML_RSC_ATTR_REMOTE_RA_ADDR), strdup(calculated_addr)); } else { pe_resource_t *bundle = user_data; /* The only way to get here is if the remote connection is * neither currently running nor scheduled to run. That means we * won't be doing any operations that require addr (only start * requires it; we additionally use it to compare digests when * unpacking status, promote, and migrate_from history, but * that's already happened by this point). */ pe_rsc_info(bundle, "Unable to determine address for bundle %s " "remote connection", bundle->id); } } if (replica->ip != NULL) { replica->ip->cmds->add_actions_to_graph(replica->ip); } if (replica->container != NULL) { replica->container->cmds->add_actions_to_graph(replica->container); } if (replica->remote != NULL) { replica->remote->cmds->add_actions_to_graph(replica->remote); } return true; } /*! * \internal * \brief Add a bundle resource's actions to the transition graph * * \param[in,out] rsc Bundle resource whose actions should be added */ void pcmk__bundle_add_actions_to_graph(pe_resource_t *rsc) { pe_resource_t *bundled_resource = NULL; CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle)); bundled_resource = pe__bundled_resource(rsc); if (bundled_resource != NULL) { bundled_resource->cmds->add_actions_to_graph(bundled_resource); } pe__foreach_bundle_replica(rsc, add_replica_actions_to_graph, rsc); } struct probe_data { pe_resource_t *bundle; // Bundle being probed pe_node_t *node; // Node to create probes on bool any_created; // Whether any probes have been created }; /*! * \internal * \brief Order a bundle replica's start after another replica's probe * * \param[in,out] replica Replica to order start for * \param[in,out] user_data Replica with probe to order after * * \return true (to indicate that any further replicas should be processed) */ static bool order_replica_start_after(pe__bundle_replica_t *replica, void *user_data) { pe__bundle_replica_t *probed_replica = user_data; if ((replica == probed_replica) || (replica->container == NULL)) { return true; } pcmk__new_ordering(probed_replica->container, pcmk__op_key(probed_replica->container->id, PCMK_ACTION_MONITOR, 0), NULL, replica->container, pcmk__op_key(replica->container->id, PCMK_ACTION_START, 0), NULL, pcmk__ar_ordered|pcmk__ar_if_on_same_node, replica->container->cluster); return true; } /*! * \internal * \brief Create probes for a bundle replica's resources * * \param[in,out] replica Replica to create probes for * \param[in,out] user_data struct probe_data * * \return true (to indicate that any further replicas should be processed) */ static bool create_replica_probes(pe__bundle_replica_t *replica, void *user_data) { struct probe_data *probe_data = user_data; if ((replica->ip != NULL) && replica->ip->cmds->create_probe(replica->ip, probe_data->node)) { probe_data->any_created = true; } if ((replica->child != NULL) && pe__same_node(probe_data->node, replica->node) && replica->child->cmds->create_probe(replica->child, probe_data->node)) { probe_data->any_created = true; } if ((replica->container != NULL) && replica->container->cmds->create_probe(replica->container, probe_data->node)) { probe_data->any_created = true; /* If we're limited to one replica per host (due to * the lack of an IP range probably), then we don't * want any of our peer containers starting until * we've established that no other copies are already * running. * * Partly this is to ensure that the maximum replicas per host is * observed, but also to ensure that the containers * don't fail to start because the necessary port * mappings (which won't include an IP for uniqueness) * are already taken */ if (probe_data->bundle->fns->max_per_node(probe_data->bundle) == 1) { pe__foreach_bundle_replica(probe_data->bundle, order_replica_start_after, replica); } } if ((replica->container != NULL) && (replica->remote != NULL) && replica->remote->cmds->create_probe(replica->remote, probe_data->node)) { /* Do not probe the remote resource until we know where the container is * running. This is required for REMOTE_CONTAINER_HACK to correctly * probe remote resources. */ char *probe_uuid = pcmk__op_key(replica->remote->id, PCMK_ACTION_MONITOR, 0); pe_action_t *probe = find_first_action(replica->remote->actions, probe_uuid, NULL, probe_data->node); free(probe_uuid); if (probe != NULL) { probe_data->any_created = true; pe_rsc_trace(probe_data->bundle, "Ordering %s probe on %s", replica->remote->id, pe__node_name(probe_data->node)); pcmk__new_ordering(replica->container, pcmk__op_key(replica->container->id, PCMK_ACTION_START, 0), NULL, replica->remote, NULL, probe, pcmk__ar_nested_remote_probe, probe_data->bundle->cluster); } } return true; } /*! * \internal * * \brief Schedule any probes needed for a bundle resource on a node * * \param[in,out] rsc Bundle resource to create probes for * \param[in,out] node Node to create probe on * * \return true if any probe was created, otherwise false */ bool pcmk__bundle_create_probe(pe_resource_t *rsc, pe_node_t *node) { struct probe_data probe_data = { rsc, node, false }; CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle)); pe__foreach_bundle_replica(rsc, create_replica_probes, &probe_data); return probe_data.any_created; } /*! * \internal * \brief Output actions for one bundle replica * * \param[in,out] replica Replica to output actions for * \param[in] user_data Unused * * \return true (to indicate that any further replicas should be processed) */ static bool output_replica_actions(pe__bundle_replica_t *replica, void *user_data) { if (replica->ip != NULL) { replica->ip->cmds->output_actions(replica->ip); } if (replica->container != NULL) { replica->container->cmds->output_actions(replica->container); } if (replica->remote != NULL) { replica->remote->cmds->output_actions(replica->remote); } if (replica->child != NULL) { replica->child->cmds->output_actions(replica->child); } return true; } /*! * \internal * \brief Output a summary of scheduled actions for a bundle resource * * \param[in,out] rsc Bundle resource to output actions for */ void pcmk__output_bundle_actions(pe_resource_t *rsc) { CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle)); pe__foreach_bundle_replica(rsc, output_replica_actions, NULL); } // Bundle implementation of resource_alloc_functions_t:add_utilization() void pcmk__bundle_add_utilization(const pe_resource_t *rsc, const pe_resource_t *orig_rsc, GList *all_rscs, GHashTable *utilization) { pe_resource_t *container = NULL; CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle)); if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) { return; } /* All bundle replicas are identical, so using the utilization of the first * is sufficient for any. Only the implicit container resource can have * utilization values. */ container = pe__first_container(rsc); if (container != NULL) { container->cmds->add_utilization(container, orig_rsc, all_rscs, utilization); } } // Bundle implementation of resource_alloc_functions_t:shutdown_lock() void pcmk__bundle_shutdown_lock(pe_resource_t *rsc) { CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle)); // Bundles currently don't support shutdown locks } diff --git a/lib/pacemaker/pcmk_sched_fencing.c b/lib/pacemaker/pcmk_sched_fencing.c index 50bf9b75af..c9239052b3 100644 --- a/lib/pacemaker/pcmk_sched_fencing.c +++ b/lib/pacemaker/pcmk_sched_fencing.c @@ -1,500 +1,500 @@ /* * Copyright 2004-2023 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include "libpacemaker_private.h" /*! * \internal * \brief Check whether a resource is known on a particular node * * \param[in] rsc Resource to check * \param[in] node Node to check * * \return TRUE if resource (or parent if an anonymous clone) is known */ static bool rsc_is_known_on(const pe_resource_t *rsc, const pe_node_t *node) { if (g_hash_table_lookup(rsc->known_on, node->details->id) != NULL) { return TRUE; } else if ((rsc->variant == pcmk_rsc_variant_primitive) && pe_rsc_is_anon_clone(rsc->parent) && (g_hash_table_lookup(rsc->parent->known_on, node->details->id) != NULL)) { /* We check only the parent, not the uber-parent, because we cannot * assume that the resource is known if it is in an anonymously cloned * group (which may be only partially known). */ return TRUE; } return FALSE; } /*! * \internal * \brief Order a resource's start and promote actions relative to fencing * * \param[in,out] rsc Resource to be ordered * \param[in,out] stonith_op Fence action */ static void order_start_vs_fencing(pe_resource_t *rsc, pe_action_t *stonith_op) { pe_node_t *target; CRM_CHECK(stonith_op && stonith_op->node, return); target = stonith_op->node; for (GList *iter = rsc->actions; iter != NULL; iter = iter->next) { pe_action_t *action = iter->data; switch (action->needs) { case pcmk_requires_nothing: // Anything other than start or promote requires nothing break; case pcmk_requires_fencing: order_actions(stonith_op, action, pcmk__ar_ordered); break; case pcmk_requires_quorum: if (pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_none) && (g_hash_table_lookup(rsc->allowed_nodes, target->details->id) != NULL) && !rsc_is_known_on(rsc, target)) { /* If we don't know the status of the resource on the node * we're about to shoot, we have to assume it may be active * there. Order the resource start after the fencing. This * is analogous to waiting for all the probes for a resource * to complete before starting it. * * The most likely explanation is that the DC died and took * its status with it. */ pe_rsc_debug(rsc, "Ordering %s after %s recovery", action->uuid, pe__node_name(target)); order_actions(stonith_op, action, pcmk__ar_ordered |pcmk__ar_unrunnable_first_blocks); } break; } } } /*! * \internal * \brief Order a resource's stop and demote actions relative to fencing * * \param[in,out] rsc Resource to be ordered * \param[in,out] stonith_op Fence action */ static void order_stop_vs_fencing(pe_resource_t *rsc, pe_action_t *stonith_op) { GList *iter = NULL; GList *action_list = NULL; bool order_implicit = false; pe_resource_t *top = uber_parent(rsc); pe_action_t *parent_stop = NULL; pe_node_t *target; CRM_CHECK(stonith_op && stonith_op->node, return); target = stonith_op->node; /* Get a list of stop actions potentially implied by the fencing */ action_list = pe__resource_actions(rsc, target, PCMK_ACTION_STOP, FALSE); /* If resource requires fencing, implicit actions must occur after fencing. * * Implied stops and demotes of resources running on guest nodes are always * ordered after fencing, even if the resource does not require fencing, * because guest node "fencing" is actually just a resource stop. */ if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing) || pe__is_guest_node(target)) { order_implicit = true; } if (action_list && order_implicit) { parent_stop = find_first_action(top->actions, NULL, PCMK_ACTION_STOP, NULL); } for (iter = action_list; iter != NULL; iter = iter->next) { pe_action_t *action = iter->data; // The stop would never complete, so convert it into a pseudo-action. pe__set_action_flags(action, pcmk_action_pseudo|pcmk_action_runnable); if (order_implicit) { /* Order the stonith before the parent stop (if any). * * Also order the stonith before the resource stop, unless the * resource is inside a bundle -- that would cause a graph loop. * We can rely on the parent stop's ordering instead. * * User constraints must not order a resource in a guest node * relative to the guest node container resource. The - * pe_order_preserve flag marks constraints as generated by the + * pcmk__ar_guest_allowed flag marks constraints as generated by the * cluster and thus immune to that check (and is irrelevant if * target is not a guest). */ if (!pe_rsc_is_bundled(rsc)) { - order_actions(stonith_op, action, pe_order_preserve); + order_actions(stonith_op, action, pcmk__ar_guest_allowed); } - order_actions(stonith_op, parent_stop, pe_order_preserve); + order_actions(stonith_op, parent_stop, pcmk__ar_guest_allowed); } if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) { crm_notice("Stop of failed resource %s is implicit %s %s is fenced", rsc->id, (order_implicit? "after" : "because"), pe__node_name(target)); } else { crm_info("%s is implicit %s %s is fenced", action->uuid, (order_implicit? "after" : "because"), pe__node_name(target)); } if (pcmk_is_set(rsc->flags, pcmk_rsc_notify)) { pe__order_notifs_after_fencing(action, rsc, stonith_op); } #if 0 /* It might be a good idea to stop healthy resources on a node about to * be fenced, when possible. * * However, fencing must be done before a failed resource's * (pseudo-)stop action, so that could create a loop. For example, given * a group of A and B running on node N with a failed stop of B: * * fence N -> stop B (pseudo-op) -> stop A -> fence N * * The block below creates the stop A -> fence N ordering and therefore * must (at least for now) be disabled. Instead, run the block above and * treat all resources on N as B would be (i.e., as a pseudo-op after * the fencing). * * @TODO Maybe break the "A requires B" dependency in * pcmk__update_action_for_orderings() and use this block for healthy * resources instead of the above. */ crm_info("Moving healthy resource %s off %s before fencing", rsc->id, pe__node_name(node)); pcmk__new_ordering(rsc, stop_key(rsc), NULL, NULL, strdup(PCMK_ACTION_STONITH), stonith_op, pcmk__ar_ordered, rsc->cluster); #endif } g_list_free(action_list); /* Get a list of demote actions potentially implied by the fencing */ action_list = pe__resource_actions(rsc, target, PCMK_ACTION_DEMOTE, FALSE); for (iter = action_list; iter != NULL; iter = iter->next) { pe_action_t *action = iter->data; if (!(action->node->details->online) || action->node->details->unclean || pcmk_is_set(rsc->flags, pcmk_rsc_failed)) { if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) { pe_rsc_info(rsc, "Demote of failed resource %s is implicit " "after %s is fenced", rsc->id, pe__node_name(target)); } else { pe_rsc_info(rsc, "%s is implicit after %s is fenced", action->uuid, pe__node_name(target)); } /* The demote would never complete and is now implied by the * fencing, so convert it into a pseudo-action. */ pe__set_action_flags(action, pcmk_action_pseudo|pcmk_action_runnable); if (pe_rsc_is_bundled(rsc)) { // Recovery will be ordered as usual after parent's implied stop } else if (order_implicit) { order_actions(stonith_op, action, - pe_order_preserve|pcmk__ar_ordered); + pcmk__ar_guest_allowed|pcmk__ar_ordered); } } } g_list_free(action_list); } /*! * \internal * \brief Order resource actions properly relative to fencing * * \param[in,out] rsc Resource whose actions should be ordered * \param[in,out] stonith_op Fencing operation to be ordered against */ static void rsc_stonith_ordering(pe_resource_t *rsc, pe_action_t *stonith_op) { if (rsc->children) { for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { pe_resource_t *child_rsc = iter->data; rsc_stonith_ordering(child_rsc, stonith_op); } } else if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) { pe_rsc_trace(rsc, "Skipping fencing constraints for unmanaged resource: %s", rsc->id); } else { order_start_vs_fencing(rsc, stonith_op); order_stop_vs_fencing(rsc, stonith_op); } } /*! * \internal * \brief Order all actions appropriately relative to a fencing operation * * Ensure start operations of affected resources are ordered after fencing, * imply stop and demote operations of affected resources by marking them as * pseudo-actions, etc. * * \param[in,out] stonith_op Fencing operation * \param[in,out] data_set Working set of cluster */ void pcmk__order_vs_fence(pe_action_t *stonith_op, pe_working_set_t *data_set) { CRM_CHECK(stonith_op && data_set, return); for (GList *r = data_set->resources; r != NULL; r = r->next) { rsc_stonith_ordering((pe_resource_t *) r->data, stonith_op); } } /*! * \internal * \brief Order an action after unfencing * * \param[in] rsc Resource that action is for * \param[in,out] node Node that action is on * \param[in,out] action Action to be ordered after unfencing * \param[in] order Ordering flags */ void pcmk__order_vs_unfence(const pe_resource_t *rsc, pe_node_t *node, pe_action_t *action, enum pcmk__action_relation_flags order) { /* When unfencing is in use, we order unfence actions before any probe or * start of resources that require unfencing, and also of fence devices. * * This might seem to violate the principle that fence devices require * only quorum. However, fence agents that unfence often don't have enough * information to even probe or start unless the node is first unfenced. */ if ((pcmk_is_set(rsc->flags, pcmk_rsc_fence_device) && pcmk_is_set(rsc->cluster->flags, pcmk_sched_enable_unfencing)) || pcmk_is_set(rsc->flags, pcmk_rsc_needs_unfencing)) { /* Start with an optional ordering. Requiring unfencing would result in * the node being unfenced, and all its resources being stopped, * whenever a new resource is added -- which would be highly suboptimal. */ pe_action_t *unfence = pe_fence_op(node, PCMK_ACTION_ON, TRUE, NULL, FALSE, node->details->data_set); order_actions(unfence, action, order); if (!pcmk__node_unfenced(node)) { // But unfencing is required if it has never been done char *reason = crm_strdup_printf("required by %s %s", rsc->id, action->task); trigger_unfencing(NULL, node, reason, NULL, node->details->data_set); free(reason); } } } /*! * \internal * \brief Create pseudo-op for guest node fence, and order relative to it * * \param[in,out] node Guest node to fence */ void pcmk__fence_guest(pe_node_t *node) { pe_resource_t *container = NULL; pe_action_t *stop = NULL; pe_action_t *stonith_op = NULL; /* The fence action is just a label; we don't do anything differently for * off vs. reboot. We specify it explicitly, rather than let it default to * cluster's default action, because we are not _initiating_ fencing -- we * are creating a pseudo-event to describe fencing that is already occurring * by other means (container recovery). */ const char *fence_action = PCMK_ACTION_OFF; CRM_ASSERT(node != NULL); /* Check whether guest's container resource has any explicit stop or * start (the stop may be implied by fencing of the guest's host). */ container = node->details->remote_rsc->container; if (container) { stop = find_first_action(container->actions, NULL, PCMK_ACTION_STOP, NULL); if (find_first_action(container->actions, NULL, PCMK_ACTION_START, NULL)) { fence_action = PCMK_ACTION_REBOOT; } } /* Create a fence pseudo-event, so we have an event to order actions * against, and the controller can always detect it. */ stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", FALSE, node->details->data_set); pe__set_action_flags(stonith_op, pcmk_action_pseudo|pcmk_action_runnable); /* We want to imply stops/demotes after the guest is stopped, not wait until * it is restarted, so we always order pseudo-fencing after stop, not start * (even though start might be closer to what is done for a real reboot). */ if ((stop != NULL) && pcmk_is_set(stop->flags, pcmk_action_pseudo)) { pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, FALSE, node->details->data_set); crm_info("Implying guest %s is down (action %d) after %s fencing", pe__node_name(node), stonith_op->id, pe__node_name(stop->node)); order_actions(parent_stonith_op, stonith_op, pcmk__ar_unrunnable_first_blocks |pcmk__ar_first_implies_then); } else if (stop) { order_actions(stop, stonith_op, pcmk__ar_unrunnable_first_blocks |pcmk__ar_first_implies_then); crm_info("Implying guest %s is down (action %d) " "after container %s is stopped (action %d)", pe__node_name(node), stonith_op->id, container->id, stop->id); } else { /* If we're fencing the guest node but there's no stop for the guest * resource, we must think the guest is already stopped. However, we may * think so because its resource history was just cleaned. To avoid * unnecessarily considering the guest node down if it's really up, * order the pseudo-fencing after any stop of the connection resource, * which will be ordered after any container (re-)probe. */ stop = find_first_action(node->details->remote_rsc->actions, NULL, PCMK_ACTION_STOP, NULL); if (stop) { order_actions(stop, stonith_op, pcmk__ar_ordered); crm_info("Implying guest %s is down (action %d) " "after connection is stopped (action %d)", pe__node_name(node), stonith_op->id, stop->id); } else { /* Not sure why we're fencing, but everything must already be * cleanly stopped. */ crm_info("Implying guest %s is down (action %d) ", pe__node_name(node), stonith_op->id); } } // Order/imply other actions relative to pseudo-fence as with real fence pcmk__order_vs_fence(stonith_op, node->details->data_set); } /*! * \internal * \brief Check whether node has already been unfenced * * \param[in] node Node to check * * \return true if node has a nonzero #node-unfenced attribute (or none), * otherwise false */ bool pcmk__node_unfenced(const pe_node_t *node) { const char *unfenced = pe_node_attribute_raw(node, CRM_ATTR_UNFENCED); return !pcmk__str_eq(unfenced, "0", pcmk__str_null_matches); } /*! * \internal * \brief Order a resource's start and stop relative to unfencing of a node * * \param[in,out] data Node that could be unfenced * \param[in,out] user_data Resource to order */ void pcmk__order_restart_vs_unfence(gpointer data, gpointer user_data) { pe_node_t *node = (pe_node_t *) data; pe_resource_t *rsc = (pe_resource_t *) user_data; pe_action_t *unfence = pe_fence_op(node, PCMK_ACTION_ON, true, NULL, false, rsc->cluster); crm_debug("Ordering any stops of %s before %s, and any starts after", rsc->id, unfence->uuid); /* * It would be more efficient to order clone resources once, * rather than order each instance, but ordering the instance * allows us to avoid unnecessary dependencies that might conflict * with user constraints. * * @TODO: This constraint can still produce a transition loop if the * resource has a stop scheduled on the node being unfenced, and * there is a user ordering constraint to start some other resource * (which will be ordered after the unfence) before stopping this * resource. An example is "start some slow-starting cloned service * before stopping an associated virtual IP that may be moving to * it": * stop this -> unfencing -> start that -> stop this */ pcmk__new_ordering(rsc, stop_key(rsc), NULL, NULL, strdup(unfence->uuid), unfence, pcmk__ar_ordered|pcmk__ar_if_on_same_node, rsc->cluster); pcmk__new_ordering(NULL, strdup(unfence->uuid), unfence, rsc, start_key(rsc), NULL, pcmk__ar_first_implies_same_node_then |pcmk__ar_if_on_same_node, rsc->cluster); } diff --git a/lib/pacemaker/pcmk_sched_ordering.c b/lib/pacemaker/pcmk_sched_ordering.c index 3800bb2ade..b056e8f272 100644 --- a/lib/pacemaker/pcmk_sched_ordering.c +++ b/lib/pacemaker/pcmk_sched_ordering.c @@ -1,1499 +1,1499 @@ /* * Copyright 2004-2023 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include // PRIx32 #include #include #include #include #include "libpacemaker_private.h" enum pe_order_kind { pe_order_kind_optional, pe_order_kind_mandatory, pe_order_kind_serialize, }; enum ordering_symmetry { ordering_asymmetric, // the only relation in an asymmetric ordering ordering_symmetric, // the normal relation in a symmetric ordering ordering_symmetric_inverse, // the inverse relation in a symmetric ordering }; #define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do { \ __rsc = pcmk__find_constraint_resource(data_set->resources, \ __name); \ if (__rsc == NULL) { \ pcmk__config_err("%s: No resource found for %s", __set, __name);\ return pcmk_rc_unpack_error; \ } \ } while (0) static const char * invert_action(const char *action) { if (pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) { return PCMK_ACTION_STOP; } else if (pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) { return PCMK_ACTION_START; } else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTE, pcmk__str_none)) { return PCMK_ACTION_DEMOTE; } else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTE, pcmk__str_none)) { return PCMK_ACTION_PROMOTE; } else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTED, pcmk__str_none)) { return PCMK_ACTION_DEMOTED; } else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTED, pcmk__str_none)) { return PCMK_ACTION_PROMOTED; } else if (pcmk__str_eq(action, PCMK_ACTION_RUNNING, pcmk__str_none)) { return PCMK_ACTION_STOPPED; } else if (pcmk__str_eq(action, PCMK_ACTION_STOPPED, pcmk__str_none)) { return PCMK_ACTION_RUNNING; } crm_warn("Unknown action '%s' specified in order constraint", action); return NULL; } static enum pe_order_kind get_ordering_type(const xmlNode *xml_obj) { enum pe_order_kind kind_e = pe_order_kind_mandatory; const char *kind = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND); if (kind == NULL) { const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE); kind_e = pe_order_kind_mandatory; if (score) { // @COMPAT deprecated informally since 1.0.7, formally since 2.0.1 int score_i = char2score(score); if (score_i == 0) { kind_e = pe_order_kind_optional; } pe_warn_once(pe_wo_order_score, "Support for 'score' in rsc_order is deprecated " "and will be removed in a future release " "(use 'kind' instead)"); } } else if (pcmk__str_eq(kind, "Mandatory", pcmk__str_none)) { kind_e = pe_order_kind_mandatory; } else if (pcmk__str_eq(kind, "Optional", pcmk__str_none)) { kind_e = pe_order_kind_optional; } else if (pcmk__str_eq(kind, "Serialize", pcmk__str_none)) { kind_e = pe_order_kind_serialize; } else { pcmk__config_err("Resetting '" XML_ORDER_ATTR_KIND "' for constraint " "%s to 'Mandatory' because '%s' is not valid", pcmk__s(ID(xml_obj), "missing ID"), kind); } return kind_e; } /*! * \internal * \brief Get ordering symmetry from XML * * \param[in] xml_obj Ordering XML * \param[in] parent_kind Default ordering kind * \param[in] parent_symmetrical_s Parent element's symmetrical setting, if any * * \retval ordering_symmetric Ordering is symmetric * \retval ordering_asymmetric Ordering is asymmetric */ static enum ordering_symmetry get_ordering_symmetry(const xmlNode *xml_obj, enum pe_order_kind parent_kind, const char *parent_symmetrical_s) { int rc = pcmk_rc_ok; bool symmetric = false; enum pe_order_kind kind = parent_kind; // Default to parent's kind // Check ordering XML for explicit kind if ((crm_element_value(xml_obj, XML_ORDER_ATTR_KIND) != NULL) || (crm_element_value(xml_obj, XML_RULE_ATTR_SCORE) != NULL)) { kind = get_ordering_type(xml_obj); } // Check ordering XML (and parent) for explicit symmetrical setting rc = pcmk__xe_get_bool_attr(xml_obj, XML_CONS_ATTR_SYMMETRICAL, &symmetric); if (rc != pcmk_rc_ok && parent_symmetrical_s != NULL) { symmetric = crm_is_true(parent_symmetrical_s); rc = pcmk_rc_ok; } if (rc == pcmk_rc_ok) { if (symmetric) { if (kind == pe_order_kind_serialize) { pcmk__config_warn("Ignoring " XML_CONS_ATTR_SYMMETRICAL " for '%s' because not valid with " XML_ORDER_ATTR_KIND " of 'Serialize'", ID(xml_obj)); } else { return ordering_symmetric; } } return ordering_asymmetric; } // Use default symmetry if (kind == pe_order_kind_serialize) { return ordering_asymmetric; } return ordering_symmetric; } /*! * \internal * \brief Get ordering flags appropriate to ordering kind * * \param[in] kind Ordering kind * \param[in] first Action name for 'first' action * \param[in] symmetry This ordering's symmetry role * * \return Minimal ordering flags appropriate to \p kind */ static uint32_t ordering_flags_for_kind(enum pe_order_kind kind, const char *first, enum ordering_symmetry symmetry) { uint32_t flags = pcmk__ar_none; // so we trace-log all flags set switch (kind) { case pe_order_kind_optional: pe__set_order_flags(flags, pcmk__ar_ordered); break; case pe_order_kind_serialize: /* This flag is not used anywhere directly but means the relation * will not match an equality comparison against pcmk__ar_none or * pcmk__ar_ordered. */ pe__set_order_flags(flags, pcmk__ar_serialize); break; case pe_order_kind_mandatory: pe__set_order_flags(flags, pcmk__ar_ordered); switch (symmetry) { case ordering_asymmetric: pe__set_order_flags(flags, pcmk__ar_asymmetric); break; case ordering_symmetric: pe__set_order_flags(flags, pcmk__ar_first_implies_then); if (pcmk__strcase_any_of(first, PCMK_ACTION_START, PCMK_ACTION_PROMOTE, NULL)) { pe__set_order_flags(flags, pcmk__ar_unrunnable_first_blocks); } break; case ordering_symmetric_inverse: pe__set_order_flags(flags, pcmk__ar_then_implies_first); break; } break; } return flags; } /*! * \internal * \brief Find resource corresponding to ID specified in ordering * * \param[in] xml Ordering XML * \param[in] resource_attr XML attribute name for resource ID * \param[in] instance_attr XML attribute name for instance number. * This option is deprecated and will be removed in a * future release. * \param[in] data_set Cluster working set * * \return Resource corresponding to \p id, or NULL if none */ static pe_resource_t * get_ordering_resource(const xmlNode *xml, const char *resource_attr, const char *instance_attr, const pe_working_set_t *data_set) { // @COMPAT: instance_attr and instance_id variables deprecated since 2.1.5 pe_resource_t *rsc = NULL; const char *rsc_id = crm_element_value(xml, resource_attr); const char *instance_id = crm_element_value(xml, instance_attr); if (rsc_id == NULL) { pcmk__config_err("Ignoring constraint '%s' without %s", ID(xml), resource_attr); return NULL; } rsc = pcmk__find_constraint_resource(data_set->resources, rsc_id); if (rsc == NULL) { pcmk__config_err("Ignoring constraint '%s' because resource '%s' " "does not exist", ID(xml), rsc_id); return NULL; } if (instance_id != NULL) { pe_warn_once(pe_wo_order_inst, "Support for " XML_ORDER_ATTR_FIRST_INSTANCE " and " XML_ORDER_ATTR_THEN_INSTANCE " is deprecated and will be " "removed in a future release."); if (!pe_rsc_is_clone(rsc)) { pcmk__config_err("Ignoring constraint '%s' because resource '%s' " "is not a clone but instance '%s' was requested", ID(xml), rsc_id, instance_id); return NULL; } rsc = find_clone_instance(rsc, instance_id); if (rsc == NULL) { pcmk__config_err("Ignoring constraint '%s' because resource '%s' " "does not have an instance '%s'", "'%s'", ID(xml), rsc_id, instance_id); return NULL; } } return rsc; } /*! * \internal * \brief Determine minimum number of 'first' instances required in ordering * * \param[in] rsc 'First' resource in ordering * \param[in] xml Ordering XML * * \return Minimum 'first' instances required (or 0 if not applicable) */ static int get_minimum_first_instances(const pe_resource_t *rsc, const xmlNode *xml) { const char *clone_min = NULL; bool require_all = false; if (!pe_rsc_is_clone(rsc)) { return 0; } clone_min = g_hash_table_lookup(rsc->meta, PCMK_META_CLONE_MIN); if (clone_min != NULL) { int clone_min_int = 0; pcmk__scan_min_int(clone_min, &clone_min_int, 0); return clone_min_int; } /* @COMPAT 1.1.13: * require-all=false is deprecated equivalent of clone-min=1 */ if (pcmk__xe_get_bool_attr(xml, "require-all", &require_all) != ENODATA) { pe_warn_once(pe_wo_require_all, "Support for require-all in ordering constraints " "is deprecated and will be removed in a future release" " (use clone-min clone meta-attribute instead)"); if (!require_all) { return 1; } } return 0; } /*! * \internal * \brief Create orderings for a constraint with clone-min > 0 * * \param[in] id Ordering ID * \param[in,out] rsc_first 'First' resource in ordering (a clone) * \param[in] action_first 'First' action in ordering * \param[in] rsc_then 'Then' resource in ordering * \param[in] action_then 'Then' action in ordering * \param[in] flags Ordering flags * \param[in] clone_min Minimum required instances of 'first' */ static void clone_min_ordering(const char *id, pe_resource_t *rsc_first, const char *action_first, pe_resource_t *rsc_then, const char *action_then, uint32_t flags, int clone_min) { // Create a pseudo-action for when the minimum instances are active char *task = crm_strdup_printf(PCMK_ACTION_CLONE_ONE_OR_MORE ":%s", id); pe_action_t *clone_min_met = get_pseudo_op(task, rsc_first->cluster); free(task); /* Require the pseudo-action to have the required number of actions to be * considered runnable before allowing the pseudo-action to be runnable. */ clone_min_met->required_runnable_before = clone_min; pe__set_action_flags(clone_min_met, pcmk_action_min_runnable); // Order the actions for each clone instance before the pseudo-action for (GList *iter = rsc_first->children; iter != NULL; iter = iter->next) { pe_resource_t *child = iter->data; pcmk__new_ordering(child, pcmk__op_key(child->id, action_first, 0), NULL, NULL, NULL, clone_min_met, pcmk__ar_min_runnable |pcmk__ar_first_implies_then_graphed, rsc_first->cluster); } // Order "then" action after the pseudo-action (if runnable) pcmk__new_ordering(NULL, NULL, clone_min_met, rsc_then, pcmk__op_key(rsc_then->id, action_then, 0), NULL, flags|pcmk__ar_unrunnable_first_blocks, rsc_first->cluster); } /*! * \internal * \brief Update ordering flags for restart-type=restart * * \param[in] rsc 'Then' resource in ordering * \param[in] kind Ordering kind * \param[in] flag Ordering flag to set (when applicable) * \param[in,out] flags Ordering flag set to update * * \compat The restart-type resource meta-attribute is deprecated. Eventually, * it will be removed, and pe_restart_ignore will be the only behavior, * at which time this can just be removed entirely. */ #define handle_restart_type(rsc, kind, flag, flags) do { \ if (((kind) == pe_order_kind_optional) \ && ((rsc)->restart_type == pe_restart_restart)) { \ pe__set_order_flags((flags), (flag)); \ } \ } while (0) /*! * \internal * \brief Create new ordering for inverse of symmetric constraint * * \param[in] id Ordering ID (for logging only) * \param[in] kind Ordering kind * \param[in] rsc_first 'First' resource in ordering (a clone) * \param[in] action_first 'First' action in ordering * \param[in,out] rsc_then 'Then' resource in ordering * \param[in] action_then 'Then' action in ordering */ static void inverse_ordering(const char *id, enum pe_order_kind kind, pe_resource_t *rsc_first, const char *action_first, pe_resource_t *rsc_then, const char *action_then) { action_then = invert_action(action_then); action_first = invert_action(action_first); if ((action_then == NULL) || (action_first == NULL)) { pcmk__config_warn("Cannot invert constraint '%s' " "(please specify inverse manually)", id); } else { uint32_t flags = ordering_flags_for_kind(kind, action_first, ordering_symmetric_inverse); handle_restart_type(rsc_then, kind, pcmk__ar_then_implies_first, flags); pcmk__order_resource_actions(rsc_then, action_then, rsc_first, action_first, flags); } } static void unpack_simple_rsc_order(xmlNode *xml_obj, pe_working_set_t *data_set) { pe_resource_t *rsc_then = NULL; pe_resource_t *rsc_first = NULL; int min_required_before = 0; enum pe_order_kind kind = pe_order_kind_mandatory; uint32_t flags = pcmk__ar_none; enum ordering_symmetry symmetry; const char *action_then = NULL; const char *action_first = NULL; const char *id = NULL; CRM_CHECK(xml_obj != NULL, return); id = crm_element_value(xml_obj, XML_ATTR_ID); if (id == NULL) { pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID, xml_obj->name); return; } rsc_first = get_ordering_resource(xml_obj, XML_ORDER_ATTR_FIRST, XML_ORDER_ATTR_FIRST_INSTANCE, data_set); if (rsc_first == NULL) { return; } rsc_then = get_ordering_resource(xml_obj, XML_ORDER_ATTR_THEN, XML_ORDER_ATTR_THEN_INSTANCE, data_set); if (rsc_then == NULL) { return; } action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION); if (action_first == NULL) { action_first = PCMK_ACTION_START; } action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION); if (action_then == NULL) { action_then = action_first; } kind = get_ordering_type(xml_obj); symmetry = get_ordering_symmetry(xml_obj, kind, NULL); flags = ordering_flags_for_kind(kind, action_first, symmetry); handle_restart_type(rsc_then, kind, pcmk__ar_first_implies_then, flags); /* If there is a minimum number of instances that must be runnable before * the 'then' action is runnable, we use a pseudo-action for convenience: * minimum number of clone instances have runnable actions -> * pseudo-action is runnable -> dependency is runnable. */ min_required_before = get_minimum_first_instances(rsc_first, xml_obj); if (min_required_before > 0) { clone_min_ordering(id, rsc_first, action_first, rsc_then, action_then, flags, min_required_before); } else { pcmk__order_resource_actions(rsc_first, action_first, rsc_then, action_then, flags); } if (symmetry == ordering_symmetric) { inverse_ordering(id, kind, rsc_first, action_first, rsc_then, action_then); } } /*! * \internal * \brief Create a new ordering between two actions * * \param[in,out] first_rsc Resource for 'first' action (if NULL and * \p first_action is a resource action, that * resource will be used) * \param[in,out] first_action_task Action key for 'first' action (if NULL and * \p first_action is not NULL, its UUID will * be used) * \param[in,out] first_action 'first' action (if NULL, \p first_rsc and * \p first_action_task must be set) * * \param[in] then_rsc Resource for 'then' action (if NULL and * \p then_action is a resource action, that * resource will be used) * \param[in,out] then_action_task Action key for 'then' action (if NULL and * \p then_action is not NULL, its UUID will * be used) * \param[in] then_action 'then' action (if NULL, \p then_rsc and * \p then_action_task must be set) * * \param[in] flags Group of enum pcmk__action_relation_flags * \param[in,out] sched Cluster working set to add ordering to * * \note This function takes ownership of first_action_task and * then_action_task, which do not need to be freed by the caller. */ void pcmk__new_ordering(pe_resource_t *first_rsc, char *first_action_task, pe_action_t *first_action, pe_resource_t *then_rsc, char *then_action_task, pe_action_t *then_action, uint32_t flags, pe_working_set_t *sched) { pe__ordering_t *order = NULL; // One of action or resource must be specified for each side CRM_CHECK(((first_action != NULL) || (first_rsc != NULL)) && ((then_action != NULL) || (then_rsc != NULL)), free(first_action_task); free(then_action_task); return); if ((first_rsc == NULL) && (first_action != NULL)) { first_rsc = first_action->rsc; } if ((then_rsc == NULL) && (then_action != NULL)) { then_rsc = then_action->rsc; } order = calloc(1, sizeof(pe__ordering_t)); CRM_ASSERT(order != NULL); order->id = sched->order_id++; order->flags = flags; order->lh_rsc = first_rsc; order->rh_rsc = then_rsc; order->lh_action = first_action; order->rh_action = then_action; order->lh_action_task = first_action_task; order->rh_action_task = then_action_task; if ((order->lh_action_task == NULL) && (first_action != NULL)) { order->lh_action_task = strdup(first_action->uuid); } if ((order->rh_action_task == NULL) && (then_action != NULL)) { order->rh_action_task = strdup(then_action->uuid); } if ((order->lh_rsc == NULL) && (first_action != NULL)) { order->lh_rsc = first_action->rsc; } if ((order->rh_rsc == NULL) && (then_action != NULL)) { order->rh_rsc = then_action->rsc; } pe_rsc_trace(first_rsc, "Created ordering %d for %s then %s", (sched->order_id - 1), pcmk__s(order->lh_action_task, "an underspecified action"), pcmk__s(order->rh_action_task, "an underspecified action")); sched->ordering_constraints = g_list_prepend(sched->ordering_constraints, order); pcmk__order_migration_equivalents(order); } /*! * \brief Unpack a set in an ordering constraint * * \param[in] set Set XML to unpack * \param[in] parent_kind rsc_order XML "kind" attribute * \param[in] parent_symmetrical_s rsc_order XML "symmetrical" attribute * \param[in,out] data_set Cluster working set * * \return Standard Pacemaker return code */ static int unpack_order_set(const xmlNode *set, enum pe_order_kind parent_kind, const char *parent_symmetrical_s, pe_working_set_t *data_set) { GList *set_iter = NULL; GList *resources = NULL; pe_resource_t *last = NULL; pe_resource_t *resource = NULL; int local_kind = parent_kind; bool sequential = false; uint32_t flags = pcmk__ar_ordered; enum ordering_symmetry symmetry; char *key = NULL; const char *id = ID(set); const char *action = crm_element_value(set, "action"); const char *sequential_s = crm_element_value(set, "sequential"); const char *kind_s = crm_element_value(set, XML_ORDER_ATTR_KIND); if (action == NULL) { action = PCMK_ACTION_START; } if (kind_s) { local_kind = get_ordering_type(set); } if (sequential_s == NULL) { sequential_s = "1"; } sequential = crm_is_true(sequential_s); symmetry = get_ordering_symmetry(set, parent_kind, parent_symmetrical_s); flags = ordering_flags_for_kind(local_kind, action, symmetry); for (const xmlNode *xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF); xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { EXPAND_CONSTRAINT_IDREF(id, resource, ID(xml_rsc)); resources = g_list_append(resources, resource); } if (pcmk__list_of_1(resources)) { crm_trace("Single set: %s", id); goto done; } set_iter = resources; while (set_iter != NULL) { resource = (pe_resource_t *) set_iter->data; set_iter = set_iter->next; key = pcmk__op_key(resource->id, action, 0); if (local_kind == pe_order_kind_serialize) { /* Serialize before everything that comes after */ for (GList *iter = set_iter; iter != NULL; iter = iter->next) { pe_resource_t *then_rsc = iter->data; char *then_key = pcmk__op_key(then_rsc->id, action, 0); pcmk__new_ordering(resource, strdup(key), NULL, then_rsc, then_key, NULL, flags, data_set); } } else if (sequential) { if (last != NULL) { pcmk__order_resource_actions(last, action, resource, action, flags); } last = resource; } free(key); } if (symmetry == ordering_asymmetric) { goto done; } last = NULL; action = invert_action(action); flags = ordering_flags_for_kind(local_kind, action, ordering_symmetric_inverse); set_iter = resources; while (set_iter != NULL) { resource = (pe_resource_t *) set_iter->data; set_iter = set_iter->next; if (sequential) { if (last != NULL) { pcmk__order_resource_actions(resource, action, last, action, flags); } last = resource; } } done: g_list_free(resources); return pcmk_rc_ok; } /*! * \brief Order two resource sets relative to each other * * \param[in] id Ordering ID (for logging) * \param[in] set1 First listed set * \param[in] set2 Second listed set * \param[in] kind Ordering kind * \param[in,out] data_set Cluster working set * \param[in] symmetry Which ordering symmetry applies to this relation * * \return Standard Pacemaker return code */ static int order_rsc_sets(const char *id, const xmlNode *set1, const xmlNode *set2, enum pe_order_kind kind, pe_working_set_t *data_set, enum ordering_symmetry symmetry) { const xmlNode *xml_rsc = NULL; const xmlNode *xml_rsc_2 = NULL; pe_resource_t *rsc_1 = NULL; pe_resource_t *rsc_2 = NULL; const char *action_1 = crm_element_value(set1, "action"); const char *action_2 = crm_element_value(set2, "action"); uint32_t flags = pcmk__ar_none; bool require_all = true; (void) pcmk__xe_get_bool_attr(set1, "require-all", &require_all); if (action_1 == NULL) { action_1 = PCMK_ACTION_START; } if (action_2 == NULL) { action_2 = PCMK_ACTION_START; } if (symmetry == ordering_symmetric_inverse) { action_1 = invert_action(action_1); action_2 = invert_action(action_2); } if (pcmk__str_eq(PCMK_ACTION_STOP, action_1, pcmk__str_none) || pcmk__str_eq(PCMK_ACTION_DEMOTE, action_1, pcmk__str_none)) { /* Assuming: A -> ( B || C) -> D * The one-or-more logic only applies during the start/promote phase. * During shutdown neither B nor can shutdown until D is down, so simply * turn require_all back on. */ require_all = true; } flags = ordering_flags_for_kind(kind, action_1, symmetry); /* If we have an unordered set1, whether it is sequential or not is * irrelevant in regards to set2. */ if (!require_all) { char *task = crm_strdup_printf(PCMK_ACTION_ONE_OR_MORE ":%s", ID(set1)); pe_action_t *unordered_action = get_pseudo_op(task, data_set); free(task); pe__set_action_flags(unordered_action, pcmk_action_min_runnable); for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF); xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); /* Add an ordering constraint between every element in set1 and the * pseudo action. If any action in set1 is runnable the pseudo * action will be runnable. */ pcmk__new_ordering(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0), NULL, NULL, NULL, unordered_action, pcmk__ar_min_runnable |pcmk__ar_first_implies_then_graphed, data_set); } for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF); xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) { EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2)); /* Add an ordering constraint between the pseudo-action and every * element in set2. If the pseudo-action is runnable, every action * in set2 will be runnable. */ pcmk__new_ordering(NULL, NULL, unordered_action, rsc_2, pcmk__op_key(rsc_2->id, action_2, 0), NULL, flags|pcmk__ar_unrunnable_first_blocks, data_set); } return pcmk_rc_ok; } if (pcmk__xe_attr_is_true(set1, "sequential")) { if (symmetry == ordering_symmetric_inverse) { // Get the first one xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF); if (xml_rsc != NULL) { EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); } } else { // Get the last one const char *rid = NULL; for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF); xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { rid = ID(xml_rsc); } EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid); } } if (pcmk__xe_attr_is_true(set2, "sequential")) { if (symmetry == ordering_symmetric_inverse) { // Get the last one const char *rid = NULL; for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF); xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { rid = ID(xml_rsc); } EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid); } else { // Get the first one xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF); if (xml_rsc != NULL) { EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc)); } } } if ((rsc_1 != NULL) && (rsc_2 != NULL)) { pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags); } else if (rsc_1 != NULL) { for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF); xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc)); pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags); } } else if (rsc_2 != NULL) { for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF); xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags); } } else { for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF); xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) { EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc)); for (xmlNode *xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF); xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) { EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2)); pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags); } } } return pcmk_rc_ok; } /*! * \internal * \brief If an ordering constraint uses resource tags, expand them * * \param[in,out] xml_obj Ordering constraint XML * \param[out] expanded_xml Equivalent XML with tags expanded * \param[in] data_set Cluster working set * * \return Standard Pacemaker return code (specifically, pcmk_rc_ok on success, * and pcmk_rc_unpack_error on invalid configuration) */ static int unpack_order_tags(xmlNode *xml_obj, xmlNode **expanded_xml, const pe_working_set_t *data_set) { const char *id_first = NULL; const char *id_then = NULL; const char *action_first = NULL; const char *action_then = NULL; pe_resource_t *rsc_first = NULL; pe_resource_t *rsc_then = NULL; pe_tag_t *tag_first = NULL; pe_tag_t *tag_then = NULL; xmlNode *rsc_set_first = NULL; xmlNode *rsc_set_then = NULL; bool any_sets = false; // Check whether there are any resource sets with template or tag references *expanded_xml = pcmk__expand_tags_in_sets(xml_obj, data_set); if (*expanded_xml != NULL) { crm_log_xml_trace(*expanded_xml, "Expanded rsc_order"); return pcmk_rc_ok; } id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST); id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN); if ((id_first == NULL) || (id_then == NULL)) { return pcmk_rc_ok; } if (!pcmk__valid_resource_or_tag(data_set, id_first, &rsc_first, &tag_first)) { pcmk__config_err("Ignoring constraint '%s' because '%s' is not a " "valid resource or tag", ID(xml_obj), id_first); return pcmk_rc_unpack_error; } if (!pcmk__valid_resource_or_tag(data_set, id_then, &rsc_then, &tag_then)) { pcmk__config_err("Ignoring constraint '%s' because '%s' is not a " "valid resource or tag", ID(xml_obj), id_then); return pcmk_rc_unpack_error; } if ((rsc_first != NULL) && (rsc_then != NULL)) { // Neither side references a template or tag return pcmk_rc_ok; } action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION); action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION); *expanded_xml = copy_xml(xml_obj); // Convert template/tag reference in "first" into constraint resource_set if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_first, XML_ORDER_ATTR_FIRST, true, data_set)) { free_xml(*expanded_xml); *expanded_xml = NULL; return pcmk_rc_unpack_error; } if (rsc_set_first != NULL) { if (action_first != NULL) { // Move "first-action" into converted resource_set as "action" crm_xml_add(rsc_set_first, "action", action_first); xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_FIRST_ACTION); } any_sets = true; } // Convert template/tag reference in "then" into constraint resource_set if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_then, XML_ORDER_ATTR_THEN, true, data_set)) { free_xml(*expanded_xml); *expanded_xml = NULL; return pcmk_rc_unpack_error; } if (rsc_set_then != NULL) { if (action_then != NULL) { // Move "then-action" into converted resource_set as "action" crm_xml_add(rsc_set_then, "action", action_then); xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_THEN_ACTION); } any_sets = true; } if (any_sets) { crm_log_xml_trace(*expanded_xml, "Expanded rsc_order"); } else { free_xml(*expanded_xml); *expanded_xml = NULL; } return pcmk_rc_ok; } /*! * \internal * \brief Unpack ordering constraint XML * * \param[in,out] xml_obj Ordering constraint XML to unpack * \param[in,out] data_set Cluster working set */ void pcmk__unpack_ordering(xmlNode *xml_obj, pe_working_set_t *data_set) { xmlNode *set = NULL; xmlNode *last = NULL; xmlNode *orig_xml = NULL; xmlNode *expanded_xml = NULL; const char *id = crm_element_value(xml_obj, XML_ATTR_ID); const char *invert = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL); enum pe_order_kind kind = get_ordering_type(xml_obj); enum ordering_symmetry symmetry = get_ordering_symmetry(xml_obj, kind, NULL); // Expand any resource tags in the constraint XML if (unpack_order_tags(xml_obj, &expanded_xml, data_set) != pcmk_rc_ok) { return; } if (expanded_xml != NULL) { orig_xml = xml_obj; xml_obj = expanded_xml; } // If the constraint has resource sets, unpack them for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET); set != NULL; set = crm_next_same_xml(set)) { set = expand_idref(set, data_set->input); if ((set == NULL) // Configuration error, message already logged || (unpack_order_set(set, kind, invert, data_set) != pcmk_rc_ok)) { if (expanded_xml != NULL) { free_xml(expanded_xml); } return; } if (last != NULL) { if (order_rsc_sets(id, last, set, kind, data_set, symmetry) != pcmk_rc_ok) { if (expanded_xml != NULL) { free_xml(expanded_xml); } return; } if ((symmetry == ordering_symmetric) && (order_rsc_sets(id, set, last, kind, data_set, ordering_symmetric_inverse) != pcmk_rc_ok)) { if (expanded_xml != NULL) { free_xml(expanded_xml); } return; } } last = set; } if (expanded_xml) { free_xml(expanded_xml); xml_obj = orig_xml; } // If the constraint has no resource sets, unpack it as a simple ordering if (last == NULL) { return unpack_simple_rsc_order(xml_obj, data_set); } } static bool ordering_is_invalid(pe_action_t *action, pe_action_wrapper_t *input) { /* Prevent user-defined ordering constraints between resources * running in a guest node and the resource that defines that node. */ - if (!pcmk_is_set(input->type, pe_order_preserve) + if (!pcmk_is_set(input->type, pcmk__ar_guest_allowed) && (input->action->rsc != NULL) && pcmk__rsc_corresponds_to_guest(action->rsc, input->action->node)) { crm_warn("Invalid ordering constraint between %s and %s", input->action->rsc->id, action->rsc->id); return true; } /* If there's an order like * "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1" * * then rscA is being migrated from node1 to node2, while rscB is being * migrated from node2 to node1. If there would be a graph loop, * break the order "load_stopped_node2" -> "rscA_migrate_to node1". */ if (((uint32_t) input->type == pcmk__ar_if_on_same_node_or_target) && (action->rsc != NULL) && pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO, pcmk__str_none) && pcmk__graph_has_loop(action, action, input)) { return true; } return false; } void pcmk__disable_invalid_orderings(pe_working_set_t *data_set) { for (GList *iter = data_set->actions; iter != NULL; iter = iter->next) { pe_action_t *action = (pe_action_t *) iter->data; pe_action_wrapper_t *input = NULL; for (GList *input_iter = action->actions_before; input_iter != NULL; input_iter = input_iter->next) { input = (pe_action_wrapper_t *) input_iter->data; if (ordering_is_invalid(action, input)) { input->type = pcmk__ar_none; } } } } /*! * \internal * \brief Order stops on a node before the node's shutdown * * \param[in,out] node Node being shut down * \param[in] shutdown_op Shutdown action for node */ void pcmk__order_stops_before_shutdown(pe_node_t *node, pe_action_t *shutdown_op) { for (GList *iter = node->details->data_set->actions; iter != NULL; iter = iter->next) { pe_action_t *action = (pe_action_t *) iter->data; // Only stops on the node shutting down are relevant if (!pe__same_node(action->node, node) || !pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) { continue; } // Resources and nodes in maintenance mode won't be touched if (pcmk_is_set(action->rsc->flags, pcmk_rsc_maintenance)) { pe_rsc_trace(action->rsc, "Not ordering %s before shutdown of %s because " "resource in maintenance mode", action->uuid, pe__node_name(node)); continue; } else if (node->details->maintenance) { pe_rsc_trace(action->rsc, "Not ordering %s before shutdown of %s because " "node in maintenance mode", action->uuid, pe__node_name(node)); continue; } /* Don't touch a resource that is unmanaged or blocked, to avoid * blocking the shutdown (though if another action depends on this one, * we may still end up blocking) */ if (!pcmk_any_flags_set(action->rsc->flags, pcmk_rsc_managed|pcmk_rsc_blocked)) { pe_rsc_trace(action->rsc, "Not ordering %s before shutdown of %s because " "resource is unmanaged or blocked", action->uuid, pe__node_name(node)); continue; } pe_rsc_trace(action->rsc, "Ordering %s before shutdown of %s", action->uuid, pe__node_name(node)); pe__clear_action_flags(action, pcmk_action_optional); pcmk__new_ordering(action->rsc, NULL, action, NULL, strdup(PCMK_ACTION_DO_SHUTDOWN), shutdown_op, pcmk__ar_ordered|pcmk__ar_unrunnable_first_blocks, node->details->data_set); } } /*! * \brief Find resource actions matching directly or as child * * \param[in] rsc Resource to check * \param[in] original_key Action key to search for (possibly referencing * parent of \rsc) * * \return Newly allocated list of matching actions * \note It is the caller's responsibility to free the result with g_list_free() */ static GList * find_actions_by_task(const pe_resource_t *rsc, const char *original_key) { // Search under given task key directly GList *list = find_actions(rsc->actions, original_key, NULL); if (list == NULL) { // Search again using this resource's ID char *key = NULL; char *task = NULL; guint interval_ms = 0; if (parse_op_key(original_key, NULL, &task, &interval_ms)) { key = pcmk__op_key(rsc->id, task, interval_ms); list = find_actions(rsc->actions, key, NULL); free(key); free(task); } else { crm_err("Invalid operation key (bug?): %s", original_key); } } return list; } /*! * \internal * \brief Order relevant resource actions after a given action * * \param[in,out] first_action Action to order after (or NULL if none runnable) * \param[in] rsc Resource whose actions should be ordered * \param[in,out] order Ordering constraint being applied */ static void order_resource_actions_after(pe_action_t *first_action, const pe_resource_t *rsc, pe__ordering_t *order) { GList *then_actions = NULL; uint32_t flags = pcmk__ar_none; CRM_CHECK((rsc != NULL) && (order != NULL), return); flags = order->flags; pe_rsc_trace(rsc, "Applying ordering %d for 'then' resource %s", order->id, rsc->id); if (order->rh_action != NULL) { then_actions = g_list_prepend(NULL, order->rh_action); } else { then_actions = find_actions_by_task(rsc, order->rh_action_task); } if (then_actions == NULL) { pe_rsc_trace(rsc, "Ignoring ordering %d: no %s actions found for %s", order->id, order->rh_action_task, rsc->id); return; } if ((first_action != NULL) && (first_action->rsc == rsc) && pcmk_is_set(first_action->flags, pcmk_action_migration_abort)) { pe_rsc_trace(rsc, "Detected dangling migration ordering (%s then %s %s)", first_action->uuid, order->rh_action_task, rsc->id); pe__clear_order_flags(flags, pcmk__ar_first_implies_then); } if ((first_action == NULL) && !pcmk_is_set(flags, pcmk__ar_first_implies_then)) { pe_rsc_debug(rsc, "Ignoring ordering %d for %s: No first action found", order->id, rsc->id); g_list_free(then_actions); return; } for (GList *iter = then_actions; iter != NULL; iter = iter->next) { pe_action_t *then_action_iter = (pe_action_t *) iter->data; if (first_action != NULL) { order_actions(first_action, then_action_iter, flags); } else { pe__clear_action_flags(then_action_iter, pcmk_action_runnable); crm_warn("%s of %s is unrunnable because there is no %s of %s " "to order it after", then_action_iter->task, rsc->id, order->lh_action_task, order->lh_rsc->id); } } g_list_free(then_actions); } static void rsc_order_first(pe_resource_t *first_rsc, pe__ordering_t *order) { GList *first_actions = NULL; pe_action_t *first_action = order->lh_action; pe_resource_t *then_rsc = order->rh_rsc; CRM_ASSERT(first_rsc != NULL); pe_rsc_trace(first_rsc, "Applying ordering constraint %d (first: %s)", order->id, first_rsc->id); if (first_action != NULL) { first_actions = g_list_prepend(NULL, first_action); } else { first_actions = find_actions_by_task(first_rsc, order->lh_action_task); } if ((first_actions == NULL) && (first_rsc == then_rsc)) { pe_rsc_trace(first_rsc, "Ignoring constraint %d: first (%s for %s) not found", order->id, order->lh_action_task, first_rsc->id); } else if (first_actions == NULL) { char *key = NULL; char *op_type = NULL; guint interval_ms = 0; parse_op_key(order->lh_action_task, NULL, &op_type, &interval_ms); key = pcmk__op_key(first_rsc->id, op_type, interval_ms); if ((first_rsc->fns->state(first_rsc, TRUE) == pcmk_role_stopped) && pcmk__str_eq(op_type, PCMK_ACTION_STOP, pcmk__str_none)) { free(key); pe_rsc_trace(first_rsc, "Ignoring constraint %d: first (%s for %s) not found", order->id, order->lh_action_task, first_rsc->id); } else if ((first_rsc->fns->state(first_rsc, TRUE) == pcmk_role_unpromoted) && pcmk__str_eq(op_type, PCMK_ACTION_DEMOTE, pcmk__str_none)) { free(key); pe_rsc_trace(first_rsc, "Ignoring constraint %d: first (%s for %s) not found", order->id, order->lh_action_task, first_rsc->id); } else { pe_rsc_trace(first_rsc, "Creating first (%s for %s) for constraint %d ", order->lh_action_task, first_rsc->id, order->id); first_action = custom_action(first_rsc, key, op_type, NULL, TRUE, TRUE, first_rsc->cluster); first_actions = g_list_prepend(NULL, first_action); } free(op_type); } if (then_rsc == NULL) { if (order->rh_action == NULL) { pe_rsc_trace(first_rsc, "Ignoring constraint %d: then not found", order->id); return; } then_rsc = order->rh_action->rsc; } for (GList *iter = first_actions; iter != NULL; iter = iter->next) { first_action = iter->data; if (then_rsc == NULL) { order_actions(first_action, order->rh_action, order->flags); } else { order_resource_actions_after(first_action, then_rsc, order); } } g_list_free(first_actions); } // GFunc to call pcmk__block_colocation_dependents() static void block_colocation_dependents(gpointer data, gpointer user_data) { pcmk__block_colocation_dependents(data); } // GFunc to call pcmk__update_action_for_orderings() static void update_action_for_orderings(gpointer data, gpointer user_data) { pcmk__update_action_for_orderings((pe_action_t *) data, (pe_working_set_t *) user_data); } /*! * \internal * \brief Apply all ordering constraints * * \param[in,out] sched Cluster working set */ void pcmk__apply_orderings(pe_working_set_t *sched) { crm_trace("Applying ordering constraints"); /* Ordering constraints need to be processed in the order they were created. * rsc_order_first() and order_resource_actions_after() require the relevant * actions to already exist in some cases, but rsc_order_first() will create * the 'first' action in certain cases. Thus calling rsc_order_first() can * change the behavior of later-created orderings. * * Also, g_list_append() should be avoided for performance reasons, so we * prepend orderings when creating them and reverse the list here. * * @TODO This is brittle and should be carefully redesigned so that the * order of creation doesn't matter, and the reverse becomes unneeded. */ sched->ordering_constraints = g_list_reverse(sched->ordering_constraints); for (GList *iter = sched->ordering_constraints; iter != NULL; iter = iter->next) { pe__ordering_t *order = iter->data; pe_resource_t *rsc = order->lh_rsc; if (rsc != NULL) { rsc_order_first(rsc, order); continue; } rsc = order->rh_rsc; if (rsc != NULL) { order_resource_actions_after(order->lh_action, rsc, order); } else { crm_trace("Applying ordering constraint %d (non-resource actions)", order->id); order_actions(order->lh_action, order->rh_action, order->flags); } } g_list_foreach(sched->actions, block_colocation_dependents, NULL); crm_trace("Ordering probes"); pcmk__order_probes(sched); crm_trace("Updating %d actions", g_list_length(sched->actions)); g_list_foreach(sched->actions, update_action_for_orderings, sched); pcmk__disable_invalid_orderings(sched); } /*! * \internal * \brief Order a given action after each action in a given list * * \param[in,out] after "After" action * \param[in,out] list List of "before" actions */ void pcmk__order_after_each(pe_action_t *after, GList *list) { const char *after_desc = (after->task == NULL)? after->uuid : after->task; for (GList *iter = list; iter != NULL; iter = iter->next) { pe_action_t *before = (pe_action_t *) iter->data; const char *before_desc = before->task? before->task : before->uuid; crm_debug("Ordering %s on %s before %s on %s", before_desc, pe__node_name(before->node), after_desc, pe__node_name(after->node)); order_actions(before, after, pcmk__ar_ordered); } } /*! * \internal * \brief Order promotions and demotions for restarts of a clone or bundle * * \param[in,out] rsc Clone or bundle to order */ void pcmk__promotable_restart_ordering(pe_resource_t *rsc) { // Order start and promote after all instances are stopped pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED, rsc, PCMK_ACTION_START, pcmk__ar_ordered); pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED, rsc, PCMK_ACTION_PROMOTE, pcmk__ar_ordered); // Order stop, start, and promote after all instances are demoted pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED, rsc, PCMK_ACTION_STOP, pcmk__ar_ordered); pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED, rsc, PCMK_ACTION_START, pcmk__ar_ordered); pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED, rsc, PCMK_ACTION_PROMOTE, pcmk__ar_ordered); // Order promote after all instances are started pcmk__order_resource_actions(rsc, PCMK_ACTION_RUNNING, rsc, PCMK_ACTION_PROMOTE, pcmk__ar_ordered); // Order demote after all instances are demoted pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE, rsc, PCMK_ACTION_DEMOTED, pcmk__ar_ordered); } diff --git a/lib/pacemaker/pcmk_sched_remote.c b/lib/pacemaker/pcmk_sched_remote.c index 57f8655a6f..c743e906b5 100644 --- a/lib/pacemaker/pcmk_sched_remote.c +++ b/lib/pacemaker/pcmk_sched_remote.c @@ -1,728 +1,728 @@ /* * Copyright 2004-2023 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include "libpacemaker_private.h" enum remote_connection_state { remote_state_unknown = 0, remote_state_alive = 1, remote_state_resting = 2, remote_state_failed = 3, remote_state_stopped = 4 }; static const char * state2text(enum remote_connection_state state) { switch (state) { case remote_state_unknown: return "unknown"; case remote_state_alive: return "alive"; case remote_state_resting: return "resting"; case remote_state_failed: return "failed"; case remote_state_stopped: return "stopped"; } return "impossible"; } -/* We always use pe_order_preserve with these convenience functions to exempt - * internally generated constraints from the prohibition of user constraints - * involving remote connection resources. +/* We always use pcmk__ar_guest_allowed with these convenience functions to + * exempt internally generated constraints from the prohibition of user + * constraints involving remote connection resources. * * The start ordering additionally uses pcmk__ar_unrunnable_first_blocks so that * the specified action is not runnable if the start is not runnable. */ static inline void order_start_then_action(pe_resource_t *first_rsc, pe_action_t *then_action, uint32_t extra) { if ((first_rsc != NULL) && (then_action != NULL)) { pcmk__new_ordering(first_rsc, start_key(first_rsc), NULL, then_action->rsc, NULL, then_action, - pe_order_preserve + pcmk__ar_guest_allowed |pcmk__ar_unrunnable_first_blocks |extra, first_rsc->cluster); } } static inline void order_action_then_stop(pe_action_t *first_action, pe_resource_t *then_rsc, uint32_t extra) { if ((first_action != NULL) && (then_rsc != NULL)) { pcmk__new_ordering(first_action->rsc, NULL, first_action, then_rsc, stop_key(then_rsc), NULL, - pe_order_preserve|extra, then_rsc->cluster); + pcmk__ar_guest_allowed|extra, then_rsc->cluster); } } static enum remote_connection_state get_remote_node_state(const pe_node_t *node) { const pe_resource_t *remote_rsc = NULL; const pe_node_t *cluster_node = NULL; CRM_ASSERT(node != NULL); remote_rsc = node->details->remote_rsc; CRM_ASSERT(remote_rsc != NULL); cluster_node = pe__current_node(remote_rsc); /* If the cluster node the remote connection resource resides on * is unclean or went offline, we can't process any operations * on that remote node until after it starts elsewhere. */ if ((remote_rsc->next_role == pcmk_role_stopped) || (remote_rsc->allocated_to == NULL)) { // The connection resource is not going to run anywhere if ((cluster_node != NULL) && cluster_node->details->unclean) { /* The remote connection is failed because its resource is on a * failed node and can't be recovered elsewhere, so we must fence. */ return remote_state_failed; } if (!pcmk_is_set(remote_rsc->flags, pcmk_rsc_failed)) { /* Connection resource is cleanly stopped */ return remote_state_stopped; } /* Connection resource is failed */ if ((remote_rsc->next_role == pcmk_role_stopped) && remote_rsc->remote_reconnect_ms && node->details->remote_was_fenced && !pe__shutdown_requested(node)) { /* We won't know whether the connection is recoverable until the * reconnect interval expires and we reattempt connection. */ return remote_state_unknown; } /* The remote connection is in a failed state. If there are any * resources known to be active on it (stop) or in an unknown state * (probe), we must assume the worst and fence it. */ return remote_state_failed; } else if (cluster_node == NULL) { /* Connection is recoverable but not currently running anywhere, so see * if we can recover it first */ return remote_state_unknown; } else if (cluster_node->details->unclean || !(cluster_node->details->online)) { // Connection is running on a dead node, see if we can recover it first return remote_state_resting; } else if (pcmk__list_of_multiple(remote_rsc->running_on) && (remote_rsc->partial_migration_source != NULL) && (remote_rsc->partial_migration_target != NULL)) { /* We're in the middle of migrating a connection resource, so wait until * after the migration completes before performing any actions. */ return remote_state_resting; } return remote_state_alive; } /*! * \internal * \brief Order actions on remote node relative to actions for the connection * * \param[in,out] action An action scheduled on a Pacemaker Remote node */ static void apply_remote_ordering(pe_action_t *action) { pe_resource_t *remote_rsc = NULL; enum action_tasks task = text2task(action->task); enum remote_connection_state state = get_remote_node_state(action->node); uint32_t order_opts = pcmk__ar_none; if (action->rsc == NULL) { return; } CRM_ASSERT(pe__is_guest_or_remote_node(action->node)); remote_rsc = action->node->details->remote_rsc; CRM_ASSERT(remote_rsc != NULL); crm_trace("Order %s action %s relative to %s%s (state: %s)", action->task, action->uuid, pcmk_is_set(remote_rsc->flags, pcmk_rsc_failed)? "failed " : "", remote_rsc->id, state2text(state)); if (pcmk__strcase_any_of(action->task, PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM, NULL)) { /* Migration ops map to pcmk_action_unspecified, but we need to apply * the same ordering as for stop or demote (see get_router_node()). */ task = pcmk_action_stop; } switch (task) { case pcmk_action_start: case pcmk_action_promote: order_opts = pcmk__ar_none; if (state == remote_state_failed) { /* Force recovery, by making this action required */ pe__set_order_flags(order_opts, pcmk__ar_first_implies_then); } /* Ensure connection is up before running this action */ order_start_then_action(remote_rsc, action, order_opts); break; case pcmk_action_stop: if (state == remote_state_alive) { order_action_then_stop(action, remote_rsc, pcmk__ar_then_implies_first); } else if (state == remote_state_failed) { /* The resource is active on the node, but since we don't have a * valid connection, the only way to stop the resource is by * fencing the node. There is no need to order the stop relative * to the remote connection, since the stop will become implied * by the fencing. */ pe_fence_node(remote_rsc->cluster, action->node, "resources are active but " "connection is unrecoverable", FALSE); } else if (remote_rsc->next_role == pcmk_role_stopped) { /* State must be remote_state_unknown or remote_state_stopped. * Since the connection is not coming back up in this * transition, stop this resource first. */ order_action_then_stop(action, remote_rsc, pcmk__ar_then_implies_first); } else { /* The connection is going to be started somewhere else, so * stop this resource after that completes. */ order_start_then_action(remote_rsc, action, pcmk__ar_none); } break; case pcmk_action_demote: /* Only order this demote relative to the connection start if the * connection isn't being torn down. Otherwise, the demote would be * blocked because the connection start would not be allowed. */ if ((state == remote_state_resting) || (state == remote_state_unknown)) { order_start_then_action(remote_rsc, action, pcmk__ar_none); } /* Otherwise we can rely on the stop ordering */ break; default: /* Wait for the connection resource to be up */ if (pcmk__action_is_recurring(action)) { /* In case we ever get the recovery logic wrong, force * recurring monitors to be restarted, even if just * the connection was re-established */ order_start_then_action(remote_rsc, action, pcmk__ar_first_implies_then); } else { pe_node_t *cluster_node = pe__current_node(remote_rsc); if ((task == pcmk_action_monitor) && (state == remote_state_failed)) { /* We would only be here if we do not know the state of the * resource on the remote node. Since we have no way to find * out, it is necessary to fence the node. */ pe_fence_node(remote_rsc->cluster, action->node, "resources are in unknown state " "and connection is unrecoverable", FALSE); } if ((cluster_node != NULL) && (state == remote_state_stopped)) { /* The connection is currently up, but is going down * permanently. Make sure we check services are actually * stopped _before_ we let the connection get closed. */ order_action_then_stop(action, remote_rsc, pcmk__ar_unrunnable_first_blocks); } else { order_start_then_action(remote_rsc, action, pcmk__ar_none); } } break; } } static void apply_container_ordering(pe_action_t *action) { /* VMs are also classified as containers for these purposes... in * that they both involve a 'thing' running on a real or remote * cluster node. * * This allows us to be smarter about the type and extent of * recovery actions required in various scenarios */ pe_resource_t *remote_rsc = NULL; pe_resource_t *container = NULL; enum action_tasks task = text2task(action->task); CRM_ASSERT(action->rsc != NULL); CRM_ASSERT(action->node != NULL); CRM_ASSERT(pe__is_guest_or_remote_node(action->node)); remote_rsc = action->node->details->remote_rsc; CRM_ASSERT(remote_rsc != NULL); container = remote_rsc->container; CRM_ASSERT(container != NULL); if (pcmk_is_set(container->flags, pcmk_rsc_failed)) { pe_fence_node(action->rsc->cluster, action->node, "container failed", FALSE); } crm_trace("Order %s action %s relative to %s%s for %s%s", action->task, action->uuid, pcmk_is_set(remote_rsc->flags, pcmk_rsc_failed)? "failed " : "", remote_rsc->id, pcmk_is_set(container->flags, pcmk_rsc_failed)? "failed " : "", container->id); if (pcmk__strcase_any_of(action->task, PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM, NULL)) { /* Migration ops map to pcmk_action_unspecified, but we need to apply * the same ordering as for stop or demote (see get_router_node()). */ task = pcmk_action_stop; } switch (task) { case pcmk_action_start: case pcmk_action_promote: // Force resource recovery if the container is recovered order_start_then_action(container, action, pcmk__ar_first_implies_then); // Wait for the connection resource to be up, too order_start_then_action(remote_rsc, action, pcmk__ar_none); break; case pcmk_action_stop: case pcmk_action_demote: if (pcmk_is_set(container->flags, pcmk_rsc_failed)) { /* When the container representing a guest node fails, any stop * or demote actions for resources running on the guest node * are implied by the container stopping. This is similar to * how fencing operations work for cluster nodes and remote * nodes. */ } else { /* Ensure the operation happens before the connection is brought * down. * * If we really wanted to, we could order these after the * connection start, IFF the container's current role was * stopped (otherwise we re-introduce an ordering loop when the * connection is restarting). */ order_action_then_stop(action, remote_rsc, pcmk__ar_none); } break; default: /* Wait for the connection resource to be up */ if (pcmk__action_is_recurring(action)) { /* In case we ever get the recovery logic wrong, force * recurring monitors to be restarted, even if just * the connection was re-established */ if (task != pcmk_action_unspecified) { order_start_then_action(remote_rsc, action, pcmk__ar_first_implies_then); } } else { order_start_then_action(remote_rsc, action, pcmk__ar_none); } break; } } /*! * \internal * \brief Order all relevant actions relative to remote connection actions * * \param[in,out] data_set Cluster working set */ void pcmk__order_remote_connection_actions(pe_working_set_t *data_set) { if (!pcmk_is_set(data_set->flags, pcmk_sched_have_remote_nodes)) { return; } crm_trace("Creating remote connection orderings"); for (GList *iter = data_set->actions; iter != NULL; iter = iter->next) { pe_action_t *action = iter->data; pe_resource_t *remote = NULL; // We are only interested in resource actions if (action->rsc == NULL) { continue; } /* Special case: If we are clearing the failcount of an actual * remote connection resource, then make sure this happens before * any start of the resource in this transition. */ if (action->rsc->is_remote_node && pcmk__str_eq(action->task, PCMK_ACTION_CLEAR_FAILCOUNT, pcmk__str_none)) { pcmk__new_ordering(action->rsc, NULL, action, action->rsc, pcmk__op_key(action->rsc->id, PCMK_ACTION_START, 0), NULL, pcmk__ar_ordered, data_set); continue; } // We are only interested in actions assigned to a node if (action->node == NULL) { continue; } if (!pe__is_guest_or_remote_node(action->node)) { continue; } /* We are only interested in real actions. * * @TODO This is probably wrong; pseudo-actions might be converted to * real actions and vice versa later in update_actions() at the end of * pcmk__apply_orderings(). */ if (pcmk_is_set(action->flags, pcmk_action_pseudo)) { continue; } remote = action->node->details->remote_rsc; if (remote == NULL) { // Orphaned continue; } /* Another special case: if a resource is moving to a Pacemaker Remote * node, order the stop on the original node after any start of the * remote connection. This ensures that if the connection fails to * start, we leave the resource running on the original node. */ if (pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_none)) { for (GList *item = action->rsc->actions; item != NULL; item = item->next) { pe_action_t *rsc_action = item->data; if (!pe__same_node(rsc_action->node, action->node) && pcmk__str_eq(rsc_action->task, PCMK_ACTION_STOP, pcmk__str_none)) { pcmk__new_ordering(remote, start_key(remote), NULL, action->rsc, NULL, rsc_action, pcmk__ar_ordered, data_set); } } } /* The action occurs across a remote connection, so create * ordering constraints that guarantee the action occurs while the node * is active (after start, before stop ... things like that). * * This is somewhat brittle in that we need to make sure the results of * this ordering are compatible with the result of get_router_node(). * It would probably be better to add XML_LRM_ATTR_ROUTER_NODE as part * of this logic rather than create_graph_action(). */ if (remote->container) { crm_trace("Container ordering for %s", action->uuid); apply_container_ordering(action); } else { crm_trace("Remote ordering for %s", action->uuid); apply_remote_ordering(action); } } } /*! * \internal * \brief Check whether a node is a failed remote node * * \param[in] node Node to check * * \return true if \p node is a failed remote node, false otherwise */ bool pcmk__is_failed_remote_node(const pe_node_t *node) { return pe__is_remote_node(node) && (node->details->remote_rsc != NULL) && (get_remote_node_state(node) == remote_state_failed); } /*! * \internal * \brief Check whether a given resource corresponds to a given node as guest * * \param[in] rsc Resource to check * \param[in] node Node to check * * \return true if \p node is a guest node and \p rsc is its containing * resource, otherwise false */ bool pcmk__rsc_corresponds_to_guest(const pe_resource_t *rsc, const pe_node_t *node) { return (rsc != NULL) && (rsc->fillers != NULL) && (node != NULL) && (node->details->remote_rsc != NULL) && (node->details->remote_rsc->container == rsc); } /*! * \internal * \brief Get proper connection host that a remote action must be routed through * * A remote connection resource might be starting, stopping, or migrating in the * same transition that an action needs to be executed on its Pacemaker Remote * node. Determine the proper node that the remote action should be routed * through. * * \param[in] action (Potentially remote) action to route * * \return Connection host that action should be routed through if remote, * otherwise NULL */ pe_node_t * pcmk__connection_host_for_action(const pe_action_t *action) { pe_node_t *began_on = NULL; pe_node_t *ended_on = NULL; bool partial_migration = false; const char *task = action->task; if (pcmk__str_eq(task, PCMK_ACTION_STONITH, pcmk__str_none) || !pe__is_guest_or_remote_node(action->node)) { return NULL; } CRM_ASSERT(action->node->details->remote_rsc != NULL); began_on = pe__current_node(action->node->details->remote_rsc); ended_on = action->node->details->remote_rsc->allocated_to; if (action->node->details->remote_rsc && (action->node->details->remote_rsc->container == NULL) && action->node->details->remote_rsc->partial_migration_target) { partial_migration = true; } if (began_on == NULL) { crm_trace("Routing %s for %s through remote connection's " "next node %s (starting)%s", action->task, (action->rsc? action->rsc->id : "no resource"), (ended_on? ended_on->details->uname : "none"), partial_migration? " (partial migration)" : ""); return ended_on; } if (ended_on == NULL) { crm_trace("Routing %s for %s through remote connection's " "current node %s (stopping)%s", action->task, (action->rsc? action->rsc->id : "no resource"), (began_on? began_on->details->uname : "none"), partial_migration? " (partial migration)" : ""); return began_on; } if (pe__same_node(began_on, ended_on)) { crm_trace("Routing %s for %s through remote connection's " "current node %s (not moving)%s", action->task, (action->rsc? action->rsc->id : "no resource"), (began_on? began_on->details->uname : "none"), partial_migration? " (partial migration)" : ""); return began_on; } /* If we get here, the remote connection is moving during this transition. * This means some actions for resources behind the connection will get * routed through the cluster node the connection resource is currently on, * and others are routed through the cluster node the connection will end up * on. */ if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)) { task = g_hash_table_lookup(action->meta, "notify_operation"); } /* * Stop, demote, and migration actions must occur before the connection can * move (these actions are required before the remote resource can stop). In * this case, we know these actions have to be routed through the initial * cluster node the connection resource lived on before the move takes * place. * * The exception is a partial migration of a (non-guest) remote connection * resource; in that case, all actions (even these) will be ordered after * the connection's pseudo-start on the migration target, so the target is * the router node. */ if (pcmk__strcase_any_of(task, PCMK_ACTION_CANCEL, PCMK_ACTION_STOP, PCMK_ACTION_DEMOTE, PCMK_ACTION_MIGRATE_FROM, PCMK_ACTION_MIGRATE_TO, NULL) && !partial_migration) { crm_trace("Routing %s for %s through remote connection's " "current node %s (moving)%s", action->task, (action->rsc? action->rsc->id : "no resource"), (began_on? began_on->details->uname : "none"), partial_migration? " (partial migration)" : ""); return began_on; } /* Everything else (start, promote, monitor, probe, refresh, * clear failcount, delete, ...) must occur after the connection starts on * the node it is moving to. */ crm_trace("Routing %s for %s through remote connection's " "next node %s (moving)%s", action->task, (action->rsc? action->rsc->id : "no resource"), (ended_on? ended_on->details->uname : "none"), partial_migration? " (partial migration)" : ""); return ended_on; } /*! * \internal * \brief Replace remote connection's addr="#uname" with actual address * * REMOTE_CONTAINER_HACK: If a given resource is a remote connection resource * with its "addr" parameter set to "#uname", pull the actual value from the * parameters evaluated without a node (which was put there earlier in * pcmk__create_graph() when the bundle's expand() method was called). * * \param[in,out] rsc Resource to check * \param[in,out] params Resource parameters evaluated per node */ void pcmk__substitute_remote_addr(pe_resource_t *rsc, GHashTable *params) { const char *remote_addr = g_hash_table_lookup(params, XML_RSC_ATTR_REMOTE_RA_ADDR); if (pcmk__str_eq(remote_addr, "#uname", pcmk__str_none)) { GHashTable *base = pe_rsc_params(rsc, NULL, rsc->cluster); remote_addr = g_hash_table_lookup(base, XML_RSC_ATTR_REMOTE_RA_ADDR); if (remote_addr != NULL) { g_hash_table_insert(params, strdup(XML_RSC_ATTR_REMOTE_RA_ADDR), strdup(remote_addr)); } } } /*! * \brief Add special bundle meta-attributes to XML * * If a given action will be executed on a guest node (including a bundle), * add the special bundle meta-attribute "container-attribute-target" and * environment variable "physical_host" as XML attributes (using meta-attribute * naming). * * \param[in,out] args_xml XML to add attributes to * \param[in] action Action to check */ void pcmk__add_bundle_meta_to_xml(xmlNode *args_xml, const pe_action_t *action) { const pe_node_t *guest = action->node; const pe_node_t *host = NULL; enum action_tasks task; if (!pe__is_guest_node(guest)) { return; } task = text2task(action->task); if ((task == pcmk_action_notify) || (task == pcmk_action_notified)) { task = text2task(g_hash_table_lookup(action->meta, "notify_operation")); } switch (task) { case pcmk_action_stop: case pcmk_action_stopped: case pcmk_action_demote: case pcmk_action_demoted: // "Down" actions take place on guest's current host host = pe__current_node(guest->details->remote_rsc->container); break; case pcmk_action_start: case pcmk_action_started: case pcmk_action_monitor: case pcmk_action_promote: case pcmk_action_promoted: // "Up" actions take place on guest's next host host = guest->details->remote_rsc->container->allocated_to; break; default: break; } if (host != NULL) { hash2metafield((gpointer) XML_RSC_ATTR_TARGET, (gpointer) g_hash_table_lookup(action->rsc->meta, XML_RSC_ATTR_TARGET), (gpointer) args_xml); hash2metafield((gpointer) PCMK__ENV_PHYSICAL_HOST, (gpointer) host->details->uname, (gpointer) args_xml); } } diff --git a/lib/pacemaker/pcmk_scheduler.c b/lib/pacemaker/pcmk_scheduler.c index e1bf952256..db0e056b99 100644 --- a/lib/pacemaker/pcmk_scheduler.c +++ b/lib/pacemaker/pcmk_scheduler.c @@ -1,816 +1,816 @@ /* * Copyright 2004-2023 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include "libpacemaker_private.h" CRM_TRACE_INIT_DATA(pacemaker); /*! * \internal * \brief Do deferred action checks after assignment * * When unpacking the resource history, the scheduler checks for resource * configurations that have changed since an action was run. However, at that * time, bundles using the REMOTE_CONTAINER_HACK don't have their final * parameter information, so instead they add a deferred check to a list. This * function processes one entry in that list. * * \param[in,out] rsc Resource that action history is for * \param[in,out] node Node that action history is for * \param[in] rsc_op Action history entry * \param[in] check Type of deferred check to do */ static void check_params(pe_resource_t *rsc, pe_node_t *node, const xmlNode *rsc_op, enum pcmk__check_parameters check) { const char *reason = NULL; op_digest_cache_t *digest_data = NULL; switch (check) { case pcmk__check_active: if (pcmk__check_action_config(rsc, node, rsc_op) && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL)) { reason = "action definition changed"; } break; case pcmk__check_last_failure: digest_data = rsc_action_digest_cmp(rsc, rsc_op, node, rsc->cluster); switch (digest_data->rc) { case RSC_DIGEST_UNKNOWN: crm_trace("Resource %s history entry %s on %s has " "no digest to compare", rsc->id, ID(rsc_op), node->details->id); break; case RSC_DIGEST_MATCH: break; default: reason = "resource parameters have changed"; break; } break; } if (reason != NULL) { pe__clear_failcount(rsc, node, reason, rsc->cluster); } } /*! * \internal * \brief Check whether a resource has failcount clearing scheduled on a node * * \param[in] node Node to check * \param[in] rsc Resource to check * * \return true if \p rsc has failcount clearing scheduled on \p node, * otherwise false */ static bool failcount_clear_action_exists(const pe_node_t *node, const pe_resource_t *rsc) { GList *list = pe__resource_actions(rsc, node, PCMK_ACTION_CLEAR_FAILCOUNT, TRUE); if (list != NULL) { g_list_free(list); return true; } return false; } /*! * \internal * \brief Ban a resource from a node if it reached its failure threshold there * * \param[in,out] data Resource to check failure threshold for * \param[in] user_data Node to check resource on */ static void check_failure_threshold(gpointer data, gpointer user_data) { pe_resource_t *rsc = data; const pe_node_t *node = user_data; // If this is a collective resource, apply recursively to children instead if (rsc->children != NULL) { g_list_foreach(rsc->children, check_failure_threshold, user_data); return; } if (!failcount_clear_action_exists(node, rsc)) { /* Don't force the resource away from this node due to a failcount * that's going to be cleared. * * @TODO Failcount clearing can be scheduled in * pcmk__handle_rsc_config_changes() via process_rsc_history(), or in * schedule_resource_actions() via check_params(). This runs well before * then, so it cannot detect those, meaning we might check the migration * threshold when we shouldn't. Worst case, we stop or move the * resource, then move it back in the next transition. */ pe_resource_t *failed = NULL; if (pcmk__threshold_reached(rsc, node, &failed)) { resource_location(failed, node, -INFINITY, "__fail_limit__", rsc->cluster); } } } /*! * \internal * \brief If resource has exclusive discovery, ban node if not allowed * * Location constraints have a resource-discovery option that allows users to * specify where probes are done for the affected resource. If this is set to * exclusive, probes will only be done on nodes listed in exclusive constraints. * This function bans the resource from the node if the node is not listed. * * \param[in,out] data Resource to check * \param[in] user_data Node to check resource on */ static void apply_exclusive_discovery(gpointer data, gpointer user_data) { pe_resource_t *rsc = data; const pe_node_t *node = user_data; if (rsc->exclusive_discover || pe__const_top_resource(rsc, false)->exclusive_discover) { pe_node_t *match = NULL; // If this is a collective resource, apply recursively to children g_list_foreach(rsc->children, apply_exclusive_discovery, user_data); match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id); if ((match != NULL) && (match->rsc_discover_mode != pcmk_probe_exclusive)) { match->weight = -INFINITY; } } } /*! * \internal * \brief Apply stickiness to a resource if appropriate * * \param[in,out] data Resource to check for stickiness * \param[in] user_data Ignored */ static void apply_stickiness(gpointer data, gpointer user_data) { pe_resource_t *rsc = data; pe_node_t *node = NULL; // If this is a collective resource, apply recursively to children instead if (rsc->children != NULL) { g_list_foreach(rsc->children, apply_stickiness, NULL); return; } /* A resource is sticky if it is managed, has stickiness configured, and is * active on a single node. */ if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed) || (rsc->stickiness < 1) || !pcmk__list_of_1(rsc->running_on)) { return; } node = rsc->running_on->data; /* In a symmetric cluster, stickiness can always be used. In an * asymmetric cluster, we have to check whether the resource is still * allowed on the node, so we don't keep the resource somewhere it is no * longer explicitly enabled. */ if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_symmetric_cluster) && (g_hash_table_lookup(rsc->allowed_nodes, node->details->id) == NULL)) { pe_rsc_debug(rsc, "Ignoring %s stickiness because the cluster is " "asymmetric and %s is not explicitly allowed", rsc->id, pe__node_name(node)); return; } pe_rsc_debug(rsc, "Resource %s has %d stickiness on %s", rsc->id, rsc->stickiness, pe__node_name(node)); resource_location(rsc, node, rsc->stickiness, "stickiness", rsc->cluster); } /*! * \internal * \brief Apply shutdown locks for all resources as appropriate * * \param[in,out] data_set Cluster working set */ static void apply_shutdown_locks(pe_working_set_t *data_set) { if (!pcmk_is_set(data_set->flags, pcmk_sched_shutdown_lock)) { return; } for (GList *iter = data_set->resources; iter != NULL; iter = iter->next) { pe_resource_t *rsc = (pe_resource_t *) iter->data; rsc->cmds->shutdown_lock(rsc); } } /*! * \internal * \brief Calculate the number of available nodes in the cluster * * \param[in,out] data_set Cluster working set */ static void count_available_nodes(pe_working_set_t *data_set) { if (pcmk_is_set(data_set->flags, pcmk_sched_no_compat)) { return; } // @COMPAT for API backward compatibility only (cluster does not use value) for (GList *iter = data_set->nodes; iter != NULL; iter = iter->next) { pe_node_t *node = (pe_node_t *) iter->data; if ((node != NULL) && (node->weight >= 0) && node->details->online && (node->details->type != node_ping)) { data_set->max_valid_nodes++; } } crm_trace("Online node count: %d", data_set->max_valid_nodes); } /* * \internal * \brief Apply node-specific scheduling criteria * * After the CIB has been unpacked, process node-specific scheduling criteria * including shutdown locks, location constraints, resource stickiness, * migration thresholds, and exclusive resource discovery. */ static void apply_node_criteria(pe_working_set_t *data_set) { crm_trace("Applying node-specific scheduling criteria"); apply_shutdown_locks(data_set); count_available_nodes(data_set); pcmk__apply_locations(data_set); g_list_foreach(data_set->resources, apply_stickiness, NULL); for (GList *node_iter = data_set->nodes; node_iter != NULL; node_iter = node_iter->next) { for (GList *rsc_iter = data_set->resources; rsc_iter != NULL; rsc_iter = rsc_iter->next) { check_failure_threshold(rsc_iter->data, node_iter->data); apply_exclusive_discovery(rsc_iter->data, node_iter->data); } } } /*! * \internal * \brief Assign resources to nodes * * \param[in,out] data_set Cluster working set */ static void assign_resources(pe_working_set_t *data_set) { GList *iter = NULL; crm_trace("Assigning resources to nodes"); if (!pcmk__str_eq(data_set->placement_strategy, "default", pcmk__str_casei)) { pcmk__sort_resources(data_set); } pcmk__show_node_capacities("Original", data_set); if (pcmk_is_set(data_set->flags, pcmk_sched_have_remote_nodes)) { /* Assign remote connection resources first (which will also assign any * colocation dependencies). If the connection is migrating, always * prefer the partial migration target. */ for (iter = data_set->resources; iter != NULL; iter = iter->next) { pe_resource_t *rsc = (pe_resource_t *) iter->data; if (rsc->is_remote_node) { pe_rsc_trace(rsc, "Assigning remote connection resource '%s'", rsc->id); rsc->cmds->assign(rsc, rsc->partial_migration_target, true); } } } /* now do the rest of the resources */ for (iter = data_set->resources; iter != NULL; iter = iter->next) { pe_resource_t *rsc = (pe_resource_t *) iter->data; if (!rsc->is_remote_node) { pe_rsc_trace(rsc, "Assigning %s resource '%s'", rsc->xml->name, rsc->id); rsc->cmds->assign(rsc, NULL, true); } } pcmk__show_node_capacities("Remaining", data_set); } /*! * \internal * \brief Schedule fail count clearing on online nodes if resource is orphaned * * \param[in,out] data Resource to check * \param[in] user_data Ignored */ static void clear_failcounts_if_orphaned(gpointer data, gpointer user_data) { pe_resource_t *rsc = data; if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)) { return; } crm_trace("Clear fail counts for orphaned resource %s", rsc->id); /* There's no need to recurse into rsc->children because those * should just be unassigned clone instances. */ for (GList *iter = rsc->cluster->nodes; iter != NULL; iter = iter->next) { pe_node_t *node = (pe_node_t *) iter->data; pe_action_t *clear_op = NULL; if (!node->details->online) { continue; } if (pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL) == 0) { continue; } clear_op = pe__clear_failcount(rsc, node, "it is orphaned", rsc->cluster); /* We can't use order_action_then_stop() here because its - * pe_order_preserve breaks things + * pcmk__ar_guest_allowed breaks things */ pcmk__new_ordering(clear_op->rsc, NULL, clear_op, rsc, stop_key(rsc), NULL, pcmk__ar_ordered, rsc->cluster); } } /*! * \internal * \brief Schedule any resource actions needed * * \param[in,out] data_set Cluster working set */ static void schedule_resource_actions(pe_working_set_t *data_set) { // Process deferred action checks pe__foreach_param_check(data_set, check_params); pe__free_param_checks(data_set); if (pcmk_is_set(data_set->flags, pcmk_sched_probe_resources)) { crm_trace("Scheduling probes"); pcmk__schedule_probes(data_set); } if (pcmk_is_set(data_set->flags, pcmk_sched_stop_removed_resources)) { g_list_foreach(data_set->resources, clear_failcounts_if_orphaned, NULL); } crm_trace("Scheduling resource actions"); for (GList *iter = data_set->resources; iter != NULL; iter = iter->next) { pe_resource_t *rsc = (pe_resource_t *) iter->data; rsc->cmds->create_actions(rsc); } } /*! * \internal * \brief Check whether a resource or any of its descendants are managed * * \param[in] rsc Resource to check * * \return true if resource or any descendant is managed, otherwise false */ static bool is_managed(const pe_resource_t *rsc) { if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) { return true; } for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { if (is_managed((pe_resource_t *) iter->data)) { return true; } } return false; } /*! * \internal * \brief Check whether any resources in the cluster are managed * * \param[in] data_set Cluster working set * * \return true if any resource is managed, otherwise false */ static bool any_managed_resources(const pe_working_set_t *data_set) { for (const GList *iter = data_set->resources; iter != NULL; iter = iter->next) { if (is_managed((const pe_resource_t *) iter->data)) { return true; } } return false; } /*! * \internal * \brief Check whether a node requires fencing * * \param[in] node Node to check * \param[in] have_managed Whether any resource in cluster is managed * * \return true if \p node should be fenced, otherwise false */ static bool needs_fencing(const pe_node_t *node, bool have_managed) { return have_managed && node->details->unclean && pe_can_fence(node->details->data_set, node); } /*! * \internal * \brief Check whether a node requires shutdown * * \param[in] node Node to check * * \return true if \p node should be shut down, otherwise false */ static bool needs_shutdown(const pe_node_t *node) { if (pe__is_guest_or_remote_node(node)) { /* Do not send shutdown actions for Pacemaker Remote nodes. * @TODO We might come up with a good use for this in the future. */ return false; } return node->details->online && node->details->shutdown; } /*! * \internal * \brief Track and order non-DC fencing * * \param[in,out] list List of existing non-DC fencing actions * \param[in,out] action Fencing action to prepend to \p list * \param[in] data_set Cluster working set * * \return (Possibly new) head of \p list */ static GList * add_nondc_fencing(GList *list, pe_action_t *action, const pe_working_set_t *data_set) { if (!pcmk_is_set(data_set->flags, pcmk_sched_concurrent_fencing) && (list != NULL)) { /* Concurrent fencing is disabled, so order each non-DC * fencing in a chain. If there is any DC fencing or * shutdown, it will be ordered after the last action in the * chain later. */ order_actions((pe_action_t *) list->data, action, pcmk__ar_ordered); } return g_list_prepend(list, action); } /*! * \internal * \brief Schedule a node for fencing * * \param[in,out] node Node that requires fencing */ static pe_action_t * schedule_fencing(pe_node_t *node) { pe_action_t *fencing = pe_fence_op(node, NULL, FALSE, "node is unclean", FALSE, node->details->data_set); pe_warn("Scheduling node %s for fencing", pe__node_name(node)); pcmk__order_vs_fence(fencing, node->details->data_set); return fencing; } /*! * \internal * \brief Create and order node fencing and shutdown actions * * \param[in,out] data_set Cluster working set */ static void schedule_fencing_and_shutdowns(pe_working_set_t *data_set) { pe_action_t *dc_down = NULL; bool integrity_lost = false; bool have_managed = any_managed_resources(data_set); GList *fencing_ops = NULL; GList *shutdown_ops = NULL; crm_trace("Scheduling fencing and shutdowns as needed"); if (!have_managed) { crm_notice("No fencing will be done until there are resources " "to manage"); } // Check each node for whether it needs fencing or shutdown for (GList *iter = data_set->nodes; iter != NULL; iter = iter->next) { pe_node_t *node = (pe_node_t *) iter->data; pe_action_t *fencing = NULL; /* Guest nodes are "fenced" by recovering their container resource, * so handle them separately. */ if (pe__is_guest_node(node)) { if (node->details->remote_requires_reset && have_managed && pe_can_fence(data_set, node)) { pcmk__fence_guest(node); } continue; } if (needs_fencing(node, have_managed)) { fencing = schedule_fencing(node); // Track DC and non-DC fence actions separately if (node->details->is_dc) { dc_down = fencing; } else { fencing_ops = add_nondc_fencing(fencing_ops, fencing, data_set); } } else if (needs_shutdown(node)) { pe_action_t *down_op = pcmk__new_shutdown_action(node); // Track DC and non-DC shutdown actions separately if (node->details->is_dc) { dc_down = down_op; } else { shutdown_ops = g_list_prepend(shutdown_ops, down_op); } } if ((fencing == NULL) && node->details->unclean) { integrity_lost = true; pe_warn("Node %s is unclean but cannot be fenced", pe__node_name(node)); } } if (integrity_lost) { if (!pcmk_is_set(data_set->flags, pcmk_sched_fencing_enabled)) { pe_warn("Resource functionality and data integrity cannot be " "guaranteed (configure, enable, and test fencing to " "correct this)"); } else if (!pcmk_is_set(data_set->flags, pcmk_sched_quorate)) { crm_notice("Unclean nodes will not be fenced until quorum is " "attained or no-quorum-policy is set to ignore"); } } if (dc_down != NULL) { /* Order any non-DC shutdowns before any DC shutdown, to avoid repeated * DC elections. However, we don't want to order non-DC shutdowns before * a DC *fencing*, because even though we don't want a node that's * shutting down to become DC, the DC fencing could be ordered before a * clone stop that's also ordered before the shutdowns, thus leading to * a graph loop. */ if (pcmk__str_eq(dc_down->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) { pcmk__order_after_each(dc_down, shutdown_ops); } // Order any non-DC fencing before any DC fencing or shutdown if (pcmk_is_set(data_set->flags, pcmk_sched_concurrent_fencing)) { /* With concurrent fencing, order each non-DC fencing action * separately before any DC fencing or shutdown. */ pcmk__order_after_each(dc_down, fencing_ops); } else if (fencing_ops != NULL) { /* Without concurrent fencing, the non-DC fencing actions are * already ordered relative to each other, so we just need to order * the DC fencing after the last action in the chain (which is the * first item in the list). */ order_actions((pe_action_t *) fencing_ops->data, dc_down, pcmk__ar_ordered); } } g_list_free(fencing_ops); g_list_free(shutdown_ops); } static void log_resource_details(pe_working_set_t *data_set) { pcmk__output_t *out = data_set->priv; GList *all = NULL; /* Due to the `crm_mon --node=` feature, out->message() for all the * resource-related messages expects a list of nodes that we are allowed to * output information for. Here, we create a wildcard to match all nodes. */ all = g_list_prepend(all, (gpointer) "*"); for (GList *item = data_set->resources; item != NULL; item = item->next) { pe_resource_t *rsc = (pe_resource_t *) item->data; // Log all resources except inactive orphans if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed) || (rsc->role != pcmk_role_stopped)) { out->message(out, crm_map_element_name(rsc->xml), 0, rsc, all, all); } } g_list_free(all); } static void log_all_actions(pe_working_set_t *data_set) { /* This only ever outputs to the log, so ignore whatever output object was * previously set and just log instead. */ pcmk__output_t *prev_out = data_set->priv; pcmk__output_t *out = NULL; if (pcmk__log_output_new(&out) != pcmk_rc_ok) { return; } pe__register_messages(out); pcmk__register_lib_messages(out); pcmk__output_set_log_level(out, LOG_NOTICE); data_set->priv = out; out->begin_list(out, NULL, NULL, "Actions"); pcmk__output_actions(data_set); out->end_list(out); out->finish(out, CRM_EX_OK, true, NULL); pcmk__output_free(out); data_set->priv = prev_out; } /*! * \internal * \brief Log all required but unrunnable actions at trace level * * \param[in] data_set Cluster working set */ static void log_unrunnable_actions(const pe_working_set_t *data_set) { const uint64_t flags = pcmk_action_optional |pcmk_action_runnable |pcmk_action_pseudo; crm_trace("Required but unrunnable actions:"); for (const GList *iter = data_set->actions; iter != NULL; iter = iter->next) { const pe_action_t *action = (const pe_action_t *) iter->data; if (!pcmk_any_flags_set(action->flags, flags)) { pcmk__log_action("\t", action, true); } } } /*! * \internal * \brief Unpack the CIB for scheduling * * \param[in,out] cib CIB XML to unpack (may be NULL if already unpacked) * \param[in] flags Working set flags to set in addition to defaults * \param[in,out] data_set Cluster working set */ static void unpack_cib(xmlNode *cib, unsigned long long flags, pe_working_set_t *data_set) { const char* localhost_save = NULL; if (pcmk_is_set(data_set->flags, pcmk_sched_have_status)) { crm_trace("Reusing previously calculated cluster status"); pe__set_working_set_flags(data_set, flags); return; } if (data_set->localhost) { localhost_save = data_set->localhost; } CRM_ASSERT(cib != NULL); crm_trace("Calculating cluster status"); /* This will zero the entire struct without freeing anything first, so * callers should never call pcmk__schedule_actions() with a populated data * set unless pcmk_sched_have_status is set (i.e. cluster_status() was * previously called, whether directly or via pcmk__schedule_actions()). */ set_working_set_defaults(data_set); if (localhost_save) { data_set->localhost = localhost_save; } pe__set_working_set_flags(data_set, flags); data_set->input = cib; cluster_status(data_set); // Sets pcmk_sched_have_status } /*! * \internal * \brief Run the scheduler for a given CIB * * \param[in,out] cib CIB XML to use as scheduler input * \param[in] flags Working set flags to set in addition to defaults * \param[in,out] data_set Cluster working set */ void pcmk__schedule_actions(xmlNode *cib, unsigned long long flags, pe_working_set_t *data_set) { unpack_cib(cib, flags, data_set); pcmk__set_assignment_methods(data_set); pcmk__apply_node_health(data_set); pcmk__unpack_constraints(data_set); if (pcmk_is_set(data_set->flags, pcmk_sched_validate_only)) { return; } if (!pcmk_is_set(data_set->flags, pcmk_sched_location_only) && pcmk__is_daemon) { log_resource_details(data_set); } apply_node_criteria(data_set); if (pcmk_is_set(data_set->flags, pcmk_sched_location_only)) { return; } pcmk__create_internal_constraints(data_set); pcmk__handle_rsc_config_changes(data_set); assign_resources(data_set); schedule_resource_actions(data_set); /* Remote ordering constraints need to happen prior to calculating fencing * because it is one more place we can mark nodes as needing fencing. */ pcmk__order_remote_connection_actions(data_set); schedule_fencing_and_shutdowns(data_set); pcmk__apply_orderings(data_set); log_all_actions(data_set); pcmk__create_graph(data_set); if (get_crm_log_level() == LOG_TRACE) { log_unrunnable_actions(data_set); } }