diff --git a/daemons/controld/controld_based.c b/daemons/controld/controld_based.c index cb79eefdc6..23f0baa0e9 100644 --- a/daemons/controld/controld_based.c +++ b/daemons/controld/controld_based.c @@ -1,353 +1,353 @@ /* * Copyright 2004-2022 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include /* sleep */ #include #include #include #include #include int cib_retries = 0; void do_cib_updated(const char *event, xmlNode * msg) { if (pcmk__alert_in_patchset(msg, TRUE)) { mainloop_set_trigger(config_read); } } void do_cib_replaced(const char *event, xmlNode * msg) { int change_section = cib_change_section_nodes | cib_change_section_status; crm_debug("Updating the CIB after a replace: DC=%s", pcmk__btoa(AM_I_DC)); if (AM_I_DC == FALSE) { return; } else if ((fsa_state == S_FINALIZE_JOIN) && pcmk_is_set(fsa_input_register, R_CIB_ASKED)) { /* no need to restart the join - we asked for this replace op */ return; } crm_element_value_int(msg, F_CIB_CHANGE_SECTION, &change_section); if (change_section & (cib_change_section_nodes | cib_change_section_status)) { /* start the join process again so we get everyone's LRM status */ populate_cib_nodes(node_update_quick|node_update_all, __func__); register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); } } void controld_disconnect_cib_manager(void) { CRM_ASSERT(fsa_cib_conn != NULL); crm_info("Disconnecting from the CIB manager"); controld_clear_fsa_input_flags(R_CIB_CONNECTED); fsa_cib_conn->cmds->del_notify_callback(fsa_cib_conn, T_CIB_REPLACE_NOTIFY, do_cib_replaced); fsa_cib_conn->cmds->del_notify_callback(fsa_cib_conn, T_CIB_DIFF_NOTIFY, do_cib_updated); cib_free_callbacks(fsa_cib_conn); if (fsa_cib_conn->state != cib_disconnected) { - /* Does not require a set_slave() reply to sign out from based. */ - fsa_cib_conn->cmds->set_slave(fsa_cib_conn, cib_scope_local | cib_discard_reply); + fsa_cib_conn->cmds->set_secondary(fsa_cib_conn, + cib_scope_local|cib_discard_reply); fsa_cib_conn->cmds->signoff(fsa_cib_conn); } crm_notice("Disconnected from the CIB manager"); } /* A_CIB_STOP, A_CIB_START, O_CIB_RESTART */ void do_cib_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { CRM_ASSERT(fsa_cib_conn != NULL); if (action & A_CIB_STOP) { if (fsa_cib_conn->state != cib_disconnected && last_resource_update != 0) { crm_info("Waiting for resource update %d to complete", last_resource_update); crmd_fsa_stall(FALSE); return; } controld_disconnect_cib_manager(); } if (action & A_CIB_START) { int rc = pcmk_ok; if (cur_state == S_STOPPING) { crm_err("Ignoring request to connect to the CIB manager after shutdown"); return; } rc = fsa_cib_conn->cmds->signon(fsa_cib_conn, CRM_SYSTEM_CRMD, cib_command_nonblocking); if (rc != pcmk_ok) { /* a short wait that usually avoids stalling the FSA */ sleep(1); rc = fsa_cib_conn->cmds->signon(fsa_cib_conn, CRM_SYSTEM_CRMD, cib_command_nonblocking); } if (rc != pcmk_ok) { crm_info("Could not connect to the CIB manager: %s", pcmk_strerror(rc)); } else if (pcmk_ok != fsa_cib_conn->cmds->set_connection_dnotify(fsa_cib_conn, crmd_cib_connection_destroy)) { crm_err("Could not set dnotify callback"); } else if (pcmk_ok != fsa_cib_conn->cmds->add_notify_callback(fsa_cib_conn, T_CIB_REPLACE_NOTIFY, do_cib_replaced)) { crm_err("Could not set CIB notification callback (replace)"); } else if (pcmk_ok != fsa_cib_conn->cmds->add_notify_callback(fsa_cib_conn, T_CIB_DIFF_NOTIFY, do_cib_updated)) { crm_err("Could not set CIB notification callback (update)"); } else { controld_set_fsa_input_flags(R_CIB_CONNECTED); cib_retries = 0; } if (!pcmk_is_set(fsa_input_register, R_CIB_CONNECTED)) { cib_retries++; crm_warn("Couldn't complete CIB registration %d" " times... pause and retry", cib_retries); if (cib_retries < 30) { controld_start_timer(wait_timer); crmd_fsa_stall(FALSE); } else { crm_err("Could not complete CIB" " registration %d times..." " hard error", cib_retries); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } } } /*! * \internal * \brief Get CIB call options to use local scope if primary is unavailable * * \return CIB call options */ int crmd_cib_smart_opt(void) { int call_opt = cib_quorum_override; if (fsa_state == S_ELECTION || fsa_state == S_PENDING) { crm_info("Sending update to local CIB in state: %s", fsa_state2string(fsa_state)); cib__set_call_options(call_opt, "update", cib_scope_local); } return call_opt; } /*! * \internal * \brief Check whether an action type should be recorded in the CIB * * \param[in] action Action type * * \return TRUE if action should be recorded, FALSE otherwise */ bool controld_action_is_recordable(const char *action) { return !pcmk__strcase_any_of(action, CRMD_ACTION_CANCEL, CRMD_ACTION_DELETE, CRMD_ACTION_NOTIFY, CRMD_ACTION_METADATA, NULL); } static void cib_delete_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { char *desc = user_data; if (rc == 0) { crm_debug("Deletion of %s (via CIB call %d) succeeded", desc, call_id); } else { crm_warn("Deletion of %s (via CIB call %d) failed: %s " CRM_XS " rc=%d", desc, call_id, pcmk_strerror(rc), rc); } } // Searches for various portions of node_state to delete // Match a particular node's node_state (takes node name 1x) #define XPATH_NODE_STATE "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']" // Node's lrm section (name 1x) #define XPATH_NODE_LRM XPATH_NODE_STATE "/" XML_CIB_TAG_LRM // Node's lrm_rsc_op entries and lrm_resource entries without lock (name 2x) #define XPATH_NODE_LRM_UNLOCKED XPATH_NODE_STATE "//" XML_LRM_TAG_RSC_OP \ "|" XPATH_NODE_STATE \ "//" XML_LRM_TAG_RESOURCE \ "[not(@" XML_CONFIG_ATTR_SHUTDOWN_LOCK ")]" // Node's transient_attributes section (name 1x) #define XPATH_NODE_ATTRS XPATH_NODE_STATE "/" XML_TAG_TRANSIENT_NODEATTRS // Everything under node_state (name 1x) #define XPATH_NODE_ALL XPATH_NODE_STATE "/*" // Unlocked history + transient attributes (name 3x) #define XPATH_NODE_ALL_UNLOCKED XPATH_NODE_LRM_UNLOCKED "|" XPATH_NODE_ATTRS /*! * \internal * \brief Delete subsection of a node's CIB node_state * * \param[in] uname Desired node * \param[in] section Subsection of node_state to delete * \param[in] options CIB call options to use */ void controld_delete_node_state(const char *uname, enum controld_section_e section, int options) { char *xpath = NULL; char *desc = NULL; CRM_CHECK(uname != NULL, return); switch (section) { case controld_section_lrm: xpath = crm_strdup_printf(XPATH_NODE_LRM, uname); desc = crm_strdup_printf("resource history for node %s", uname); break; case controld_section_lrm_unlocked: xpath = crm_strdup_printf(XPATH_NODE_LRM_UNLOCKED, uname, uname); desc = crm_strdup_printf("resource history (other than shutdown " "locks) for node %s", uname); break; case controld_section_attrs: xpath = crm_strdup_printf(XPATH_NODE_ATTRS, uname); desc = crm_strdup_printf("transient attributes for node %s", uname); break; case controld_section_all: xpath = crm_strdup_printf(XPATH_NODE_ALL, uname); desc = crm_strdup_printf("all state for node %s", uname); break; case controld_section_all_unlocked: xpath = crm_strdup_printf(XPATH_NODE_ALL_UNLOCKED, uname, uname, uname); desc = crm_strdup_printf("all state (other than shutdown locks) " "for node %s", uname); break; } if (fsa_cib_conn == NULL) { crm_warn("Unable to delete %s: no CIB connection", desc); free(desc); } else { int call_id; cib__set_call_options(options, "node state deletion", cib_quorum_override|cib_xpath|cib_multiple); call_id = fsa_cib_conn->cmds->remove(fsa_cib_conn, xpath, NULL, options); crm_info("Deleting %s (via CIB call %d) " CRM_XS " xpath=%s", desc, call_id, xpath); fsa_register_cib_callback(call_id, FALSE, desc, cib_delete_callback); // CIB library handles freeing desc } free(xpath); } // Takes node name and resource ID #define XPATH_RESOURCE_HISTORY "//" XML_CIB_TAG_STATE \ "[@" XML_ATTR_UNAME "='%s']/" \ XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \ "/" XML_LRM_TAG_RESOURCE \ "[@" XML_ATTR_ID "='%s']" // @TODO could add "and @XML_CONFIG_ATTR_SHUTDOWN_LOCK" to limit to locks /*! * \internal * \brief Clear resource history from CIB for a given resource and node * * \param[in] rsc_id ID of resource to be cleared * \param[in] node Node whose resource history should be cleared * \param[in] user_name ACL user name to use * \param[in] call_options CIB call options * * \return Standard Pacemaker return code */ int controld_delete_resource_history(const char *rsc_id, const char *node, const char *user_name, int call_options) { char *desc = NULL; char *xpath = NULL; int rc = pcmk_rc_ok; CRM_CHECK((rsc_id != NULL) && (node != NULL), return EINVAL); desc = crm_strdup_printf("resource history for %s on %s", rsc_id, node); if (fsa_cib_conn == NULL) { crm_err("Unable to clear %s: no CIB connection", desc); free(desc); return ENOTCONN; } // Ask CIB to delete the entry xpath = crm_strdup_printf(XPATH_RESOURCE_HISTORY, node, rsc_id); rc = cib_internal_op(fsa_cib_conn, PCMK__CIB_REQUEST_DELETE, NULL, xpath, NULL, NULL, call_options|cib_xpath, user_name); if (rc < 0) { rc = pcmk_legacy2rc(rc); crm_err("Could not delete resource status of %s on %s%s%s: %s " CRM_XS " rc=%d", rsc_id, node, (user_name? " for user " : ""), (user_name? user_name : ""), pcmk_rc_str(rc), rc); free(desc); free(xpath); return rc; } if (pcmk_is_set(call_options, cib_sync_call)) { if (pcmk_is_set(call_options, cib_dryrun)) { crm_debug("Deletion of %s would succeed", desc); } else { crm_debug("Deletion of %s succeeded", desc); } free(desc); } else { crm_info("Clearing %s (via CIB call %d) " CRM_XS " xpath=%s", desc, rc, xpath); fsa_register_cib_callback(rc, FALSE, desc, cib_delete_callback); // CIB library handles freeing desc } free(xpath); return pcmk_rc_ok; } diff --git a/daemons/controld/controld_election.c b/daemons/controld/controld_election.c index 1b54f32c24..bab632f145 100644 --- a/daemons/controld/controld_election.c +++ b/daemons/controld/controld_election.c @@ -1,280 +1,281 @@ /* * Copyright 2004-2022 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include extern pcmk__output_t *logger_out; static election_t *fsa_election = NULL; static gboolean election_win_cb(gpointer data) { register_fsa_input(C_FSA_INTERNAL, I_ELECTION_DC, NULL); return FALSE; } void controld_election_init(const char *uname) { fsa_election = election_init("DC", uname, 60000 /*60s*/, election_win_cb); } void controld_remove_voter(const char *uname) { election_remove(fsa_election, uname); if (pcmk__str_eq(uname, fsa_our_dc, pcmk__str_casei)) { /* Clear any election dampening in effect. Otherwise, if the lost DC had * just won, an immediate new election could fizzle out with no new DC. */ election_clear_dampening(fsa_election); } } void controld_election_fini(void) { election_fini(fsa_election); fsa_election = NULL; } void controld_set_election_period(const char *value) { election_timeout_set_period(fsa_election, crm_parse_interval_spec(value)); } void controld_stop_election_timer(void) { election_timeout_stop(fsa_election); } /* A_ELECTION_VOTE */ void do_election_vote(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { gboolean not_voting = FALSE; /* don't vote if we're in one of these states or wanting to shut down */ switch (cur_state) { case S_STARTING: case S_RECOVERY: case S_STOPPING: case S_TERMINATE: crm_warn("Not voting in election, we're in state %s", fsa_state2string(cur_state)); not_voting = TRUE; break; case S_ELECTION: case S_INTEGRATION: case S_RELEASE_DC: break; default: crm_err("Broken? Voting in state %s", fsa_state2string(cur_state)); break; } if (not_voting == FALSE) { if (pcmk_is_set(fsa_input_register, R_STARTING)) { not_voting = TRUE; } } if (not_voting) { if (AM_I_DC) { register_fsa_input(C_FSA_INTERNAL, I_RELEASE_DC, NULL); } else { register_fsa_input(C_FSA_INTERNAL, I_PENDING, NULL); } return; } election_vote(fsa_election); return; } void do_election_check(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { if (fsa_state == S_ELECTION) { election_check(fsa_election); } else { crm_debug("Ignoring election check because we are not in an election"); } } /* A_ELECTION_COUNT */ void do_election_count_vote(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { enum election_result rc = 0; ha_msg_input_t *vote = fsa_typed_data(fsa_dt_ha_msg); if(crm_peer_cache == NULL) { if (!pcmk_is_set(fsa_input_register, R_SHUTDOWN)) { crm_err("Internal error, no peer cache"); } return; } rc = election_count_vote(fsa_election, vote->msg, cur_state != S_STARTING); switch(rc) { case election_start: election_reset(fsa_election); register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); break; case election_lost: update_dc(NULL); if (fsa_input_register & R_THE_DC) { register_fsa_input(C_FSA_INTERNAL, I_RELEASE_DC, NULL); - fsa_cib_conn->cmds->set_slave(fsa_cib_conn, cib_scope_local); + fsa_cib_conn->cmds->set_secondary(fsa_cib_conn, + cib_scope_local); } else if (cur_state != S_STARTING) { register_fsa_input(C_FSA_INTERNAL, I_PENDING, NULL); } break; default: crm_trace("Election message resulted in state %d", rc); } } static void feature_update_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) { if (rc != pcmk_ok) { fsa_data_t *msg_data = NULL; crm_notice("Feature update failed: %s "CRM_XS" rc=%d", pcmk_strerror(rc), rc); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } /* A_DC_TAKEOVER */ void do_dc_takeover(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { int rc = pcmk_ok; xmlNode *cib = NULL; const char *cluster_type = name_for_cluster_type(get_cluster_type()); pid_t watchdog = pcmk__locate_sbd(); crm_info("Taking over DC status for this partition"); controld_set_fsa_input_flags(R_THE_DC); execute_stonith_cleanup(); election_reset(fsa_election); controld_set_fsa_input_flags(R_JOIN_OK|R_INVOKE_PE); fsa_cib_conn->cmds->set_primary(fsa_cib_conn, cib_scope_local); cib = create_xml_node(NULL, XML_TAG_CIB); crm_xml_add(cib, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET); fsa_cib_update(XML_TAG_CIB, cib, cib_quorum_override, rc, NULL); fsa_register_cib_callback(rc, FALSE, NULL, feature_update_callback); cib__update_node_attr(logger_out, fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL, XML_ATTR_HAVE_WATCHDOG, pcmk__btoa(watchdog), NULL, NULL); cib__update_node_attr(logger_out, fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL, "dc-version", PACEMAKER_VERSION "-" BUILD_VERSION, NULL, NULL); cib__update_node_attr(logger_out, fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL, "cluster-infrastructure", cluster_type, NULL, NULL); #if SUPPORT_COROSYNC if (fsa_cluster_name == NULL && is_corosync_cluster()) { char *cluster_name = pcmk__corosync_cluster_name(); if (cluster_name) { cib__update_node_attr(logger_out, fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL, "cluster-name", cluster_name, NULL, NULL); } free(cluster_name); } #endif mainloop_set_trigger(config_read); free_xml(cib); } /* A_DC_RELEASE */ void do_dc_release(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { if (action & A_DC_RELEASE) { crm_debug("Releasing the role of DC"); controld_clear_fsa_input_flags(R_THE_DC); controld_expect_sched_reply(NULL); } else if (action & A_DC_RELEASED) { crm_info("DC role released"); #if 0 if (are there errors) { /* we can't stay up if not healthy */ /* or perhaps I_ERROR and go to S_RECOVER? */ result = I_SHUTDOWN; } #endif if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) { xmlNode *update = NULL; crm_node_t *node = crm_get_peer(0, fsa_our_uname); pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN); update = create_node_state_update(node, node_update_expected, NULL, __func__); /* Don't need a based response because controld will stop. */ fsa_cib_anon_update_discard_reply(XML_CIB_TAG_STATUS, update); free_xml(update); } register_fsa_input(C_FSA_INTERNAL, I_RELEASE_SUCCESS, NULL); } else { crm_err("Unknown DC action %s", fsa_action2string(action)); } crm_trace("Am I still the DC? %s", AM_I_DC ? XML_BOOLEAN_YES : XML_BOOLEAN_NO); } diff --git a/daemons/controld/controld_fsa.c b/daemons/controld/controld_fsa.c index 8dcb1a5e7b..270d522212 100644 --- a/daemons/controld/controld_fsa.c +++ b/daemons/controld/controld_fsa.c @@ -1,665 +1,665 @@ /* - * Copyright 2004-2021 the Pacemaker project contributors + * Copyright 2004-2022 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include // uint64_t #include #include #include #include #include #include #include #include #include #include #include char *fsa_our_dc = NULL; cib_t *fsa_cib_conn = NULL; char *fsa_our_dc_version = NULL; char *fsa_our_uuid = NULL; char *fsa_our_uname = NULL; char *fsa_cluster_name = NULL; gboolean do_fsa_stall = FALSE; uint64_t fsa_input_register = 0; uint64_t fsa_actions = A_NOTHING; enum crmd_fsa_state fsa_state = S_STARTING; extern uint highest_born_on; extern uint num_join_invites; #define DOT_PREFIX "actions:trace: " #define do_dot_log(fmt, args...) crm_trace( fmt, ##args) static void do_state_transition(enum crmd_fsa_state cur_state, enum crmd_fsa_state next_state, fsa_data_t *msg_data); void s_crmd_fsa_actions(fsa_data_t * fsa_data); void log_fsa_input(fsa_data_t * stored_msg); void init_dotfile(void); void init_dotfile(void) { do_dot_log(DOT_PREFIX "digraph \"g\" {"); do_dot_log(DOT_PREFIX " size = \"30,30\""); do_dot_log(DOT_PREFIX " graph ["); do_dot_log(DOT_PREFIX " fontsize = \"12\""); do_dot_log(DOT_PREFIX " fontname = \"Times-Roman\""); do_dot_log(DOT_PREFIX " fontcolor = \"black\""); do_dot_log(DOT_PREFIX " bb = \"0,0,398.922306,478.927856\""); do_dot_log(DOT_PREFIX " color = \"black\""); do_dot_log(DOT_PREFIX " ]"); do_dot_log(DOT_PREFIX " node ["); do_dot_log(DOT_PREFIX " fontsize = \"12\""); do_dot_log(DOT_PREFIX " fontname = \"Times-Roman\""); do_dot_log(DOT_PREFIX " fontcolor = \"black\""); do_dot_log(DOT_PREFIX " shape = \"ellipse\""); do_dot_log(DOT_PREFIX " color = \"black\""); do_dot_log(DOT_PREFIX " ]"); do_dot_log(DOT_PREFIX " edge ["); do_dot_log(DOT_PREFIX " fontsize = \"12\""); do_dot_log(DOT_PREFIX " fontname = \"Times-Roman\""); do_dot_log(DOT_PREFIX " fontcolor = \"black\""); do_dot_log(DOT_PREFIX " color = \"black\""); do_dot_log(DOT_PREFIX " ]"); do_dot_log(DOT_PREFIX "// special nodes"); do_dot_log(DOT_PREFIX " \"S_PENDING\" "); do_dot_log(DOT_PREFIX " ["); do_dot_log(DOT_PREFIX " color = \"blue\""); do_dot_log(DOT_PREFIX " fontcolor = \"blue\""); do_dot_log(DOT_PREFIX " ]"); do_dot_log(DOT_PREFIX " \"S_TERMINATE\" "); do_dot_log(DOT_PREFIX " ["); do_dot_log(DOT_PREFIX " color = \"red\""); do_dot_log(DOT_PREFIX " fontcolor = \"red\""); do_dot_log(DOT_PREFIX " ]"); do_dot_log(DOT_PREFIX "// DC only nodes"); do_dot_log(DOT_PREFIX " \"S_INTEGRATION\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX " \"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX " \"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX " \"S_RELEASE_DC\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX " \"S_IDLE\" [ fontcolor = \"green\" ]"); } static void do_fsa_action(fsa_data_t * fsa_data, long long an_action, void (*function) (long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)) { controld_clear_fsa_action_flags(an_action); crm_trace(DOT_PREFIX "\t// %s", fsa_action2string(an_action)); function(an_action, fsa_data->fsa_cause, fsa_state, fsa_data->fsa_input, fsa_data); } static const uint64_t startup_actions = A_STARTUP | A_CIB_START | A_LRM_CONNECT | A_HA_CONNECT | A_READCONFIG | A_STARTED | A_CL_JOIN_QUERY; // A_LOG, A_WARN, A_ERROR void do_log(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { unsigned log_type = LOG_TRACE; if (action & A_LOG) { log_type = LOG_INFO; } else if (action & A_WARN) { log_type = LOG_WARNING; } else if (action & A_ERROR) { log_type = LOG_ERR; } do_crm_log(log_type, "Input %s received in state %s from %s", fsa_input2string(msg_data->fsa_input), fsa_state2string(cur_state), msg_data->origin); if (msg_data->data_type == fsa_dt_ha_msg) { ha_msg_input_t *input = fsa_typed_data(msg_data->data_type); crm_log_xml_debug(input->msg, __func__); } else if (msg_data->data_type == fsa_dt_xml) { xmlNode *input = fsa_typed_data(msg_data->data_type); crm_log_xml_debug(input, __func__); } else if (msg_data->data_type == fsa_dt_lrm) { lrmd_event_data_t *input = fsa_typed_data(msg_data->data_type); do_crm_log(log_type, "Resource %s: Call ID %d returned %d (%d)." " New status if rc=0: %s", input->rsc_id, input->call_id, input->rc, input->op_status, (char *)input->user_data); } } enum crmd_fsa_state s_crmd_fsa(enum crmd_fsa_cause cause) { fsa_data_t *fsa_data = NULL; uint64_t register_copy = fsa_input_register; uint64_t new_actions = A_NOTHING; enum crmd_fsa_state last_state; crm_trace("FSA invoked with Cause: %s\tState: %s", fsa_cause2string(cause), fsa_state2string(fsa_state)); fsa_dump_actions(fsa_actions, "Initial"); do_fsa_stall = FALSE; if ((fsa_message_queue == NULL) && (fsa_actions != A_NOTHING)) { /* fake the first message so we can get into the loop */ fsa_data = calloc(1, sizeof(fsa_data_t)); fsa_data->fsa_input = I_NULL; fsa_data->fsa_cause = C_FSA_INTERNAL; fsa_data->origin = __func__; fsa_data->data_type = fsa_dt_none; fsa_message_queue = g_list_append(fsa_message_queue, fsa_data); fsa_data = NULL; } while ((fsa_message_queue != NULL) && !do_fsa_stall) { crm_trace("Checking messages (%d remaining)", g_list_length(fsa_message_queue)); fsa_data = get_message(); if(fsa_data == NULL) { continue; } log_fsa_input(fsa_data); /* add any actions back to the queue */ controld_set_fsa_action_flags(fsa_data->actions); fsa_dump_actions(fsa_data->actions, "Restored actions"); /* get the next batch of actions */ new_actions = crmd_fsa_actions[fsa_data->fsa_input][fsa_state]; controld_set_fsa_action_flags(new_actions); fsa_dump_actions(new_actions, "New actions"); if (fsa_data->fsa_input != I_NULL && fsa_data->fsa_input != I_ROUTER) { crm_debug("Processing %s: [ state=%s cause=%s origin=%s ]", fsa_input2string(fsa_data->fsa_input), fsa_state2string(fsa_state), fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin); } /* logging : *before* the state is changed */ if (pcmk_is_set(fsa_actions, A_ERROR)) { do_fsa_action(fsa_data, A_ERROR, do_log); } if (pcmk_is_set(fsa_actions, A_WARN)) { do_fsa_action(fsa_data, A_WARN, do_log); } if (pcmk_is_set(fsa_actions, A_LOG)) { do_fsa_action(fsa_data, A_LOG, do_log); } /* update state variables */ last_state = fsa_state; fsa_state = crmd_fsa_state[fsa_data->fsa_input][fsa_state]; /* * Remove certain actions during shutdown */ if (fsa_state == S_STOPPING || ((fsa_input_register & R_SHUTDOWN) == R_SHUTDOWN)) { controld_clear_fsa_action_flags(startup_actions); } /* * Hook for change of state. * Allows actions to be added or removed when entering a state */ if (last_state != fsa_state) { do_state_transition(last_state, fsa_state, fsa_data); } else { do_dot_log(DOT_PREFIX "\t// FSA input: State=%s \tCause=%s" " \tInput=%s \tOrigin=%s() \tid=%d", fsa_state2string(fsa_state), fsa_cause2string(fsa_data->fsa_cause), fsa_input2string(fsa_data->fsa_input), fsa_data->origin, fsa_data->id); } /* start doing things... */ s_crmd_fsa_actions(fsa_data); delete_fsa_input(fsa_data); fsa_data = NULL; } if ((fsa_message_queue != NULL) || (fsa_actions != A_NOTHING) || do_fsa_stall) { crm_debug("Exiting the FSA: queue=%d, fsa_actions=%#llx, stalled=%s", g_list_length(fsa_message_queue), (unsigned long long) fsa_actions, pcmk__btoa(do_fsa_stall)); } else { crm_trace("Exiting the FSA"); } /* cleanup inputs? */ if (register_copy != fsa_input_register) { uint64_t same = register_copy & fsa_input_register; fsa_dump_inputs(LOG_DEBUG, "Added", fsa_input_register ^ same); fsa_dump_inputs(LOG_DEBUG, "Removed", register_copy ^ same); } fsa_dump_actions(fsa_actions, "Remaining"); fsa_dump_queue(LOG_DEBUG); return fsa_state; } void s_crmd_fsa_actions(fsa_data_t * fsa_data) { /* * Process actions in order of priority but do only one * action at a time to avoid complicating the ordering. */ CRM_CHECK(fsa_data != NULL, return); while (fsa_actions != A_NOTHING && do_fsa_stall == FALSE) { /* regular action processing in order of action priority * * Make sure all actions that connect to required systems * are performed first */ if (fsa_actions & A_ERROR) { do_fsa_action(fsa_data, A_ERROR, do_log); } else if (fsa_actions & A_WARN) { do_fsa_action(fsa_data, A_WARN, do_log); } else if (fsa_actions & A_LOG) { do_fsa_action(fsa_data, A_LOG, do_log); /* get out of here NOW! before anything worse happens */ } else if (fsa_actions & A_EXIT_1) { do_fsa_action(fsa_data, A_EXIT_1, do_exit); /* sub-system restart */ } else if ((fsa_actions & O_LRM_RECONNECT) == O_LRM_RECONNECT) { do_fsa_action(fsa_data, O_LRM_RECONNECT, do_lrm_control); } else if ((fsa_actions & O_CIB_RESTART) == O_CIB_RESTART) { do_fsa_action(fsa_data, O_CIB_RESTART, do_cib_control); } else if ((fsa_actions & O_PE_RESTART) == O_PE_RESTART) { do_fsa_action(fsa_data, O_PE_RESTART, do_pe_control); } else if ((fsa_actions & O_TE_RESTART) == O_TE_RESTART) { do_fsa_action(fsa_data, O_TE_RESTART, do_te_control); /* essential start tasks */ } else if (fsa_actions & A_STARTUP) { do_fsa_action(fsa_data, A_STARTUP, do_startup); } else if (fsa_actions & A_CIB_START) { do_fsa_action(fsa_data, A_CIB_START, do_cib_control); } else if (fsa_actions & A_HA_CONNECT) { do_fsa_action(fsa_data, A_HA_CONNECT, do_ha_control); } else if (fsa_actions & A_READCONFIG) { do_fsa_action(fsa_data, A_READCONFIG, do_read_config); /* sub-system start/connect */ } else if (fsa_actions & A_LRM_CONNECT) { do_fsa_action(fsa_data, A_LRM_CONNECT, do_lrm_control); } else if (fsa_actions & A_TE_START) { do_fsa_action(fsa_data, A_TE_START, do_te_control); } else if (fsa_actions & A_PE_START) { do_fsa_action(fsa_data, A_PE_START, do_pe_control); /* Timers */ /* else if(fsa_actions & O_DC_TIMER_RESTART) { do_fsa_action(fsa_data, O_DC_TIMER_RESTART, do_timer_control) */ ; } else if (fsa_actions & A_DC_TIMER_STOP) { do_fsa_action(fsa_data, A_DC_TIMER_STOP, do_timer_control); } else if (fsa_actions & A_INTEGRATE_TIMER_STOP) { do_fsa_action(fsa_data, A_INTEGRATE_TIMER_STOP, do_timer_control); } else if (fsa_actions & A_INTEGRATE_TIMER_START) { do_fsa_action(fsa_data, A_INTEGRATE_TIMER_START, do_timer_control); } else if (fsa_actions & A_FINALIZE_TIMER_STOP) { do_fsa_action(fsa_data, A_FINALIZE_TIMER_STOP, do_timer_control); } else if (fsa_actions & A_FINALIZE_TIMER_START) { do_fsa_action(fsa_data, A_FINALIZE_TIMER_START, do_timer_control); /* * Highest priority actions */ } else if (fsa_actions & A_MSG_ROUTE) { do_fsa_action(fsa_data, A_MSG_ROUTE, do_msg_route); } else if (fsa_actions & A_RECOVER) { do_fsa_action(fsa_data, A_RECOVER, do_recover); } else if (fsa_actions & A_CL_JOIN_RESULT) { do_fsa_action(fsa_data, A_CL_JOIN_RESULT, do_cl_join_finalize_respond); } else if (fsa_actions & A_CL_JOIN_REQUEST) { do_fsa_action(fsa_data, A_CL_JOIN_REQUEST, do_cl_join_offer_respond); } else if (fsa_actions & A_SHUTDOWN_REQ) { do_fsa_action(fsa_data, A_SHUTDOWN_REQ, do_shutdown_req); } else if (fsa_actions & A_ELECTION_VOTE) { do_fsa_action(fsa_data, A_ELECTION_VOTE, do_election_vote); } else if (fsa_actions & A_ELECTION_COUNT) { do_fsa_action(fsa_data, A_ELECTION_COUNT, do_election_count_vote); } else if (fsa_actions & A_LRM_EVENT) { do_fsa_action(fsa_data, A_LRM_EVENT, do_lrm_event); /* * High priority actions */ } else if (fsa_actions & A_STARTED) { do_fsa_action(fsa_data, A_STARTED, do_started); } else if (fsa_actions & A_CL_JOIN_QUERY) { do_fsa_action(fsa_data, A_CL_JOIN_QUERY, do_cl_join_query); } else if (fsa_actions & A_DC_TIMER_START) { do_fsa_action(fsa_data, A_DC_TIMER_START, do_timer_control); /* * Medium priority actions * - Membership */ } else if (fsa_actions & A_DC_TAKEOVER) { do_fsa_action(fsa_data, A_DC_TAKEOVER, do_dc_takeover); } else if (fsa_actions & A_DC_RELEASE) { do_fsa_action(fsa_data, A_DC_RELEASE, do_dc_release); } else if (fsa_actions & A_DC_JOIN_FINAL) { do_fsa_action(fsa_data, A_DC_JOIN_FINAL, do_dc_join_final); } else if (fsa_actions & A_ELECTION_CHECK) { do_fsa_action(fsa_data, A_ELECTION_CHECK, do_election_check); } else if (fsa_actions & A_ELECTION_START) { do_fsa_action(fsa_data, A_ELECTION_START, do_election_vote); } else if (fsa_actions & A_DC_JOIN_OFFER_ALL) { do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ALL, do_dc_join_offer_all); } else if (fsa_actions & A_DC_JOIN_OFFER_ONE) { do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ONE, do_dc_join_offer_one); } else if (fsa_actions & A_DC_JOIN_PROCESS_REQ) { do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_REQ, do_dc_join_filter_offer); } else if (fsa_actions & A_DC_JOIN_PROCESS_ACK) { do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_ACK, do_dc_join_ack); } else if (fsa_actions & A_DC_JOIN_FINALIZE) { do_fsa_action(fsa_data, A_DC_JOIN_FINALIZE, do_dc_join_finalize); } else if (fsa_actions & A_CL_JOIN_ANNOUNCE) { do_fsa_action(fsa_data, A_CL_JOIN_ANNOUNCE, do_cl_join_announce); /* * Low(er) priority actions * Make sure the CIB is always updated before invoking the * scheduler, and the scheduler before the transition engine. */ } else if (fsa_actions & A_TE_HALT) { do_fsa_action(fsa_data, A_TE_HALT, do_te_invoke); } else if (fsa_actions & A_TE_CANCEL) { do_fsa_action(fsa_data, A_TE_CANCEL, do_te_invoke); } else if (fsa_actions & A_LRM_INVOKE) { do_fsa_action(fsa_data, A_LRM_INVOKE, do_lrm_invoke); } else if (fsa_actions & A_PE_INVOKE) { do_fsa_action(fsa_data, A_PE_INVOKE, do_pe_invoke); } else if (fsa_actions & A_TE_INVOKE) { do_fsa_action(fsa_data, A_TE_INVOKE, do_te_invoke); /* Shutdown actions */ } else if (fsa_actions & A_DC_RELEASED) { do_fsa_action(fsa_data, A_DC_RELEASED, do_dc_release); } else if (fsa_actions & A_PE_STOP) { do_fsa_action(fsa_data, A_PE_STOP, do_pe_control); } else if (fsa_actions & A_TE_STOP) { do_fsa_action(fsa_data, A_TE_STOP, do_te_control); } else if (fsa_actions & A_SHUTDOWN) { do_fsa_action(fsa_data, A_SHUTDOWN, do_shutdown); } else if (fsa_actions & A_LRM_DISCONNECT) { do_fsa_action(fsa_data, A_LRM_DISCONNECT, do_lrm_control); } else if (fsa_actions & A_HA_DISCONNECT) { do_fsa_action(fsa_data, A_HA_DISCONNECT, do_ha_control); } else if (fsa_actions & A_CIB_STOP) { do_fsa_action(fsa_data, A_CIB_STOP, do_cib_control); } else if (fsa_actions & A_STOP) { do_fsa_action(fsa_data, A_STOP, do_stop); /* exit gracefully */ } else if (fsa_actions & A_EXIT_0) { do_fsa_action(fsa_data, A_EXIT_0, do_exit); /* Error checking and reporting */ } else { crm_err("Action %s not supported "CRM_XS" %#llx", fsa_action2string(fsa_actions), (unsigned long long) fsa_actions); register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, fsa_data, NULL, __func__); } } } void log_fsa_input(fsa_data_t * stored_msg) { CRM_ASSERT(stored_msg); crm_trace("Processing queued input %d", stored_msg->id); if (stored_msg->fsa_cause == C_LRM_OP_CALLBACK) { crm_trace("FSA processing LRM callback from %s", stored_msg->origin); } else if (stored_msg->data == NULL) { crm_trace("FSA processing input from %s", stored_msg->origin); } else { ha_msg_input_t *ha_input = fsa_typed_data_adv(stored_msg, fsa_dt_ha_msg, __func__); crm_trace("FSA processing XML message from %s", stored_msg->origin); crm_log_xml_trace(ha_input->xml, "FSA message data"); } } static void check_join_counts(fsa_data_t *msg_data) { int count; guint npeers; count = crmd_join_phase_count(crm_join_finalized); if (count > 0) { crm_err("%d cluster node%s failed to confirm join", count, pcmk__plural_s(count)); crmd_join_phase_log(LOG_NOTICE); return; } npeers = crm_active_peers(); count = crmd_join_phase_count(crm_join_confirmed); if (count == npeers) { if (npeers == 1) { crm_debug("Sole active cluster node is fully joined"); } else { crm_debug("All %d active cluster nodes are fully joined", count); } } else if (count > npeers) { crm_err("New election needed because more nodes confirmed join " "than are in membership (%d > %u)", count, npeers); register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); } else if (saved_ccm_membership_id != crm_peer_seq) { crm_info("New join needed because membership changed (%llu -> %llu)", saved_ccm_membership_id, crm_peer_seq); register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL); } else { crm_warn("Only %d of %u active cluster nodes fully joined " "(%d did not respond to offer)", count, npeers, crmd_join_phase_count(crm_join_welcomed)); } } static void do_state_transition(enum crmd_fsa_state cur_state, enum crmd_fsa_state next_state, fsa_data_t *msg_data) { int level = LOG_INFO; int count = 0; gboolean clear_recovery_bit = TRUE; #if 0 uint64_t original_fsa_actions = fsa_actions; #endif enum crmd_fsa_cause cause = msg_data->fsa_cause; enum crmd_fsa_input current_input = msg_data->fsa_input; const char *state_from = fsa_state2string(cur_state); const char *state_to = fsa_state2string(next_state); const char *input = fsa_input2string(current_input); CRM_LOG_ASSERT(cur_state != next_state); do_dot_log(DOT_PREFIX "\t%s -> %s [ label=%s cause=%s origin=%s ]", state_from, state_to, input, fsa_cause2string(cause), msg_data->origin); if (cur_state == S_IDLE || next_state == S_IDLE) { level = LOG_NOTICE; } else if (cur_state == S_NOT_DC || next_state == S_NOT_DC) { level = LOG_NOTICE; } else if (cur_state == S_ELECTION) { level = LOG_NOTICE; } else if (cur_state == S_STARTING) { level = LOG_NOTICE; } else if (next_state == S_RECOVERY) { level = LOG_WARNING; } do_crm_log(level, "State transition %s -> %s " CRM_XS " input=%s cause=%s origin=%s", state_from, state_to, input, fsa_cause2string(cause), msg_data->origin); if (next_state != S_ELECTION && cur_state != S_RELEASE_DC) { controld_stop_election_timer(); } #if 0 if ((fsa_input_register & R_SHUTDOWN)) { controld_set_fsa_action_flags(A_DC_TIMER_STOP); } #endif if (next_state == S_INTEGRATION) { controld_set_fsa_action_flags(A_INTEGRATE_TIMER_START); } else { controld_set_fsa_action_flags(A_INTEGRATE_TIMER_STOP); } if (next_state == S_FINALIZE_JOIN) { controld_set_fsa_action_flags(A_FINALIZE_TIMER_START); } else { controld_set_fsa_action_flags(A_FINALIZE_TIMER_STOP); } if (next_state != S_PENDING) { controld_set_fsa_action_flags(A_DC_TIMER_STOP); } if (next_state != S_ELECTION) { highest_born_on = 0; } if (next_state != S_IDLE) { controld_stop_timer(recheck_timer); } if (cur_state == S_FINALIZE_JOIN && next_state == S_POLICY_ENGINE) { populate_cib_nodes(node_update_quick|node_update_all, __func__); } switch (next_state) { case S_PENDING: - fsa_cib_conn->cmds->set_slave(fsa_cib_conn, cib_scope_local); + fsa_cib_conn->cmds->set_secondary(fsa_cib_conn, cib_scope_local); /* fall through */ case S_ELECTION: crm_trace("Resetting our DC to NULL on transition to %s", fsa_state2string(next_state)); update_dc(NULL); break; case S_NOT_DC: election_trigger->counter = 0; purge_stonith_cleanup(); if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) { crm_info("(Re)Issuing shutdown request now" " that we have a new DC"); controld_set_fsa_action_flags(A_SHUTDOWN_REQ); } CRM_LOG_ASSERT(fsa_our_dc != NULL); if (fsa_our_dc == NULL) { crm_err("Reached S_NOT_DC without a DC" " being recorded"); } break; case S_RECOVERY: clear_recovery_bit = FALSE; break; case S_FINALIZE_JOIN: CRM_LOG_ASSERT(AM_I_DC); if (cause == C_TIMER_POPPED) { crm_warn("Progressed to state %s after %s", fsa_state2string(next_state), fsa_cause2string(cause)); } count = crmd_join_phase_count(crm_join_welcomed); if (count > 0) { crm_warn("%d cluster node%s failed to respond to join offer", count, pcmk__plural_s(count)); crmd_join_phase_log(LOG_NOTICE); } else { crm_debug("All cluster nodes (%d) responded to join offer", crmd_join_phase_count(crm_join_integrated)); } break; case S_POLICY_ENGINE: election_trigger->counter = 0; CRM_LOG_ASSERT(AM_I_DC); if (cause == C_TIMER_POPPED) { crm_info("Progressed to state %s after %s", fsa_state2string(next_state), fsa_cause2string(cause)); } check_join_counts(msg_data); break; case S_STOPPING: case S_TERMINATE: /* possibly redundant */ controld_set_fsa_input_flags(R_SHUTDOWN); break; case S_IDLE: CRM_LOG_ASSERT(AM_I_DC); if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) { crm_info("(Re)Issuing shutdown request now" " that we are the DC"); controld_set_fsa_action_flags(A_SHUTDOWN_REQ); } controld_start_recheck_timer(); break; default: break; } if (clear_recovery_bit && next_state != S_PENDING) { controld_clear_fsa_action_flags(A_RECOVER); } else if (clear_recovery_bit == FALSE) { controld_set_fsa_action_flags(A_RECOVER); } #if 0 if (original_fsa_actions != fsa_actions) { fsa_dump_actions(original_fsa_actions ^ fsa_actions, "New actions"); } #endif } diff --git a/include/crm/cib/cib_types.h b/include/crm/cib/cib_types.h index 25311a2e0a..b7d9dc0e72 100644 --- a/include/crm/cib/cib_types.h +++ b/include/crm/cib/cib_types.h @@ -1,175 +1,185 @@ /* * Copyright 2004-2022 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #ifndef PCMK__CRM_CIB_CIB_TYPES__H # define PCMK__CRM_CIB_CIB_TYPES__H # include // gboolean, GList # include // xmlNode # include # include #ifdef __cplusplus extern "C" { #endif /** * \file * \brief Data types for Cluster Information Base access * \ingroup cib */ enum cib_variant { cib_undefined, cib_native, cib_file, cib_remote, cib_database, }; enum cib_state { cib_connected_command, cib_connected_query, cib_disconnected }; enum cib_conn_type { cib_command, cib_query, cib_no_connection, cib_command_nonblocking, }; enum cib_call_options { cib_none = 0x00000000, cib_verbose = 0x00000001, //!< Prefer stderr to logs cib_xpath = 0x00000002, cib_multiple = 0x00000004, cib_can_create = 0x00000008, cib_discard_reply = 0x00000010, cib_no_children = 0x00000020, cib_xpath_address = 0x00000040, cib_mixed_update = 0x00000080, cib_scope_local = 0x00000100, cib_dryrun = 0x00000200, cib_sync_call = 0x00001000, cib_no_mtime = 0x00002000, cib_zero_copy = 0x00004000, cib_inhibit_notify = 0x00010000, cib_quorum_override = 0x00100000, cib_inhibit_bcast = 0x01000000, //!< \deprecated Will be removed in future cib_force_diff = 0x10000000 }; typedef struct cib_s cib_t; typedef struct cib_api_operations_s { int (*signon) (cib_t *cib, const char *name, enum cib_conn_type type); int (*signon_raw) (cib_t *cib, const char *name, enum cib_conn_type type, int *event_fd); int (*signoff) (cib_t *cib); int (*free) (cib_t *cib); int (*set_op_callback) (cib_t *cib, void (*callback) (const xmlNode *msg, int callid, int rc, xmlNode *output)); int (*add_notify_callback) (cib_t *cib, const char *event, void (*callback) (const char *event, xmlNode *msg)); int (*del_notify_callback) (cib_t *cib, const char *event, void (*callback) (const char *event, xmlNode *msg)); int (*set_connection_dnotify) (cib_t *cib, void (*dnotify) (gpointer user_data)); int (*inputfd) (cib_t *cib); int (*noop) (cib_t *cib, int call_options); int (*ping) (cib_t *cib, xmlNode **output_data, int call_options); int (*query) (cib_t *cib, const char *section, xmlNode **output_data, int call_options); int (*query_from) (cib_t *cib, const char *host, const char *section, xmlNode **output_data, int call_options); //! \deprecated This method will be removed and should not be used int (*is_master) (cib_t *cib); //! \deprecated Use the set_primary() method instead int (*set_master) (cib_t *cib, int call_options); int (*set_slave) (cib_t *cib, int call_options); //! \deprecated This method will be removed and should not be used int (*set_slave_all) (cib_t *cib, int call_options); int (*sync) (cib_t *cib, const char *section, int call_options); int (*sync_from) (cib_t *cib, const char *host, const char *section, int call_options); int (*upgrade) (cib_t *cib, int call_options); int (*bump_epoch) (cib_t *cib, int call_options); int (*create) (cib_t *cib, const char *section, xmlNode *data, int call_options); int (*modify) (cib_t *cib, const char *section, xmlNode *data, int call_options); int (*update) (cib_t *cib, const char *section, xmlNode *data, int call_options); int (*replace) (cib_t *cib, const char *section, xmlNode *data, int call_options); int (*remove) (cib_t *cib, const char *section, xmlNode *data, int call_options); int (*erase) (cib_t *cib, xmlNode **output_data, int call_options); //! \deprecated This method does nothing and should not be called int (*delete_absolute) (cib_t *cib, const char *section, xmlNode *data, int call_options); int (*quit) (cib_t *cib, int call_options); int (*register_notification) (cib_t *cib, const char *callback, int enabled); gboolean (*register_callback) (cib_t *cib, int call_id, int timeout, gboolean only_success, void *user_data, const char *callback_name, void (*callback) (xmlNode*, int, int, xmlNode*, void *)); gboolean (*register_callback_full)(cib_t *cib, int call_id, int timeout, gboolean only_success, void *user_data, const char *callback_name, void (*callback)(xmlNode *, int, int, xmlNode *, void *), void (*free_func)(void *)); /*! * \brief Set the local CIB manager as the cluster's primary instance * * \param[in] cib CIB connection * \param[in] call_options Group of enum cib_call_options flags * * \return Legacy Pacemaker return code (in particular, pcmk_ok on success) */ int (*set_primary)(cib_t *cib, int call_options); + + /*! + * \brief Set the local CIB manager as a secondary instance + * + * \param[in] cib CIB connection + * \param[in] call_options Group of enum cib_call_options flags + * + * \return Legacy Pacemaker return code (in particular, pcmk_ok on success) + */ + int (*set_secondary)(cib_t *cib, int call_options); } cib_api_operations_t; struct cib_s { enum cib_state state; enum cib_conn_type type; enum cib_variant variant; int call_id; int call_timeout; void *variant_opaque; void *delegate_fn; GList *notify_list; void (*op_callback) (const xmlNode *msg, int call_id, int rc, xmlNode *output); cib_api_operations_t *cmds; }; #ifdef __cplusplus } #endif #endif // PCMK__CRM_CIB_CIB_TYPES__H diff --git a/lib/cib/cib_client.c b/lib/cib/cib_client.c index 90f6d8e650..9edbdaf6f2 100644 --- a/lib/cib/cib_client.c +++ b/lib/cib/cib_client.c @@ -1,723 +1,725 @@ /* * Copyright 2004-2022 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include static GHashTable *cib_op_callback_table = NULL; int cib_client_set_op_callback(cib_t * cib, void (*callback) (const xmlNode * msg, int call_id, int rc, xmlNode * output)); int cib_client_add_notify_callback(cib_t * cib, const char *event, void (*callback) (const char *event, xmlNode * msg)); int cib_client_del_notify_callback(cib_t * cib, const char *event, void (*callback) (const char *event, xmlNode * msg)); gint ciblib_GCompareFunc(gconstpointer a, gconstpointer b); #define op_common(cib) do { \ if(cib == NULL) { \ return -EINVAL; \ } else if(cib->delegate_fn == NULL) { \ return -EPROTONOSUPPORT; \ } \ } while(0) static int cib_client_noop(cib_t * cib, int call_options) { op_common(cib); return cib_internal_op(cib, PCMK__CIB_REQUEST_NOOP, NULL, NULL, NULL, NULL, call_options, NULL); } static int cib_client_ping(cib_t * cib, xmlNode ** output_data, int call_options) { op_common(cib); return cib_internal_op(cib, CRM_OP_PING, NULL, NULL, NULL, output_data, call_options, NULL); } static int cib_client_query(cib_t * cib, const char *section, xmlNode ** output_data, int call_options) { return cib->cmds->query_from(cib, NULL, section, output_data, call_options); } static int cib_client_query_from(cib_t * cib, const char *host, const char *section, xmlNode ** output_data, int call_options) { op_common(cib); return cib_internal_op(cib, PCMK__CIB_REQUEST_QUERY, host, section, NULL, output_data, call_options, NULL); } static int is_primary(cib_t *cib) { op_common(cib); return cib_internal_op(cib, PCMK__CIB_REQUEST_IS_PRIMARY, NULL, NULL, NULL, NULL, cib_scope_local|cib_sync_call, NULL); } static int set_secondary(cib_t *cib, int call_options) { op_common(cib); return cib_internal_op(cib, PCMK__CIB_REQUEST_SECONDARY, NULL, NULL, NULL, NULL, call_options, NULL); } static int set_all_secondary(cib_t * cib, int call_options) { return -EPROTONOSUPPORT; } static int set_primary(cib_t *cib, int call_options) { op_common(cib); crm_trace("Adding cib_scope_local to options"); return cib_internal_op(cib, PCMK__CIB_REQUEST_PRIMARY, NULL, NULL, NULL, NULL, call_options|cib_scope_local, NULL); } static int cib_client_bump_epoch(cib_t * cib, int call_options) { op_common(cib); return cib_internal_op(cib, PCMK__CIB_REQUEST_BUMP, NULL, NULL, NULL, NULL, call_options, NULL); } static int cib_client_upgrade(cib_t * cib, int call_options) { op_common(cib); return cib_internal_op(cib, PCMK__CIB_REQUEST_UPGRADE, NULL, NULL, NULL, NULL, call_options, NULL); } static int cib_client_sync(cib_t * cib, const char *section, int call_options) { return cib->cmds->sync_from(cib, NULL, section, call_options); } static int cib_client_sync_from(cib_t * cib, const char *host, const char *section, int call_options) { op_common(cib); return cib_internal_op(cib, PCMK__CIB_REQUEST_SYNC_TO_ALL, host, section, NULL, NULL, call_options, NULL); } static int cib_client_create(cib_t * cib, const char *section, xmlNode * data, int call_options) { op_common(cib); return cib_internal_op(cib, PCMK__CIB_REQUEST_CREATE, NULL, section, data, NULL, call_options, NULL); } static int cib_client_modify(cib_t * cib, const char *section, xmlNode * data, int call_options) { op_common(cib); return cib_internal_op(cib, PCMK__CIB_REQUEST_MODIFY, NULL, section, data, NULL, call_options, NULL); } static int cib_client_update(cib_t * cib, const char *section, xmlNode * data, int call_options) { op_common(cib); return cib_internal_op(cib, PCMK__CIB_REQUEST_MODIFY, NULL, section, data, NULL, call_options, NULL); } static int cib_client_replace(cib_t * cib, const char *section, xmlNode * data, int call_options) { op_common(cib); return cib_internal_op(cib, PCMK__CIB_REQUEST_REPLACE, NULL, section, data, NULL, call_options, NULL); } static int cib_client_delete(cib_t * cib, const char *section, xmlNode * data, int call_options) { op_common(cib); return cib_internal_op(cib, PCMK__CIB_REQUEST_DELETE, NULL, section, data, NULL, call_options, NULL); } static int cib_client_delete_absolute(cib_t * cib, const char *section, xmlNode * data, int call_options) { op_common(cib); return cib_internal_op(cib, PCMK__CIB_REQUEST_ABS_DELETE, NULL, section, data, NULL, call_options, NULL); } static int cib_client_erase(cib_t * cib, xmlNode ** output_data, int call_options) { op_common(cib); return cib_internal_op(cib, PCMK__CIB_REQUEST_ERASE, NULL, NULL, NULL, output_data, call_options, NULL); } static void cib_destroy_op_callback(gpointer data) { cib_callback_client_t *blob = data; if (blob->timer && blob->timer->ref > 0) { g_source_remove(blob->timer->ref); } free(blob->timer); if (blob->user_data && blob->free_func) { blob->free_func(blob->user_data); } free(blob); } static void destroy_op_callback_table(void) { if (cib_op_callback_table != NULL) { g_hash_table_destroy(cib_op_callback_table); cib_op_callback_table = NULL; } } char * get_shadow_file(const char *suffix) { char *cib_home = NULL; char *fullname = NULL; char *name = crm_strdup_printf("shadow.%s", suffix); const char *dir = getenv("CIB_shadow_dir"); if (dir == NULL) { uid_t uid = geteuid(); struct passwd *pwent = getpwuid(uid); const char *user = NULL; if (pwent) { user = pwent->pw_name; } else { user = getenv("USER"); crm_perror(LOG_ERR, "Assuming %s because cannot get user details for user ID %d", (user? user : "unprivileged user"), uid); } if (pcmk__strcase_any_of(user, "root", CRM_DAEMON_USER, NULL)) { dir = CRM_CONFIG_DIR; } else { const char *home = NULL; if ((home = getenv("HOME")) == NULL) { if (pwent) { home = pwent->pw_dir; } } dir = pcmk__get_tmpdir(); if (home && home[0] == '/') { int rc = 0; cib_home = crm_strdup_printf("%s/.cib", home); rc = mkdir(cib_home, 0700); if (rc < 0 && errno != EEXIST) { crm_perror(LOG_ERR, "Couldn't create user-specific shadow directory: %s", cib_home); errno = 0; } else { dir = cib_home; } } } } fullname = crm_strdup_printf("%s/%s", dir, name); free(cib_home); free(name); return fullname; } cib_t * cib_shadow_new(const char *shadow) { cib_t *new_cib = NULL; char *shadow_file = NULL; CRM_CHECK(shadow != NULL, return NULL); shadow_file = get_shadow_file(shadow); new_cib = cib_file_new(shadow_file); free(shadow_file); return new_cib; } cib_t * cib_new_no_shadow(void) { unsetenv("CIB_shadow"); return cib_new(); } cib_t * cib_new(void) { const char *value = getenv("CIB_shadow"); int port; if (value && value[0] != 0) { return cib_shadow_new(value); } value = getenv("CIB_file"); if (value) { return cib_file_new(value); } value = getenv("CIB_port"); if (value) { gboolean encrypted = TRUE; const char *server = getenv("CIB_server"); const char *user = getenv("CIB_user"); const char *pass = getenv("CIB_passwd"); /* We don't ensure port is valid (>= 0) because cib_new() currently * can't return NULL in practice, and introducing a NULL return here * could cause core dumps that would previously just cause signon() * failures. */ pcmk__scan_port(value, &port); value = getenv("CIB_encrypted"); if (value && crm_is_true(value) == FALSE) { crm_info("Disabling TLS"); encrypted = FALSE; } if (user == NULL) { user = CRM_DAEMON_USER; crm_info("Defaulting to user: %s", user); } if (server == NULL) { server = "localhost"; crm_info("Defaulting to localhost"); } return cib_remote_new(server, user, pass, port, encrypted); } return cib_native_new(); } /*! * \internal * \brief Create a generic CIB connection instance * * \return Newly allocated and initialized cib_t instance * * \note This is called by each variant's cib_*_new() function before setting * variant-specific values. */ cib_t * cib_new_variant(void) { cib_t *new_cib = NULL; new_cib = calloc(1, sizeof(cib_t)); if (new_cib == NULL) { return NULL; } remove_cib_op_callback(0, TRUE); /* remove all */ new_cib->call_id = 1; new_cib->variant = cib_undefined; new_cib->type = cib_no_connection; new_cib->state = cib_disconnected; new_cib->op_callback = NULL; new_cib->variant_opaque = NULL; new_cib->notify_list = NULL; /* the rest will get filled in by the variant constructor */ new_cib->cmds = calloc(1, sizeof(cib_api_operations_t)); if (new_cib->cmds == NULL) { free(new_cib); return NULL; } new_cib->cmds->set_op_callback = cib_client_set_op_callback; new_cib->cmds->add_notify_callback = cib_client_add_notify_callback; new_cib->cmds->del_notify_callback = cib_client_del_notify_callback; new_cib->cmds->register_callback = cib_client_register_callback; new_cib->cmds->register_callback_full = cib_client_register_callback_full; new_cib->cmds->noop = cib_client_noop; new_cib->cmds->ping = cib_client_ping; new_cib->cmds->query = cib_client_query; new_cib->cmds->sync = cib_client_sync; new_cib->cmds->query_from = cib_client_query_from; new_cib->cmds->sync_from = cib_client_sync_from; new_cib->cmds->is_master = is_primary; // Deprecated method new_cib->cmds->set_primary = set_primary; new_cib->cmds->set_master = set_primary; // Deprecated method + + new_cib->cmds->set_secondary = set_secondary; new_cib->cmds->set_slave = set_secondary; new_cib->cmds->set_slave_all = set_all_secondary; // Deprecated method new_cib->cmds->upgrade = cib_client_upgrade; new_cib->cmds->bump_epoch = cib_client_bump_epoch; new_cib->cmds->create = cib_client_create; new_cib->cmds->modify = cib_client_modify; new_cib->cmds->update = cib_client_update; new_cib->cmds->replace = cib_client_replace; new_cib->cmds->remove = cib_client_delete; new_cib->cmds->erase = cib_client_erase; new_cib->cmds->delete_absolute = cib_client_delete_absolute; return new_cib; } void cib_free_notify(cib_t *cib) { if (cib) { GList *list = cib->notify_list; while (list != NULL) { cib_notify_client_t *client = g_list_nth_data(list, 0); list = g_list_remove(list, client); free(client); } cib->notify_list = NULL; } } /*! * \brief Free all callbacks for a CIB connection * * \param[in] cib CIB connection to clean up */ void cib_free_callbacks(cib_t *cib) { cib_free_notify(cib); destroy_op_callback_table(); } /*! * \brief Free all memory used by CIB connection * * \param[in] cib CIB connection to delete */ void cib_delete(cib_t *cib) { cib_free_callbacks(cib); if (cib) { cib->cmds->free(cib); } } int cib_client_set_op_callback(cib_t * cib, void (*callback) (const xmlNode * msg, int call_id, int rc, xmlNode * output)) { if (callback == NULL) { crm_info("Un-Setting operation callback"); } else { crm_trace("Setting operation callback"); } cib->op_callback = callback; return pcmk_ok; } int cib_client_add_notify_callback(cib_t * cib, const char *event, void (*callback) (const char *event, xmlNode * msg)) { GList *list_item = NULL; cib_notify_client_t *new_client = NULL; if (cib->variant != cib_native && cib->variant != cib_remote) { return -EPROTONOSUPPORT; } crm_trace("Adding callback for %s events (%d)", event, g_list_length(cib->notify_list)); new_client = calloc(1, sizeof(cib_notify_client_t)); new_client->event = event; new_client->callback = callback; list_item = g_list_find_custom(cib->notify_list, new_client, ciblib_GCompareFunc); if (list_item != NULL) { crm_warn("Callback already present"); free(new_client); return -EINVAL; } else { cib->notify_list = g_list_append(cib->notify_list, new_client); cib->cmds->register_notification(cib, event, 1); crm_trace("Callback added (%d)", g_list_length(cib->notify_list)); } return pcmk_ok; } static int get_notify_list_event_count(cib_t * cib, const char *event) { GList *l = NULL; int count = 0; for (l = g_list_first(cib->notify_list); l; l = g_list_next(l)) { cib_notify_client_t *client = (cib_notify_client_t *)l->data; if (strcmp(client->event, event) == 0) { count++; } } crm_trace("event(%s) count : %d", event, count); return count; } int cib_client_del_notify_callback(cib_t * cib, const char *event, void (*callback) (const char *event, xmlNode * msg)) { GList *list_item = NULL; cib_notify_client_t *new_client = NULL; if (cib->variant != cib_native && cib->variant != cib_remote) { return -EPROTONOSUPPORT; } if (get_notify_list_event_count(cib, event) == 0) { crm_debug("The callback of the event does not exist(%s)", event); return pcmk_ok; } crm_debug("Removing callback for %s events", event); new_client = calloc(1, sizeof(cib_notify_client_t)); new_client->event = event; new_client->callback = callback; list_item = g_list_find_custom(cib->notify_list, new_client, ciblib_GCompareFunc); if (list_item != NULL) { cib_notify_client_t *list_client = list_item->data; cib->notify_list = g_list_remove(cib->notify_list, list_client); free(list_client); crm_trace("Removed callback"); } else { crm_trace("Callback not present"); } if (get_notify_list_event_count(cib, event) == 0) { /* When there is not the registration of the event, the processing turns off a notice. */ cib->cmds->register_notification(cib, event, 0); } free(new_client); return pcmk_ok; } gint ciblib_GCompareFunc(gconstpointer a, gconstpointer b) { int rc = 0; const cib_notify_client_t *a_client = a; const cib_notify_client_t *b_client = b; CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0); rc = strcmp(a_client->event, b_client->event); if (rc == 0) { if (a_client->callback == b_client->callback) { return 0; } else if (((long)a_client->callback) < ((long)b_client->callback)) { crm_trace("callbacks for %s are not equal: %p < %p", a_client->event, a_client->callback, b_client->callback); return -1; } crm_trace("callbacks for %s are not equal: %p > %p", a_client->event, a_client->callback, b_client->callback); return 1; } return rc; } static gboolean cib_async_timeout_handler(gpointer data) { struct timer_rec_s *timer = data; crm_debug("Async call %d timed out after %ds", timer->call_id, timer->timeout); cib_native_callback(timer->cib, NULL, timer->call_id, -ETIME); /* Always return TRUE, never remove the handler * We do that in remove_cib_op_callback() */ return TRUE; } gboolean cib_client_register_callback(cib_t * cib, int call_id, int timeout, gboolean only_success, void *user_data, const char *callback_name, void (*callback) (xmlNode *, int, int, xmlNode *, void *)) { return cib_client_register_callback_full(cib, call_id, timeout, only_success, user_data, callback_name, callback, NULL); } gboolean cib_client_register_callback_full(cib_t *cib, int call_id, int timeout, gboolean only_success, void *user_data, const char *callback_name, void (*callback)(xmlNode *, int, int, xmlNode *, void *), void (*free_func)(void *)) { cib_callback_client_t *blob = NULL; if (call_id < 0) { if (only_success == FALSE) { callback(NULL, call_id, call_id, NULL, user_data); } else { crm_warn("CIB call failed: %s", pcmk_strerror(call_id)); } if (user_data && free_func) { free_func(user_data); } return FALSE; } blob = calloc(1, sizeof(cib_callback_client_t)); blob->id = callback_name; blob->only_success = only_success; blob->user_data = user_data; blob->callback = callback; blob->free_func = free_func; if (timeout > 0) { struct timer_rec_s *async_timer = NULL; async_timer = calloc(1, sizeof(struct timer_rec_s)); blob->timer = async_timer; async_timer->cib = cib; async_timer->call_id = call_id; async_timer->timeout = timeout * 1000; async_timer->ref = g_timeout_add(async_timer->timeout, cib_async_timeout_handler, async_timer); } crm_trace("Adding callback %s for call %d", callback_name, call_id); pcmk__intkey_table_insert(cib_op_callback_table, call_id, blob); return TRUE; } void remove_cib_op_callback(int call_id, gboolean all_callbacks) { if (all_callbacks) { destroy_op_callback_table(); cib_op_callback_table = pcmk__intkey_table(cib_destroy_op_callback); } else { pcmk__intkey_table_remove(cib_op_callback_table, call_id); } } int num_cib_op_callbacks(void) { if (cib_op_callback_table == NULL) { return 0; } return g_hash_table_size(cib_op_callback_table); } static void cib_dump_pending_op(gpointer key, gpointer value, gpointer user_data) { int call = GPOINTER_TO_INT(key); cib_callback_client_t *blob = value; crm_debug("Call %d (%s): pending", call, pcmk__s(blob->id, "without ID")); } void cib_dump_pending_callbacks(void) { if (cib_op_callback_table == NULL) { return; } return g_hash_table_foreach(cib_op_callback_table, cib_dump_pending_op, NULL); } cib_callback_client_t* cib__lookup_id (int call_id) { return pcmk__intkey_table_lookup(cib_op_callback_table, call_id); }