diff --git a/crm/admin/crm_mon.c b/crm/admin/crm_mon.c index f5a3c2570b..e783697886 100644 --- a/crm/admin/crm_mon.c +++ b/crm/admin/crm_mon.c @@ -1,1015 +1,1018 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HAVE_GETOPT_H # include #endif /* GMainLoop *mainloop = NULL; */ #define OPTARGS "V?i:nrh:cdp:s1wX:oft" void usage(const char *cmd, int exit_status); void blank_screen(void); int print_status(xmlNode *cib); void print_warn(const char *descr); int print_simple_status(xmlNode *cib); /* #define printw_at(line, fmt...) move(line, 0); printw(fmt); line++ */ void wait_for_refresh(int offset, const char *prefix, int msec); int print_html_status(xmlNode *cib, const char *filename, gboolean web_cgi); void make_daemon(gboolean daemonize, const char *pidfile); gboolean mon_timer_popped(gpointer data); void mon_update(xmlNode*, int, int, xmlNode*,void*); char *xml_file = NULL; char *as_html_file = NULL; char *pid_file = NULL; gboolean as_console = FALSE; gboolean simple_status = FALSE; gboolean group_by_node = FALSE; gboolean inactive_resources = FALSE; gboolean web_cgi = FALSE; int interval = 15000; gboolean daemonize = FALSE; GMainLoop* mainloop = NULL; guint timer_id = 0; cib_t *cib_conn = NULL; int failed_connections = 0; gboolean one_shot = FALSE; gboolean has_warnings = FALSE; gboolean print_failcount = FALSE; gboolean print_operations = FALSE; gboolean print_timing = FALSE; #if CURSES_ENABLED # define print_as(fmt...) if(as_console) { \ printw(fmt); \ clrtoeol(); \ refresh(); \ } else { \ fprintf(stdout, fmt); \ } #else # define print_as(fmt...) fprintf(stdout, fmt); #endif int main(int argc, char **argv) { int argerr = 0; int flag; #ifdef HAVE_GETOPT_H int option_index = 0; static struct option long_options[] = { /* Top-level Options */ {"verbose", 0, 0, 'V'}, {"help", 0, 0, '?'}, {"interval", 1, 0, 'i'}, {"group-by-node", 0, 0, 'n'}, {"inactive", 0, 0, 'r'}, {"failcounts", 0, 0, 'f'}, {"operations", 0, 0, 'o'}, {"timing-details", 0, 0, 't'}, {"as-html", 1, 0, 'h'}, {"web-cgi", 0, 0, 'w'}, {"simple-status", 0, 0, 's'}, {"as-console", 0, 0, 'c'}, {"one-shot", 0, 0, '1'}, {"daemonize", 0, 0, 'd'}, {"pid-file", 0, 0, 'p'}, {"xml-file", 1, 0, 'x'}, {0, 0, 0, 0} }; #endif pid_file = crm_strdup("/tmp/ClusterMon.pid"); crm_log_init(basename(argv[0]), LOG_ERR-1, FALSE, FALSE, 0, NULL); if (strcmp(crm_system_name, "crm_mon.cgi")==0) { web_cgi = TRUE; one_shot = TRUE; } while (1) { #ifdef HAVE_GETOPT_H flag = getopt_long(argc, argv, OPTARGS, long_options, &option_index); #else flag = getopt(argc, argv, OPTARGS); #endif if (flag == -1) break; switch(flag) { case 'V': cl_log_enable_stderr(TRUE); alter_debug(DEBUG_INC); break; case 'i': interval = crm_get_msec(optarg); break; case 'n': group_by_node = TRUE; break; case 'r': inactive_resources = TRUE; break; case 'd': daemonize = TRUE; break; case 't': print_timing = TRUE; print_operations = TRUE; break; case 'o': print_operations = TRUE; break; case 'f': print_failcount = TRUE; break; case 'p': pid_file = crm_strdup(optarg); break; case 'X': xml_file = crm_strdup(optarg); one_shot = TRUE; break; case 'h': as_html_file = crm_strdup(optarg); break; case 'w': web_cgi = TRUE; one_shot = TRUE; break; case 'c': #if CURSES_ENABLED as_console = TRUE; #else printf("You need to have curses available at compile time to enable console mode\n"); argerr++; #endif break; case 's': simple_status = TRUE; one_shot = TRUE; break; case '1': one_shot = TRUE; break; case '?': usage(crm_system_name, LSB_EXIT_OK); break; default: printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag); ++argerr; break; } } if (optind < argc) { printf("non-option ARGV-elements: "); while (optind < argc) printf("%s ", argv[optind++]); printf("\n"); } if (argerr) { usage(crm_system_name, LSB_EXIT_GENERIC); } if(as_html_file == NULL && !web_cgi && !simple_status) { #if CURSES_ENABLED as_console = TRUE; #else printf("Defaulting to one-shot mode\n"); printf("You need to have curses available at compile time to enable console mode\n"); one_shot = TRUE; #endif } if(daemonize) { as_console = FALSE; } if(one_shot) { daemonize = FALSE; as_console = FALSE; } if(daemonize && as_html_file == NULL) { usage(crm_system_name, LSB_EXIT_GENERIC); } make_daemon(daemonize, pid_file); #if CURSES_ENABLED if(as_console) { initscr(); cbreak(); noecho(); } #endif crm_info("Starting %s", crm_system_name); mainloop = g_main_new(FALSE); if(one_shot == FALSE) { timer_id = Gmain_timeout_add( interval, mon_timer_popped, NULL); } else if(xml_file != NULL) { FILE *xml_strm = fopen(xml_file, "r"); xmlNode *cib_object = NULL; if(strstr(xml_file, ".bz2") != NULL) { cib_object = file2xml(xml_strm, TRUE); } else { cib_object = file2xml(xml_strm, FALSE); } if(xml_strm != NULL) { fclose(xml_strm); } one_shot = TRUE; mon_update(NULL, 0, cib_ok, cib_object, NULL); } mon_timer_popped(NULL); g_main_run(mainloop); return_to_orig_privs(); crm_info("Exiting %s", crm_system_name); #if CURSES_ENABLED if(as_console) { echo(); nocbreak(); endwin(); } #endif return 0; } gboolean mon_timer_popped(gpointer data) { int rc = cib_ok; int options = cib_scope_local|cib_sync_call; static gboolean need_pass = TRUE; xmlNode *output = NULL; if(timer_id > 0) { Gmain_timeout_remove(timer_id); } if(cib_conn == NULL) { crm_debug_4("Creating CIB connection"); cib_conn = cib_new(); CRM_CHECK(cib_conn != NULL, return FALSE); } if(as_console) { print_as("Updating...\n"); } else { crm_notice("Updating..."); } if(cib_conn->state != cib_connected_query && cib_conn->state != cib_connected_command) { crm_debug_4("Connecting to the CIB"); if(as_console) { print_as("Signing on...\n"); } if(need_pass && cib_conn->variant == cib_remote) { need_pass = FALSE; print_as("Password:"); } if(cib_ok == cib_conn->cmds->signon( cib_conn, crm_system_name, cib_query)) { failed_connections = 0; } else if (simple_status || one_shot) { fprintf(stdout, "Critical: Unable to connect to the CIB\n"); exit(2); } else { failed_connections++; CRM_DEV_ASSERT(cib_conn->cmds->signoff(cib_conn) == cib_ok); wait_for_refresh(0, "Not connected: ", 2*interval); return FALSE; } } if(as_console) { blank_screen(); print_as("Querying...\n"); } rc = cib_conn->cmds->query(cib_conn, NULL, &output, options); mon_update(NULL, 0, rc, output, NULL); free_xml(output); /* add_cib_op_callback(rc, FALSE, NULL, mon_update); */ return FALSE; } void mon_update(xmlNode *msg, int call_id, int rc, xmlNode *output, void*user_data) { const char *prefix = NULL; if(rc == cib_ok) { xmlNode *cib = NULL; #if CRM_DEPRECATED_SINCE_2_0_4 if( safe_str_eq(crm_element_name(output), XML_TAG_CIB) ) { cib = output; } else { cib = find_xml_node(output,XML_TAG_CIB,TRUE); } #else cib = output; CRM_DEV_ASSERT(safe_str_eq(crm_element_name(cib), XML_TAG_CIB)); #endif if(as_html_file || web_cgi) { print_html_status(cib, as_html_file, web_cgi); } else if (simple_status) { print_simple_status(cib); if (has_warnings) { exit(1); } } else { print_status(cib); } if(one_shot) { exit(LSB_EXIT_OK); } } else if(simple_status) { fprintf(stderr, "Critical: query failed: %s", cib_error2string(rc)); exit(LSB_EXIT_GENERIC); } else if(one_shot) { fprintf(stderr, "Query failed: %s", cib_error2string(rc)); exit(LSB_EXIT_GENERIC); } else { CRM_DEV_ASSERT(cib_conn->cmds->signoff(cib_conn) == cib_ok); print_as("Query failed: %s", cib_error2string(rc)); prefix = "Query failed! "; } wait_for_refresh(0, prefix, interval); } void wait_for_refresh(int offset, const char *prefix, int msec) { int lpc = msec / 1000; if(as_console == FALSE) { timer_id = Gmain_timeout_add(msec, mon_timer_popped, NULL); return; } crm_notice("%sRefresh in %ds...", prefix?prefix:"", lpc); while(lpc > 0) { #if CURSES_ENABLED move(0, 0); /* printw("%sRefresh in \033[01;32m%ds\033[00m...", prefix?prefix:"", lpc); */ printw("%sRefresh in %ds...\n", prefix?prefix:"", lpc); clrtoeol(); refresh(); #endif lpc--; if(lpc == 0) { timer_id = Gmain_timeout_add( 1000, mon_timer_popped, NULL); } else { sleep(1); } } } #define mon_warn(fmt...) do { \ if (!has_warnings) { \ print_as("Warning:"); \ } else { \ print_as(","); \ } \ print_as(fmt); \ has_warnings = TRUE; \ } while(0) int print_simple_status(xmlNode *cib) { node_t *dc = NULL; int nodes_online = 0; int nodes_standby = 0; pe_working_set_t data_set; set_working_set_defaults(&data_set); data_set.input = cib; cluster_status(&data_set); dc = data_set.dc_node; if(dc == NULL) { mon_warn("No DC "); } slist_iter(node, node_t, data_set.nodes, lpc2, if(node->details->standby) { nodes_standby++; } else if(node->details->online) { nodes_online++; } else { mon_warn("offline node: %s", node->details->uname); } ); if (!has_warnings) { print_as("Ok: %d nodes online", nodes_online); if (nodes_standby > 0) { print_as(", %d standby nodes", nodes_standby); } print_as(", %d resources configured", g_list_length(data_set.resources)); } print_as("\n"); data_set.input = NULL; cleanup_calculations(&data_set); return 0; } extern int get_failcount(node_t *node, resource_t *rsc, int *last_failure, pe_working_set_t *data_set); static void print_date(time_t time) { int lpc = 0; char date_str[26]; asctime_r(localtime(&time), date_str); for(; lpc < 26; lpc++) { if(date_str[lpc] == '\n') { date_str[lpc] = 0; } } print_as("'%s'", date_str); } static void print_rsc_summary(pe_working_set_t *data_set, node_t *node, resource_t *rsc, gboolean all) { gboolean printed = FALSE; time_t last_failure = 0; int failcount = get_failcount(node, rsc, (int*)&last_failure, data_set); if(all || failcount || last_failure > 0) { printed = TRUE; print_as(" %s: migration-threshold=%d", rsc->id, rsc->migration_threshold); } if(failcount > 0) { printed = TRUE; print_as(" fail-count=%d", failcount); } if(last_failure > 0) { printed = TRUE; print_as(" last-failure="); print_date(last_failure); } if(printed) { print_as("\n"); } } static void print_rsc_history(pe_working_set_t *data_set, node_t *node, xmlNode *rsc_entry) { GListPtr op_list = NULL; gboolean print_name = TRUE; GListPtr sorted_op_list = NULL; const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID); resource_t *rsc = pe_find_resource(data_set->resources, rsc_id); xml_child_iter_filter( rsc_entry, rsc_op, XML_LRM_TAG_RSC_OP, op_list = g_list_append(op_list, rsc_op); ); sorted_op_list = g_list_sort(op_list, sort_op_by_callid); slist_iter(xml_op, xmlNode, sorted_op_list, lpc, const char *value = NULL; const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK); const char *op_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC); const char *interval = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL); int rc = crm_parse_int(op_rc, "0"); if(safe_str_eq(task, CRMD_ACTION_STATUS) && safe_str_eq(interval, "0")) { task = "probe"; } if(rc == 7 && safe_str_eq(task, "probe")) { continue; } else if(safe_str_eq(task, CRMD_ACTION_NOTIFY)) { continue; } if(print_name) { print_name = FALSE; print_rsc_summary(data_set, node, rsc, TRUE); } print_as(" + %s:", task); if(safe_str_neq(interval, "0")) { print_as(" interval=%sms", interval); } if(print_timing) { int int_value; value = crm_element_value(xml_op, "last_rc_change"); if(value) { int_value = crm_parse_int(value, NULL); print_as(" last-rc-change="); print_date(int_value); } value = crm_element_value(xml_op, "last_run"); if(value) { int_value = crm_parse_int(value, NULL); print_as(" last-run="); print_date(int_value); } value = crm_element_value(xml_op, "exec_time"); if(value) { int_value = crm_parse_int(value, NULL); print_as(" exec-time="); print_date(int_value); } value = crm_element_value(xml_op, "queue_time"); if(value) { int_value = crm_parse_int(value, NULL); print_as(" queue-time="); print_date(int_value); } } print_as(" rc=%s (%s)\n", op_rc, execra_code2string(rc)); ); /* no need to free the contents */ g_list_free(sorted_op_list); } static void print_node_summary(pe_working_set_t *data_set, gboolean operations) { xmlNode *lrm_rsc = NULL; xmlNode *cib_status = get_object_root(XML_CIB_TAG_STATUS, data_set->input); if(operations) { print_as("\nOperations:\n"); } else { print_as("\nMigration summary::\n"); } xml_child_iter_filter( cib_status, node_state, XML_CIB_TAG_STATE, node_t *node = pe_find_node_id(data_set->nodes, ID(node_state)); print_as("* Node %s:\n", crm_element_value(node_state, XML_ATTR_UNAME)); lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE); lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE); xml_child_iter_filter( lrm_rsc, rsc_entry, XML_LRM_TAG_RESOURCE, if(operations) { print_rsc_history(data_set, node, rsc_entry); } else { const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID); resource_t *rsc = pe_find_resource(data_set->resources, rsc_id); print_rsc_summary(data_set, node, rsc, FALSE); } ); ); } int print_status(xmlNode *cib) { node_t *dc = NULL; static int updates = 0; pe_working_set_t data_set; char *since_epoch = NULL; time_t a_time = time(NULL); int configured_resources = 0; int print_opts = pe_print_ncurses; if(as_console) { blank_screen(); } else { print_opts = pe_print_printf; } updates++; set_working_set_defaults(&data_set); data_set.input = cib; cluster_status(&data_set); dc = data_set.dc_node; print_as("\n\n============\n"); if(a_time == (time_t)-1) { cl_perror("set_node_tstamp(): Invalid time returned"); return 1; } since_epoch = ctime(&a_time); if(since_epoch != NULL) { print_as("Last updated: %s", since_epoch); } if(dc == NULL) { print_as("Current DC: NONE\n"); } else { print_as("Current DC: %s (%s)\n", dc->details->uname, dc->details->id); } slist_iter(rsc, resource_t, data_set.resources, lpc, if(is_not_set(rsc->flags, pe_rsc_orphan)) { configured_resources++; } ); print_as("%d Nodes configured.\n", g_list_length(data_set.nodes)); print_as("%d Resources configured.\n", configured_resources); print_as("============\n\n"); slist_iter(node, node_t, data_set.nodes, lpc2, const char *node_mode = "OFFLINE"; - if(node->details->standby && node->details->online) { + if(node->details->pending) { + node_mode = "(re-)joining cluster"; + + } else if(node->details->standby && node->details->online) { node_mode = "standby"; } else if(node->details->standby) { node_mode = "OFFLINE (standby)"; } else if(node->details->online) { node_mode = "online"; } print_as("Node: %s (%s): %s\n", node->details->uname, node->details->id, node_mode); if(group_by_node) { slist_iter(rsc, resource_t, node->details->running_rsc, lpc2, rsc->fns->print( rsc, "\t", print_opts|pe_print_rsconly, stdout); ); } ); if(group_by_node == FALSE && inactive_resources) { print_as("\nFull list of resources:\n"); } else if(inactive_resources) { print_as("\nInactive resources:\n"); } if(group_by_node == FALSE || inactive_resources) { print_as("\n"); slist_iter(rsc, resource_t, data_set.resources, lpc2, gboolean is_active = rsc->fns->active(rsc, TRUE); gboolean partially_active = rsc->fns->active(rsc, FALSE); if(is_set(rsc->flags, pe_rsc_orphan) && is_active == FALSE) { continue; } else if(group_by_node == FALSE) { if(partially_active || inactive_resources) { rsc->fns->print(rsc, NULL, print_opts, stdout); } } else if(is_active == FALSE && inactive_resources) { rsc->fns->print(rsc, NULL, print_opts, stdout); } ); } if(print_operations || print_failcount) { print_node_summary(&data_set, print_operations); } if(xml_has_children(data_set.failed)) { print_as("\nFailed actions:\n"); xml_child_iter(data_set.failed, xml_op, const char *id = ID(xml_op); const char *rc = crm_element_value(xml_op, XML_LRM_ATTR_RC); const char *node = crm_element_value(xml_op, XML_ATTR_UNAME); const char *last = crm_element_value(xml_op, "last_run"); const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID); const char *status_s = crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS); int status = crm_parse_int(status_s, "0"); print_as(" %s (node=%s, call=%s, rc=%s", id, node, call, rc); if(last) { time_t run_at = crm_parse_int(last, "0"); print_as(", last-run=%s, queued=%sms, exec=%sms\n", ctime(&run_at), crm_element_value(xml_op, "exec_time"), crm_element_value(xml_op, "queue_time")); } print_as("): %s\n", op_status2text(status)); ); } #if CURSES_ENABLED if(as_console) { refresh(); } #endif data_set.input = NULL; cleanup_calculations(&data_set); return 0; } int print_html_status(xmlNode *cib, const char *filename, gboolean web_cgi) { FILE *stream; node_t *dc = NULL; static int updates = 0; pe_working_set_t data_set; char *filename_tmp = NULL; if (web_cgi) { stream=stdout; fprintf(stream, "Content-type: text/html\n\n"); } else { filename_tmp = crm_concat(filename, "tmp", '.'); stream = fopen(filename_tmp, "w"); if(stream == NULL) { cl_perror("Cannot open %s for writing", filename_tmp); crm_free(filename_tmp); return -1; } } updates++; set_working_set_defaults(&data_set); data_set.input = cib; cluster_status(&data_set); dc = data_set.dc_node; fprintf(stream, ""); fprintf(stream, ""); fprintf(stream, "Cluster status"); /* content="%d;url=http://webdesign.about.com" */ fprintf(stream, "", interval); fprintf(stream, ""); /*** SUMMARY ***/ fprintf(stream, "

Cluster summary

"); { char *now_str = NULL; time_t now = time(NULL); now_str = ctime(&now); now_str[24] = EOS; /* replace the newline */ fprintf(stream, "Last updated: %s
\n", now_str); } if(dc == NULL) { fprintf(stream, "Current DC: NONE
"); } else { fprintf(stream, "Current DC: %s (%s)
", dc->details->uname, dc->details->id); } fprintf(stream, "%d Nodes configured.
", g_list_length(data_set.nodes)); fprintf(stream, "%d Resources configured.
", g_list_length(data_set.resources)); /*** CONFIG ***/ fprintf(stream, "

Config Options

\n"); fprintf(stream, "\n"); fprintf(stream, "\n", data_set.default_resource_stickiness); fprintf(stream, "\n", data_set.stonith_enabled?"enabled":"disabled"); fprintf(stream, "\n", data_set.symmetric_cluster?"":"a-"); fprintf(stream, "\n
Default resource stickiness:%d
STONITH of failed nodes:%s
Cluster is:%ssymmetric
No Quorum Policy:"); switch (data_set.no_quorum_policy) { case no_quorum_freeze: fprintf(stream, "Freeze resources"); break; case no_quorum_stop: fprintf(stream, "Stop ALL resources"); break; case no_quorum_ignore: fprintf(stream, "Ignore"); break; } fprintf(stream, "\n
\n"); /*** NODE LIST ***/ fprintf(stream, "

Node List

\n"); fprintf(stream, "
    \n"); slist_iter(node, node_t, data_set.nodes, lpc2, fprintf(stream, "
  • "); fprintf(stream, "Node: %s (%s): %s", node->details->uname, node->details->id, node->details->online?"online\n":"OFFLINE\n"); if(group_by_node) { fprintf(stream, "
      \n"); slist_iter(rsc, resource_t, node->details->running_rsc, lpc2, fprintf(stream, "
    • "); rsc->fns->print(rsc, NULL, pe_print_html, stream); fprintf(stream, "
      • \n"); ); fprintf(stream, "
    \n"); } fprintf(stream, "
    • \n"); ); fprintf(stream, "
\n"); if(group_by_node && inactive_resources) { fprintf(stream, "

(Partially) Inactive Resources

\n"); } else if(group_by_node == FALSE) { fprintf(stream, "

Resource List

\n"); } if(group_by_node == FALSE || inactive_resources) { slist_iter(rsc, resource_t, data_set.resources, lpc2, if(group_by_node && rsc->fns->active(rsc, TRUE)) { continue; } rsc->fns->print(rsc, NULL, pe_print_html, stream); ); } data_set.input = NULL; cleanup_calculations(&data_set); fprintf(stream, ""); fflush(stream); fclose(stream); if (!web_cgi) { if(rename(filename_tmp, filename) != 0) { cl_perror("Unable to rename %s->%s", filename_tmp, filename); } crm_free(filename_tmp); } return 0; } void blank_screen(void) { #if CURSES_ENABLED int lpc = 0; for(lpc = 0; lpc < LINES; lpc++) { move(lpc, 0); clrtoeol(); } move(0, 0); refresh(); #endif } void usage(const char *cmd, int exit_status) { FILE *stream; stream = exit_status ? stderr : stdout; fprintf(stream, "usage: %s [-%s]\n", cmd, OPTARGS); fprintf(stream, "\t--%s (-%c) \t: This text\n", "help", '?'); fprintf(stream, "\t--%s (-%c) \t: Increase the debug output\n", "verbose", 'V'); fprintf(stream, "\t--%s (-%c) \t: Update frequency\n", "interval", 'i'); fprintf(stream, "\t--%s (-%c) \t: Group resources by node\n", "group-by-node", 'n'); fprintf(stream, "\t--%s (-%c) \t: Display inactive resources\n", "inactive", 'r'); fprintf(stream, "\t--%s (-%c) \t: Display resource fail counts\n", "failcount", 'f'); fprintf(stream, "\t--%s (-%c) \t: Display resource operation history\n", "operations", 'o'); fprintf(stream, "\t--%s (-%c) \t: Display cluster status on the console\n", "as-console", 'c'); fprintf(stream, "\t--%s (-%c) \t: Display the cluster status once as " "a simple one line output (suitable for nagios)\n", "simple-status", 's'); fprintf(stream, "\t--%s (-%c) \t: Display the cluster status once on " "the console and exit (doesnt use ncurses)\n", "one-shot", '1'); fprintf(stream, "\t--%s (-%c) \t: Write cluster status to the named file\n", "as-html", 'h'); fprintf(stream, "\t--%s (-%c) \t: Web mode with output suitable for cgi\n", "web-cgi", 'w'); fprintf(stream, "\t--%s (-%c) \t: Run in the background as a daemon\n", "daemonize", 'd'); fprintf(stream, "\t--%s (-%c) \t: Daemon pid file location\n", "pid-file", 'p'); fflush(stream); exit(exit_status); } void make_daemon(gboolean daemonize, const char *pidfile) { long pid; const char *devnull = "/dev/null"; if (daemonize == FALSE){ return; } pid = fork(); if (pid < 0) { fprintf(stderr, "%s: could not start daemon\n", crm_system_name); perror("fork"); exit(LSB_EXIT_GENERIC); } else if (pid > 0) { exit(LSB_EXIT_OK); } if (cl_lock_pidfile(pidfile) < 0 ){ pid = cl_read_pidfile(pidfile); fprintf(stderr, "%s: already running [pid %ld].\n", crm_system_name, pid); exit(LSB_EXIT_OK); } umask(022); close(0); close(1); close(2); (void)open(devnull, O_RDONLY); /* Stdin: fd 0 */ (void)open(devnull, O_WRONLY); /* Stdout: fd 1 */ (void)open(devnull, O_WRONLY); /* Stderr: fd 2 */ } diff --git a/crmd/callbacks.c b/crmd/callbacks.c index 47c3625ca7..da77d8c40d 100644 --- a/crmd/callbacks.c +++ b/crmd/callbacks.c @@ -1,640 +1,644 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include void crmd_ha_connection_destroy(gpointer user_data); void crmd_ha_msg_filter(xmlNode *msg); /* From join_dc... */ extern gboolean check_join_state( enum crmd_fsa_state cur_state, const char *source); #define trigger_fsa(source) crm_debug_3("Triggering FSA: %s", __FUNCTION__); \ G_main_set_trigger(source); #if SUPPORT_HEARTBEAT gboolean crmd_ha_msg_dispatch(ll_cluster_t *cluster_conn, gpointer user_data) { IPC_Channel *channel = NULL; gboolean stay_connected = TRUE; crm_debug_3("Invoked"); if(cluster_conn != NULL) { channel = cluster_conn->llc_ops->ipcchan(cluster_conn); } CRM_CHECK(cluster_conn != NULL, ;); CRM_CHECK(channel != NULL, ;); if(channel != NULL && IPC_ISRCONN(channel)) { if(cluster_conn->llc_ops->msgready(cluster_conn) == 0) { crm_debug_2("no message ready yet"); } /* invoke the callbacks but dont block */ cluster_conn->llc_ops->rcvmsg(cluster_conn, 0); } if (channel == NULL || channel->ch_status != IPC_CONNECT) { if(is_set(fsa_input_register, R_HA_DISCONNECTED) == FALSE) { crm_crit("Lost connection to heartbeat service."); } else { crm_info("Lost connection to heartbeat service."); } trigger_fsa(fsa_source); stay_connected = FALSE; } return stay_connected; } #endif void crmd_ha_connection_destroy(gpointer user_data) { crm_debug_3("Invoked"); if(is_set(fsa_input_register, R_HA_DISCONNECTED)) { /* we signed out, so this is expected */ crm_info("Heartbeat disconnection complete"); return; } crm_crit("Lost connection to heartbeat service!"); register_fsa_input(C_HA_DISCONNECT, I_ERROR, NULL); trigger_fsa(fsa_source); } void crmd_ha_msg_filter(xmlNode *msg) { ha_msg_input_t new_input; const char *from = crm_element_value(msg, F_ORIG); const char *seq = crm_element_value(msg, F_SEQ); const char *op = crm_element_value(msg, F_CRM_TASK); const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO); const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM); new_input.msg = NULL; if(safe_str_eq(sys_to, CRM_SYSTEM_DC) && AM_I_DC == FALSE) { crm_debug_2("Ignoring message for the DC [F_SEQ=%s]", seq); return; } else if(safe_str_eq(sys_from, CRM_SYSTEM_DC)) { if(AM_I_DC && safe_str_neq(from, fsa_our_uname)) { crm_err("Another DC detected: %s (op=%s)", from, op); /* make sure the election happens NOW */ if(fsa_state != S_ELECTION) { new_input.msg = msg; register_fsa_error_adv( C_FSA_INTERNAL, I_ELECTION, NULL, &new_input, __FUNCTION__); } } else { crm_debug_2("Processing DC message from %s [F_SEQ=%s]", from, seq); } } if(new_input.msg == NULL) { crm_log_xml(LOG_MSG, "HA[inbound]", msg); route_message(C_HA_MESSAGE, msg); } trigger_fsa(fsa_source); } #if SUPPORT_HEARTBEAT void crmd_ha_msg_callback(HA_Message *hamsg, void* private_data) { int level = LOG_DEBUG; oc_node_t *from_node = NULL; xmlNode *msg = convert_ha_message(NULL, hamsg, __FUNCTION__); const char *from = crm_element_value(msg, F_ORIG); const char *op = crm_element_value(msg, F_CRM_TASK); const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM); CRM_CHECK(from != NULL, crm_log_xml_err(msg, "anon"); goto bail); crm_debug_2("HA[inbound]: %s from %s", op, from); if(crm_peer_cache == NULL || crm_active_members() == 0) { crm_debug("Ignoring HA messages until we are" " connected to the CCM (%s op from %s)", op, from); crm_log_xml(LOG_MSG, "HA[inbound]: Ignore (No CCM)", msg); goto bail; } from_node = g_hash_table_lookup(crm_peer_cache, from); if(from_node == NULL) { if(safe_str_eq(op, CRM_OP_VOTE)) { level = LOG_WARNING; } else if(AM_I_DC && safe_str_eq(op, CRM_OP_JOIN_ANNOUNCE)) { level = LOG_WARNING; } else if(safe_str_eq(sys_from, CRM_SYSTEM_DC)) { level = LOG_WARNING; } do_crm_log(level, "Ignoring HA message (op=%s) from %s: not in our" " membership list (size=%d)", op, from, crm_active_members()); crm_log_xml(LOG_MSG, "HA[inbound]: CCM Discard", msg); } else { crmd_ha_msg_filter(msg); return; } bail: free_xml(msg); return; } #endif /* * Apparently returning TRUE means "stay connected, keep doing stuff". * Returning FALSE means "we're all done, close the connection" */ gboolean crmd_ipc_msg_callback(IPC_Channel *client, gpointer user_data) { int lpc = 0; xmlNode *msg = NULL; crmd_client_t *curr_client = (crmd_client_t*)user_data; gboolean stay_connected = TRUE; crm_debug_2("Invoked: %s", curr_client->table_key); while(IPC_ISRCONN(client)) { if(client->ops->is_message_pending(client) == 0) { break; } msg = xmlfromIPC(client, 0); if (msg == NULL) { break; } lpc++; crm_debug_2("Processing msg from %s", curr_client->table_key); crm_log_xml(LOG_DEBUG_2, "CRMd[inbound]", msg); if(crmd_authorize_message(msg, curr_client)) { route_message(C_IPC_MESSAGE, msg); } free_xml(msg); msg = NULL; if(client->ch_status != IPC_CONNECT) { break; } } crm_debug_2("Processed %d messages", lpc); if (client->ch_status != IPC_CONNECT) { stay_connected = FALSE; process_client_disconnect(curr_client); } trigger_fsa(fsa_source); return stay_connected; } extern GCHSource *lrm_source; gboolean lrm_dispatch(IPC_Channel *src_not_used, gpointer user_data) { /* ?? src == lrm_channel ?? */ ll_lrm_t *lrm = (ll_lrm_t*)user_data; IPC_Channel *lrm_channel = lrm->lrm_ops->ipcchan(lrm); crm_debug_3("Invoked"); lrm->lrm_ops->rcvmsg(lrm, FALSE); if(lrm_channel->ch_status != IPC_CONNECT) { if(is_set(fsa_input_register, R_LRM_CONNECTED)) { crm_crit("LRM Connection failed"); register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL); clear_bit_inplace(fsa_input_register, R_LRM_CONNECTED); } else { crm_info("LRM Connection disconnected"); } lrm_source = NULL; return FALSE; } return TRUE; } extern gboolean process_lrm_event(lrm_op_t *op); void lrm_op_callback(lrm_op_t* op) { CRM_CHECK(op != NULL, return); process_lrm_event(op); } void crmd_ha_status_callback(const char *node, const char *status, void *private) { xmlNode *update = NULL; crm_node_t *member = NULL; crm_notice("Status update: Node %s now has status [%s] (DC=%s)", node, status, AM_I_DC?"true":"false"); member = g_hash_table_lookup(crm_peer_cache, node); if(member == NULL) { /* Make sure it is created so crm_update_peer_proc() succeeds */ const char *uuid = get_uuid(node); member = crm_update_peer(0, 0, -1, 0, uuid, node, NULL, NULL); } if(safe_str_eq(status, PINGSTATUS)) { return; } if(safe_str_eq(status, DEADSTATUS)) { /* this node is toast */ crm_update_peer_proc(node, crm_proc_ais, OFFLINESTATUS); if(AM_I_DC) { update = create_node_state( node, DEADSTATUS, XML_BOOLEAN_NO, OFFLINESTATUS, CRMD_JOINSTATE_DOWN, NULL, TRUE, __FUNCTION__); } } else { crm_update_peer_proc(node, crm_proc_ais, ONLINESTATUS); if(AM_I_DC) { update = create_node_state( - node, ACTIVESTATUS, NULL, NULL, NULL, NULL, - FALSE, __FUNCTION__); + node, ACTIVESTATUS, NULL, NULL, + CRMD_JOINSTATE_PENDING, NULL, FALSE, __FUNCTION__); } } trigger_fsa(fsa_source); if(update != NULL) { fsa_cib_anon_update( XML_CIB_TAG_STATUS, update, cib_scope_local|cib_quorum_override|cib_can_create); free_xml(update); } } void crmd_client_status_callback(const char * node, const char * client, const char * status, void * private) { const char *join = NULL; crm_node_t *member = NULL; xmlNode *update = NULL; gboolean clear_shutdown = FALSE; crm_debug_3("Invoked"); if(safe_str_neq(client, CRM_SYSTEM_CRMD)) { return; } if(safe_str_eq(status, JOINSTATUS)){ - status = ONLINESTATUS; clear_shutdown = TRUE; + status = ONLINESTATUS; + join = CRMD_JOINSTATE_PENDING; } else if(safe_str_eq(status, LEAVESTATUS)){ status = OFFLINESTATUS; join = CRMD_JOINSTATE_DOWN; /* clear_shutdown = TRUE; */ } set_bit_inplace(fsa_input_register, R_PEER_DATA); crm_notice("Status update: Client %s/%s now has status [%s] (DC=%s)", node, client, status, AM_I_DC?"true":"false"); if(safe_str_eq(status, ONLINESTATUS)) { /* remove the cached value in case it changed */ crm_debug_2("Uncaching UUID for %s", node); unget_uuid(node); } member = g_hash_table_lookup(crm_peer_cache, node); if(member == NULL) { /* Make sure it is created so crm_update_peer_proc() succeeds */ const char *uuid = get_uuid(node); member = crm_update_peer(0, 0, -1, 0, uuid, node, NULL, NULL); } crm_update_peer_proc(node, crm_proc_crmd, status); if(is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) { return; } else if(fsa_state == S_STOPPING) { return; } if(safe_str_eq(node, fsa_our_dc) && safe_str_eq(status, OFFLINESTATUS)){ /* did our DC leave us */ crm_info("Got client status callback - our DC is dead"); register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL); } else if(AM_I_DC == FALSE) { crm_info("Not the DC"); } else { crm_debug_3("Got client status callback"); if(fsa_cib_conn != NULL && safe_str_eq(status, ONLINESTATUS)) { erase_status_tag(fsa_our_uname, XML_CIB_TAG_LRM); erase_status_tag(fsa_our_uname, XML_TAG_TRANSIENT_NODEATTRS); } update = create_node_state( node, NULL, NULL, status, join, NULL, clear_shutdown, __FUNCTION__); fsa_cib_anon_update( XML_CIB_TAG_STATUS, update, cib_scope_local|cib_quorum_override|cib_can_create); free_xml(update); if(safe_str_eq(status, OFFLINESTATUS)) { erase_node_from_join(node); check_join_state(fsa_state, __FUNCTION__); } } trigger_fsa(fsa_source); } void crmd_ipc_connection_destroy(gpointer user_data) { GCHSource *source = NULL; crmd_client_t *client = user_data; /* Calling this function on an _active_ connection results in: * crmd_ipc_connection_destroy (callbacks.c:431) * -> G_main_del_IPC_Channel (GSource.c:478) * -> g_source_unref * -> G_CH_destroy_int (GSource.c:647) * -> crmd_ipc_connection_destroy (callbacks.c:437)\ * * A better alternative is to call G_main_del_IPC_Channel() directly */ if(client == NULL) { crm_debug_4("No client to delete"); return; } crm_debug_2("Disconnecting client %s (%p)", client->table_key, client); source = client->client_source; client->client_source = NULL; if(source != NULL) { crm_debug_3("Deleting %s (%p) from mainloop", client->table_key, source); G_main_del_IPC_Channel(source); } crm_free(client->table_key); crm_free(client->sub_sys); crm_free(client->uuid); crm_free(client); return; } gboolean crmd_client_connect(IPC_Channel *client_channel, gpointer user_data) { crm_debug_3("Invoked"); if (client_channel == NULL) { crm_err("Channel was NULL"); } else if (client_channel->ch_status == IPC_DISCONNECT) { crm_err("Channel was disconnected"); } else { crmd_client_t *blank_client = NULL; crm_debug_3("Channel connected"); crm_malloc0(blank_client, sizeof(crmd_client_t)); CRM_ASSERT(blank_client != NULL); crm_debug_2("Created client: %p", blank_client); client_channel->ops->set_recv_qlen(client_channel, 1024); client_channel->ops->set_send_qlen(client_channel, 1024); blank_client->client_channel = client_channel; blank_client->sub_sys = NULL; blank_client->uuid = NULL; blank_client->table_key = NULL; blank_client->client_source = G_main_add_IPC_Channel( G_PRIORITY_LOW, client_channel, FALSE, crmd_ipc_msg_callback, blank_client, crmd_ipc_connection_destroy); } return TRUE; } #if SUPPORT_HEARTBEAT static gboolean fsa_have_quorum = FALSE; gboolean ccm_dispatch(int fd, gpointer user_data) { int rc = 0; oc_ev_t *ccm_token = (oc_ev_t*)user_data; gboolean was_error = FALSE; crm_debug_3("Invoked"); rc = oc_ev_handle_event(ccm_token); if(rc != 0) { if(is_set(fsa_input_register, R_CCM_DISCONNECTED) == FALSE) { /* we signed out, so this is expected */ register_fsa_input(C_CCM_CALLBACK, I_ERROR, NULL); crm_err("CCM connection appears to have failed: rc=%d.", rc); } was_error = TRUE; } trigger_fsa(fsa_source); return !was_error; } void crmd_ccm_msg_callback( oc_ed_t event, void *cookie, size_t size, const void *data) { gboolean update_cache = FALSE; const oc_ev_membership_t *membership = data; gboolean update_quorum = FALSE; crm_debug_3("Invoked"); CRM_ASSERT(data != NULL); crm_info("Quorum %s after event=%s (id=%d)", ccm_have_quorum(event)?"(re)attained":"lost", ccm_event_name(event), membership->m_instance); if(crm_peer_seq > membership->m_instance) { crm_err("Membership instance ID went backwards! %llu->%d", crm_peer_seq, membership->m_instance); CRM_ASSERT(crm_peer_seq <= membership->m_instance); return; } /* * OC_EV_MS_NEW_MEMBERSHIP: membership with quorum * OC_EV_MS_MS_INVALID: membership without quorum * OC_EV_MS_NOT_PRIMARY: previous membership no longer valid * OC_EV_MS_PRIMARY_RESTORED: previous membership restored * OC_EV_MS_EVICTED: the client is evicted from ccm. */ switch(event) { case OC_EV_MS_NEW_MEMBERSHIP: case OC_EV_MS_INVALID: update_cache = TRUE; update_quorum = TRUE; break; case OC_EV_MS_NOT_PRIMARY: break; case OC_EV_MS_PRIMARY_RESTORED: update_cache = TRUE; crm_peer_seq = membership->m_instance; break; case OC_EV_MS_EVICTED: update_quorum = TRUE; register_fsa_input(C_FSA_INTERNAL, I_STOP, NULL); crm_err("Shutting down after CCM event: %s", ccm_event_name(event)); break; default: crm_err("Unknown CCM event: %d", event); } if(update_quorum) { crm_have_quorum = ccm_have_quorum(event); crm_update_quorum(crm_have_quorum); if(crm_have_quorum == FALSE) { /* did we just loose quorum? */ if(fsa_have_quorum) { crm_info("Quorum lost: %s", ccm_event_name(event)); } } } if(update_cache) { crm_debug_2("Updating cache after event %s", ccm_event_name(event)); do_ccm_update_cache(C_CCM_CALLBACK, fsa_state, event, data, NULL); } else if(event != OC_EV_MS_NOT_PRIMARY) { crm_peer_seq = membership->m_instance; register_fsa_action(A_TE_CANCEL); } oc_ev_callback_done(cookie); return; } #endif void crmd_cib_connection_destroy(gpointer user_data) { + CRM_CHECK(user_data == fsa_cib_conn, ;); + crm_debug_3("Invoked"); trigger_fsa(fsa_source); - + fsa_cib_conn->state = cib_disconnected; + if(is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) { crm_info("Connection to the CIB terminated..."); return; } /* eventually this will trigger a reconnect, not a shutdown */ crm_err("Connection to the CIB terminated..."); register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL); clear_bit_inplace(fsa_input_register, R_CIB_CONNECTED); return; } longclock_t fsa_start = 0; longclock_t fsa_stop = 0; longclock_t fsa_diff = 0; gboolean crm_fsa_trigger(gpointer user_data) { unsigned int fsa_diff_ms = 0; if(fsa_diff_max_ms > 0) { fsa_start = time_longclock(); } crm_debug_2("Invoked (queue len: %d)", g_list_length(fsa_message_queue)); s_crmd_fsa(C_FSA_INTERNAL); crm_debug_2("Exited (queue len: %d)", g_list_length(fsa_message_queue)); if(fsa_diff_max_ms > 0) { fsa_stop = time_longclock(); fsa_diff = sub_longclock(fsa_stop, fsa_start); fsa_diff_ms = longclockto_ms(fsa_diff); if(fsa_diff_ms > fsa_diff_max_ms) { crm_err("FSA took %dms to complete", fsa_diff_ms); } else if(fsa_diff_ms > fsa_diff_warn_ms) { crm_warn("FSA took %dms to complete", fsa_diff_ms); } } return TRUE; } diff --git a/crmd/join_dc.c b/crmd/join_dc.c index 1f959aa5c9..1845653290 100644 --- a/crmd/join_dc.c +++ b/crmd/join_dc.c @@ -1,679 +1,679 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include GHashTable *welcomed_nodes = NULL; GHashTable *integrated_nodes = NULL; GHashTable *finalized_nodes = NULL; GHashTable *confirmed_nodes = NULL; char *max_epoch = NULL; char *max_generation_from = NULL; xmlNode *max_generation_xml = NULL; void initialize_join(gboolean before); gboolean finalize_join_for(gpointer key, gpointer value, gpointer user_data); void finalize_sync_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data); gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source); void finalize_join(const char *caller); static int current_join_id = 0; unsigned long long saved_ccm_membership_id = 0; static void update_attrd(void) { static IPC_Channel *attrd = NULL; if(attrd == NULL) { crm_info("Connecting to attrd..."); attrd = init_client_ipc_comms_nodispatch(T_ATTRD); } if(attrd != NULL) { xmlNode *update = create_xml_node(NULL, __FUNCTION__); crm_xml_add(update, F_TYPE, T_ATTRD); crm_xml_add(update, F_ORIG, "crmd"); crm_xml_add(update, "task", "refresh"); if(send_ipc_message(attrd, update) == FALSE) { crm_err("attrd refresh failed"); attrd = NULL; } else { crm_debug("sent attrd refresh"); } free_xml(update); } else { crm_info("Couldn't connect to attrd this time"); } } void initialize_join(gboolean before) { /* clear out/reset a bunch of stuff */ crm_debug("join-%d: Initializing join data (flag=%s)", current_join_id, before?"true":"false"); g_hash_table_destroy(welcomed_nodes); g_hash_table_destroy(integrated_nodes); g_hash_table_destroy(finalized_nodes); g_hash_table_destroy(confirmed_nodes); if(before) { if(max_generation_from != NULL) { crm_free(max_generation_from); max_generation_from = NULL; } if(max_generation_xml != NULL) { free_xml(max_generation_xml); max_generation_xml = NULL; } clear_bit_inplace(fsa_input_register, R_HAVE_CIB); clear_bit_inplace(fsa_input_register, R_CIB_ASKED); } welcomed_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); integrated_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); finalized_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); confirmed_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); } void erase_node_from_join(const char *uname) { gboolean w = FALSE, i = FALSE, f = FALSE, c = FALSE; if(uname == NULL) { return; } if(welcomed_nodes != NULL) { w = g_hash_table_remove(welcomed_nodes, uname); } if(integrated_nodes != NULL) { i = g_hash_table_remove(integrated_nodes, uname); } if(finalized_nodes != NULL) { f = g_hash_table_remove(finalized_nodes, uname); } if(confirmed_nodes != NULL) { c = g_hash_table_remove(confirmed_nodes, uname); } crm_info("Removed dead node %s from join calculations:" " welcomed=%d itegrated=%d finalized=%d confirmed=%d", uname, w, i, f, c); } static void join_make_offer(gpointer key, gpointer value, gpointer user_data) { const char *join_to = NULL; const crm_node_t *member = value; CRM_ASSERT(member != NULL); if(crm_is_member_active(member) == FALSE) { return; } join_to = member->uname; if(join_to == NULL) { crm_err("No recipient for welcome message"); return; } erase_node_from_join(join_to); if(saved_ccm_membership_id != crm_peer_seq) { saved_ccm_membership_id = crm_peer_seq; crm_info("Making join offers based on membership %llu", crm_peer_seq); } if(member->processes & crm_proc_crmd) { xmlNode *offer = create_request( CRM_OP_JOIN_OFFER, NULL, join_to, CRM_SYSTEM_CRMD, CRM_SYSTEM_DC, NULL); char *join_offered = crm_itoa(current_join_id); crm_xml_add_int(offer, F_CRM_JOIN_ID, current_join_id); /* send the welcome */ crm_debug("join-%d: Sending offer to %s", current_join_id, join_to); send_msg_via_ha(offer); free_xml(offer); g_hash_table_insert( welcomed_nodes, crm_strdup(join_to), join_offered); } else { crm_info("Peer process on %s is not active (yet?): %.8lx %d", join_to, (long)member->processes, g_hash_table_size(crm_peer_cache)); } } /* A_DC_JOIN_OFFER_ALL */ void do_dc_join_offer_all(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { /* reset everyones status back to down or in_ccm in the CIB * * any nodes that are active in the CIB but not in the CCM list * will be seen as offline by the PE anyway */ current_join_id++; initialize_join(TRUE); /* do_update_cib_nodes(TRUE, __FUNCTION__); */ update_dc(NULL, FALSE); if(cause == C_HA_MESSAGE && current_input == I_NODE_JOIN) { crm_info("A new node joined the cluster"); } g_hash_table_foreach(crm_peer_cache, join_make_offer, NULL); /* dont waste time by invoking the PE yet; */ crm_info("join-%d: Waiting on %d outstanding join acks", current_join_id, g_hash_table_size(welcomed_nodes)); } /* A_DC_JOIN_OFFER_ONE */ void do_dc_join_offer_one(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { crm_node_t *member; ha_msg_input_t *welcome = fsa_typed_data(fsa_dt_ha_msg); const char *join_to = NULL; if(welcome == NULL) { crm_err("Attempt to send welcome message " "without a message to reply to!"); return; } join_to = crm_element_value(welcome->msg, F_CRM_HOST_FROM); if(join_to != NULL && (cur_state == S_INTEGRATION || cur_state == S_FINALIZE_JOIN)) { /* note: it _is_ possible that a node will have been * sick or starting up when the original offer was made. * however, it will either re-announce itself in due course * _or_ we can re-store the original offer on the client. */ crm_debug("Re-offering membership to %s...", join_to); } crm_info("join-%d: Processing annouce request from %s in state %s", current_join_id, join_to, fsa_state2string(cur_state)); /* always offer to the DC (ourselves) * this ensures the correct value for max_generation_from */ member = g_hash_table_lookup(crm_peer_cache, fsa_our_uname); join_make_offer(NULL, member, NULL); member = g_hash_table_lookup(crm_peer_cache, join_to); join_make_offer(NULL, member, NULL); /* this was a genuine join request, cancel any existing * transition and invoke the PE */ start_transition(fsa_state); /* dont waste time by invoking the pe yet; */ crm_debug("Waiting on %d outstanding join acks for join-%d", g_hash_table_size(welcomed_nodes), current_join_id); } /* A_DC_JOIN_PROCESS_REQ */ void do_dc_join_filter_offer(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { xmlNode *generation = NULL; int cmp = 0; int join_id = -1; gboolean ack_nack_bool = TRUE; const char *ack_nack = CRMD_JOINSTATE_MEMBER; ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg); const char *join_from = crm_element_value(join_ack->msg,F_CRM_HOST_FROM); const char *ref = crm_element_value(join_ack->msg,XML_ATTR_REFERENCE); gpointer join_node = g_hash_table_lookup( crm_peer_cache, join_from); crm_debug("Processing req from %s", join_from); generation = join_ack->xml; crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id); if(max_generation_xml != NULL && generation != NULL) { cmp = cib_compare_generation(max_generation_xml, generation); } if(join_node == NULL) { crm_err("Node %s is not a member", join_from); ack_nack_bool = FALSE; } else if(generation == NULL) { crm_err("Generation was NULL"); ack_nack_bool = FALSE; } else if(join_id != current_join_id) { crm_debug("Invalid response from %s: join-%d vs. join-%d", join_from, join_id, current_join_id); check_join_state(cur_state, __FUNCTION__); return; } else if(max_generation_xml == NULL) { max_generation_xml = copy_xml(generation); max_generation_from = crm_strdup(join_from); } else if(cmp < 0 || (cmp == 0 && safe_str_eq(join_from, fsa_our_uname))) { crm_debug("%s has a better generation number than" " the current max %s", join_from, max_generation_from); if(max_generation_xml) { crm_log_xml_debug(max_generation_xml, "Max generation"); } crm_log_xml_debug(generation, "Their generation"); crm_free(max_generation_from); free_xml(max_generation_xml); max_generation_from = crm_strdup(join_from); max_generation_xml = copy_xml(join_ack->xml); } if(ack_nack_bool == FALSE) { /* NACK this client */ - ack_nack = CRMD_JOINSTATE_DOWN; + ack_nack = CRMD_JOINSTATE_NACK; crm_err("join-%d: NACK'ing node %s (ref %s)", join_id, join_from, ref); } else { crm_debug("join-%d: Welcoming node %s (ref %s)", join_id, join_from, ref); } /* add them to our list of CRMD_STATE_ACTIVE nodes */ g_hash_table_insert( integrated_nodes, crm_strdup(join_from), crm_strdup(ack_nack)); crm_debug("%u nodes have been integrated into join-%d", g_hash_table_size(integrated_nodes), join_id); g_hash_table_remove(welcomed_nodes, join_from); if(check_join_state(cur_state, __FUNCTION__) == FALSE) { /* dont waste time by invoking the PE yet; */ crm_debug("join-%d: Still waiting on %d outstanding offers", join_id, g_hash_table_size(welcomed_nodes)); } } /* A_DC_JOIN_FINALIZE */ void do_dc_join_finalize(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { enum cib_errors rc = cib_ok; /* This we can do straight away and avoid clients timing us out * while we compute the latest CIB */ crm_debug("Finializing join-%d for %d clients", current_join_id, g_hash_table_size(integrated_nodes)); if(g_hash_table_size(integrated_nodes) == 0) { return; } clear_bit_inplace(fsa_input_register, R_HAVE_CIB); if(max_generation_from == NULL || safe_str_eq(max_generation_from, fsa_our_uname)){ set_bit_inplace(fsa_input_register, R_HAVE_CIB); } if(is_set(fsa_input_register, R_IN_TRANSITION)) { crm_warn("join-%d: We are still in a transition." " Delaying until the TE completes.", current_join_id); crmd_fsa_stall(NULL); return; } if(is_set(fsa_input_register, R_HAVE_CIB) == FALSE) { /* ask for the agreed best CIB */ crm_info("join-%d: Asking %s for its copy of the CIB", current_join_id, crm_str(max_generation_from)); crm_log_xml_debug(max_generation_xml, "Requesting version"); set_bit_inplace(fsa_input_register, R_CIB_ASKED); rc = fsa_cib_conn->cmds->sync_from( fsa_cib_conn, max_generation_from, NULL, cib_quorum_override); fsa_cib_conn->cmds->register_callback( fsa_cib_conn, rc, 60, FALSE, crm_strdup(max_generation_from), "finalize_sync_callback", finalize_sync_callback); return; } else { /* Send _our_ CIB out to everyone */ fsa_cib_conn->cmds->sync_from( fsa_cib_conn, fsa_our_uname, NULL,cib_quorum_override); update_attrd(); } finalize_join(__FUNCTION__); } void finalize_sync_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { CRM_DEV_ASSERT(cib_not_master != rc); clear_bit_inplace(fsa_input_register, R_CIB_ASKED); if(rc != cib_ok) { do_crm_log((rc==cib_old_data?LOG_WARNING:LOG_ERR), "Sync from %s resulted in an error: %s", (char*)user_data, cib_error2string(rc)); /* restart the whole join process */ register_fsa_error_adv( C_FSA_INTERNAL, I_ELECTION_DC,NULL,NULL,__FUNCTION__); } else if(AM_I_DC && fsa_state == S_FINALIZE_JOIN) { finalize_join(__FUNCTION__); update_attrd(); } else { crm_debug("No longer the DC in S_FINALIZE_JOIN: %s/%s", AM_I_DC?"DC":"CRMd", fsa_state2string(fsa_state)); } crm_free(user_data); } void finalize_join(const char *caller) { xmlNode *cib = createEmptyCib(); set_bit_inplace(fsa_input_register, R_HAVE_CIB); clear_bit_inplace(fsa_input_register, R_CIB_ASKED); set_uuid(cib, XML_ATTR_DC_UUID, fsa_our_uname); crm_debug_3("Update %s in the CIB to our uuid: %s", XML_ATTR_DC_UUID, crm_element_value(cib, XML_ATTR_DC_UUID)); fsa_cib_anon_update(NULL, cib, cib_quorum_override); free_xml(cib); crm_debug_3("Syncing to %d clients", g_hash_table_size(finalized_nodes)); /* make sure dc_uuid is re-set to us */ if(check_join_state(fsa_state, caller) == FALSE) { crm_debug("Notifying %d clients of join-%d results", g_hash_table_size(integrated_nodes), current_join_id); g_hash_table_foreach_remove( integrated_nodes, finalize_join_for, NULL); } } static void join_update_complete_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { fsa_data_t *msg_data = NULL; if(rc == cib_ok) { crm_debug("Join update %d complete", call_id); check_join_state(fsa_state, __FUNCTION__); } else { crm_err("Join update %d failed", call_id); crm_log_xml(LOG_DEBUG, "failed", msg); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } /* A_DC_JOIN_PROCESS_ACK */ void do_dc_join_ack(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { int join_id = -1; int call_id = 0; ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg); const char *join_id_s = NULL; const char *join_state = NULL; const char *op = crm_element_value(join_ack->msg, F_CRM_TASK); const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM); if(safe_str_neq(op, CRM_OP_JOIN_CONFIRM)) { crm_debug("Ignoring op=%s message from %s", op, join_from); return; } crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id); join_id_s = crm_element_value(join_ack->msg, F_CRM_JOIN_ID); /* now update them to "member" */ crm_debug_2("Processing ack from %s", join_from); join_state = (const char *) g_hash_table_lookup(finalized_nodes, join_from); if(join_state == NULL) { crm_err("Join not in progress: ignoring join-%d from %s", join_id, join_from); return; } else if(safe_str_neq(join_state, CRMD_JOINSTATE_MEMBER)) { crm_err("Node %s wasnt invited to join the cluster",join_from); g_hash_table_remove(finalized_nodes, join_from); return; } else if(join_id != current_join_id) { crm_err("Invalid response from %s: join-%d vs. join-%d", join_from, join_id, current_join_id); g_hash_table_remove(finalized_nodes, join_from); return; } g_hash_table_remove(finalized_nodes, join_from); if(g_hash_table_lookup(confirmed_nodes, join_from) != NULL) { crm_err("join-%d: hash already contains confirmation from %s", join_id, join_from); } g_hash_table_insert( confirmed_nodes, crm_strdup(join_from), crm_strdup(join_id_s)); crm_info("join-%d: Updating node state to %s for %s", join_id, CRMD_JOINSTATE_MEMBER, join_from); /* update CIB with the current LRM status from the node * We dont need to notify the TE of these updates, a transition will * be started in due time */ erase_status_tag(join_from, XML_CIB_TAG_LRM); fsa_cib_update(XML_CIB_TAG_STATUS, join_ack->xml, cib_scope_local|cib_quorum_override|cib_can_create, call_id); add_cib_op_callback( fsa_cib_conn, call_id, FALSE, NULL, join_update_complete_callback); crm_debug("join-%d: Registered callback for LRM update %d", join_id, call_id); } gboolean finalize_join_for(gpointer key, gpointer value, gpointer user_data) { const char *join_to = NULL; const char *join_state = NULL; xmlNode *acknak = NULL; if(key == NULL || value == NULL) { return TRUE; } join_to = (const char *)key; join_state = (const char *)value; /* make sure the node exists in the config section */ create_node_entry(join_to, join_to, NORMALNODE); /* send the ack/nack to the node */ acknak = create_request( CRM_OP_JOIN_ACKNAK, NULL, join_to, CRM_SYSTEM_CRMD, CRM_SYSTEM_DC, NULL); crm_xml_add_int(acknak, F_CRM_JOIN_ID, current_join_id); /* set the ack/nack */ if(safe_str_eq(join_state, CRMD_JOINSTATE_MEMBER)) { crm_debug("join-%d: ACK'ing join request from %s, state %s", current_join_id, join_to, join_state); crm_xml_add(acknak, CRM_OP_JOIN_ACKNAK, XML_BOOLEAN_TRUE); g_hash_table_insert( finalized_nodes, crm_strdup(join_to), crm_strdup(CRMD_JOINSTATE_MEMBER)); } else { crm_warn("join-%d: NACK'ing join request from %s, state %s", current_join_id, join_to, join_state); crm_xml_add(acknak, CRM_OP_JOIN_ACKNAK, XML_BOOLEAN_FALSE); } send_msg_via_ha(acknak); free_xml(acknak); return TRUE; } void ghash_print_node(gpointer key, gpointer value, gpointer user_data); gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source) { crm_debug("Invoked by %s in state: %s", source, fsa_state2string(cur_state)); if(saved_ccm_membership_id != crm_peer_seq) { crm_info("%s: Membership changed since join started: %llu -> %llu", source, saved_ccm_membership_id, crm_peer_seq); register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL); } else if(cur_state == S_INTEGRATION) { if(g_hash_table_size(welcomed_nodes) == 0) { crm_debug("join-%d: Integration of %d peers complete: %s", current_join_id, g_hash_table_size(integrated_nodes), source); register_fsa_input_before( C_FSA_INTERNAL, I_INTEGRATED, NULL); return TRUE; } } else if(cur_state == S_FINALIZE_JOIN) { if(is_set(fsa_input_register, R_HAVE_CIB) == FALSE) { crm_debug("join-%d: Delaying I_FINALIZED until we have the CIB", current_join_id); return TRUE; } else if(g_hash_table_size(integrated_nodes) == 0 && g_hash_table_size(finalized_nodes) == 0) { crm_debug("join-%d complete: %s", current_join_id, source); register_fsa_input_later( C_FSA_INTERNAL, I_FINALIZED, NULL); } else if(g_hash_table_size(integrated_nodes) != 0 && g_hash_table_size(finalized_nodes) != 0) { char *msg = NULL; crm_err("join-%d: Waiting on %d integrated nodes" " AND %d finalized nodes", current_join_id, g_hash_table_size(integrated_nodes), g_hash_table_size(finalized_nodes)); msg = crm_strdup("Integrated node"); g_hash_table_foreach(integrated_nodes, ghash_print_node, msg); crm_free(msg); msg = crm_strdup("Finalized node"); g_hash_table_foreach(finalized_nodes, ghash_print_node, msg); crm_free(msg); } else if(g_hash_table_size(integrated_nodes) != 0) { crm_debug("join-%d: Still waiting on %d integrated nodes", current_join_id, g_hash_table_size(integrated_nodes)); } else if(g_hash_table_size(finalized_nodes) != 0) { crm_debug("join-%d: Still waiting on %d finalized nodes", current_join_id, g_hash_table_size(finalized_nodes)); } } return FALSE; } diff --git a/crmd/te_utils.c b/crmd/te_utils.c index 1a50c47ddd..0b91263d27 100644 --- a/crmd/te_utils.c +++ b/crmd/te_utils.c @@ -1,225 +1,228 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include GCHSource *stonith_src = NULL; GTRIGSource *stonith_reconnect = NULL; gboolean te_connect_stonith(gpointer user_data) { int lpc = 0; int rc = ST_OK; IPC_Channel *fence_ch = NULL; if(stonith_src != NULL) { crm_debug("Still connected"); return TRUE; } for(lpc = 0; lpc < 30; lpc++) { crm_info("Attempting connection to fencing daemon..."); sleep(1); rc = stonithd_signon("tengine"); if(rc == ST_OK) { break; } if(user_data != NULL) { crm_err("Sign-in failed: triggered a retry"); G_main_set_trigger(stonith_reconnect); return TRUE; } crm_err("Sign-in failed: pausing and trying again in 2s..."); sleep(1); } CRM_ASSERT(rc == ST_OK); /* If not, we failed 30 times... just get out */ CRM_ASSERT(stonithd_set_stonith_ops_callback( tengine_stonith_callback) == ST_OK); crm_debug_2("Grabbing IPC channel"); fence_ch = stonithd_input_IPC_channel(); CRM_ASSERT(fence_ch != NULL); crm_debug_2("Attaching to mainloop"); stonith_src = G_main_add_IPC_Channel( G_PRIORITY_LOW, fence_ch, FALSE, tengine_stonith_dispatch, NULL, tengine_stonith_connection_destroy); CRM_ASSERT(stonith_src != NULL); crm_info("Connected"); return TRUE; } gboolean start_global_timer(crm_action_timer_t *timer, int timeout) { CRM_ASSERT(timer != NULL); CRM_CHECK(timer > 0, return FALSE); CRM_CHECK(timer->source_id == 0, return FALSE); if(timeout <= 0) { crm_err("Tried to start timer with period: %d", timeout); } else if(timer->source_id == 0) { - crm_debug_2("Starting abort timer: %dms", timeout); + crm_debug("Starting abort timer: %dms", timeout); timer->timeout = timeout; timer->source_id = Gmain_timeout_add( timeout, global_timer_callback, (void*)timer); CRM_ASSERT(timer->source_id != 0); return TRUE; } else { crm_err("Timer is already active with period: %d", timer->timeout); } return FALSE; } gboolean stop_te_timer(crm_action_timer_t *timer) { const char *timer_desc = "action timer"; if(timer == NULL) { return FALSE; } if(timer->reason == timeout_abort) { timer_desc = "global timer"; + crm_debug("Stopping %s", timer_desc); } if(timer->source_id != 0) { crm_debug_2("Stopping %s", timer_desc); Gmain_timeout_remove(timer->source_id); timer->source_id = 0; } else { + crm_debug("%s was already stopped", timer_desc); return FALSE; } return TRUE; } gboolean te_graph_trigger(gpointer user_data) { int timeout = 0; enum transition_status graph_rc = -1; crm_debug("Invoking the TE graph in state %s", fsa_state2string(fsa_state)); switch(fsa_state) { case S_STARTING: case S_PENDING: case S_NOT_DC: case S_HALT: case S_ILLEGAL: case S_STOPPING: case S_TERMINATE: return TRUE; break; default: break; } if(transition_graph->complete == FALSE) { graph_rc = run_graph(transition_graph); timeout = transition_graph->transition_timeout; print_graph(LOG_DEBUG_3, transition_graph); if(graph_rc == transition_active) { crm_debug_3("Transition not yet complete"); + crm_debug("Restarting TE timer"); stop_te_timer(transition_timer); start_global_timer(transition_timer, timeout); return TRUE; } else if(graph_rc == transition_pending) { crm_debug_3("Transition not yet complete - no actions fired"); return TRUE; } if(graph_rc != transition_complete) { crm_err("Transition failed: %s", transition_status(graph_rc)); print_graph(LOG_WARNING, transition_graph); } } transition_graph->complete = TRUE; notify_crmd(transition_graph); return TRUE; } void trigger_graph_processing(const char *fn, int line) { G_main_set_trigger(transition_trigger); crm_debug_2("%s:%d - Triggered graph processing", fn, line); } void abort_transition_graph( int abort_priority, enum transition_action abort_action, const char *abort_text, xmlNode *reason, const char *fn, int line) { int log_level = LOG_DEBUG; /* if(abort_priority >= INFINITY) { log_level = LOG_INFO; } */ do_crm_log(log_level, "%s:%d - Triggered graph processing (complete=%d) : %s", fn, line, transition_graph->complete, abort_text); if(transition_graph && transition_graph->complete) { register_fsa_input(C_FSA_INTERNAL, I_PE_CALC, NULL); return; } update_abort_priority( transition_graph, abort_priority, abort_action, abort_text); if(reason != NULL) { const char *magic = crm_element_value( reason, XML_ATTR_TRANSITION_MAGIC); if(magic) { do_crm_log(log_level, "Caused by update to %s: %s", ID(reason), magic); } else { crm_log_xml(log_level, "Cause", reason); } } G_main_set_trigger(transition_trigger); } diff --git a/include/crm/crm.h b/include/crm/crm.h index 7b085ef668..b7131b13e0 100644 --- a/include/crm/crm.h +++ b/include/crm/crm.h @@ -1,350 +1,351 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef CRM__H #define CRM__H #include #include #include #undef MIN #undef MAX #include #include #include #ifdef MCHECK #include #endif #include #define CRM_FEATURE_SET "2.1" #define MINIMUM_SCHEMA_VERSION "pacemaker-0.7" #define LATEST_SCHEMA_VERSION "pacemaker-"DTD_VERSION #define EOS '\0' #define DIMOF(a) ((int) (sizeof(a)/sizeof(a[0])) ) #define HAURL(url) HA_URLBASE url #ifndef CRM_DEV_BUILD # define CRM_DEV_BUILD 0 #endif #define CRM_DEPRECATED_SINCE_2_0_1 0 #define CRM_DEPRECATED_SINCE_2_0_2 0 #define CRM_DEPRECATED_SINCE_2_0_3 0 #define CRM_DEPRECATED_SINCE_2_0_4 0 #define CRM_DEPRECATED_SINCE_2_0_5 0 #define CRM_DEPRECATED_SINCE_2_0_6 1 #define CRM_DEPRECATED_SINCE_2_0_7 1 #define CRM_DEPRECATED_SINCE_2_0_8 1 #define CRM_DEPRECATED_SINCE_2_1_0 1 #define CRM_META "CRM_meta" #define crm_meta_name(field) CRM_META"_"field #define ipc_call_diff_max_ms 5000 #define action_diff_warn_ms 5000 #define action_diff_max_ms 20000 #define fsa_diff_warn_ms 10000 #define fsa_diff_max_ms 30000 #include #define CRM_ASSERT(expr) if((expr) == FALSE) { \ crm_abort(__FILE__, __PRETTY_FUNCTION__, __LINE__, #expr, TRUE, FALSE); \ } extern gboolean crm_assert_failed; #define CRM_DEV_ASSERT(expr) \ crm_assert_failed = FALSE; \ if((expr) == FALSE) { \ crm_assert_failed = TRUE; \ crm_abort(__FILE__,__PRETTY_FUNCTION__,__LINE__, #expr, FALSE, TRUE); \ } #define CRM_CHECK(expr, failure_action) if((expr) == FALSE) { \ crm_abort(__FILE__,__PRETTY_FUNCTION__,__LINE__, #expr, FALSE, TRUE); \ failure_action; \ } #define CRM_CHECK_AND_STORE(expr, failure_action) if((expr) == FALSE) { \ crm_abort(__FILE__,__PRETTY_FUNCTION__,__LINE__, #expr, TRUE, TRUE); \ failure_action; \ } extern const char *crm_system_name; /* Clean these up at some point, some probably should be runtime options */ #define WORKING_DIR HA_VARLIBDIR"/heartbeat/crm" #define CRM_SOCK_DIR HA_VARRUNDIR"/heartbeat/crm" #define BIN_DIR HA_LIBDIR"/heartbeat" #define SOCKET_LEN 1024 #define APPNAME_LEN 256 #define MAX_IPC_FAIL 5 #define CIB_FILENAME WORKING_DIR"/cib.xml" #define CIB_BACKUP WORKING_DIR"/cib_backup.xml" #define MSG_LOG 1 #define DOT_FSA_ACTIONS 1 #define DOT_ALL_FSA_INPUTS 1 /* #define FSA_TRACE 1 */ #define INFINITY_S "INFINITY" #define MINUS_INFINITY_S "-INFINITY" #define INFINITY 1000000 /* Sub-systems */ #define CRM_SYSTEM_DC "dc" #define CRM_SYSTEM_DCIB "dcib" /* The master CIB */ #define CRM_SYSTEM_CIB "cib" #define CRM_SYSTEM_CRMD "crmd" #define CRM_SYSTEM_LRMD "lrmd" #define CRM_SYSTEM_PENGINE "pengine" #define CRM_SYSTEM_TENGINE "tengine" #define CRM_SYSTEM_STONITHD "stonithd" /* Valid operations */ #define CRM_OP_NOOP "noop" #define CRM_OP_JOIN_ANNOUNCE "join_announce" #define CRM_OP_JOIN_OFFER "join_offer" #define CRM_OP_JOIN_REQUEST "join_request" #define CRM_OP_JOIN_ACKNAK "join_ack_nack" #define CRM_OP_JOIN_CONFIRM "join_confirm" #define CRM_OP_DIE "die_no_respawn" #define CRM_OP_RETRIVE_CIB "retrieve_cib" #define CRM_OP_PING "ping" #define CRM_OP_VOTE "vote" #define CRM_OP_NOVOTE "no-vote" #define CRM_OP_HELLO "hello" #define CRM_OP_HBEAT "dc_beat" #define CRM_OP_PECALC "pe_calc" #define CRM_OP_ABORT "abort" #define CRM_OP_QUIT "quit" #define CRM_OP_LOCAL_SHUTDOWN "start_shutdown" #define CRM_OP_SHUTDOWN_REQ "req_shutdown" #define CRM_OP_SHUTDOWN "do_shutdown" #define CRM_OP_FENCE "stonith" #define CRM_OP_EVENTCC "event_cc" #define CRM_OP_TEABORT "te_abort" #define CRM_OP_TEABORTED "te_abort_confirmed" /* we asked */ #define CRM_OP_TE_HALT "te_halt" #define CRM_OP_TECOMPLETE "te_complete" #define CRM_OP_TETIMEOUT "te_timeout" #define CRM_OP_TRANSITION "transition" #define CRM_OP_REGISTER "register" #define CRM_OP_DEBUG_UP "debug_inc" #define CRM_OP_DEBUG_DOWN "debug_dec" #define CRM_OP_INVOKE_LRM "lrm_invoke" #define CRM_OP_LRM_REFRESH "lrm_refresh" #define CRM_OP_LRM_QUERY "lrm_query" #define CRM_OP_LRM_DELETE "lrm_delete" #define CRM_OP_LRM_FAIL "lrm_fail" #define CRM_OP_PROBED "probe_complete" #define CRM_OP_REPROBE "probe_again" #define CRMD_STATE_ACTIVE "member" #define CRMD_STATE_INACTIVE "down" #define CRMD_JOINSTATE_DOWN CRMD_STATE_INACTIVE #define CRMD_JOINSTATE_PENDING "pending" #define CRMD_JOINSTATE_MEMBER CRMD_STATE_ACTIVE +#define CRMD_JOINSTATE_NACK "banned" #define CRMD_ACTION_DELETE "delete" #define CRMD_ACTION_CANCEL "cancel" #define CRMD_ACTION_MIGRATE "migrate_to" #define CRMD_ACTION_MIGRATED "migrate_from" #define CRMD_ACTION_START "start" #define CRMD_ACTION_STARTED "running" #define CRMD_ACTION_STOP "stop" #define CRMD_ACTION_STOPPED "stopped" #define CRMD_ACTION_PROMOTE "promote" #define CRMD_ACTION_PROMOTED "promoted" #define CRMD_ACTION_DEMOTE "demote" #define CRMD_ACTION_DEMOTED "demoted" #define CRMD_ACTION_NOTIFY "notify" #define CRMD_ACTION_NOTIFIED "notified" #define CRMD_ACTION_STATUS "monitor" /* short names */ #define RSC_DELETE CRMD_ACTION_DELETE #define RSC_CANCEL CRMD_ACTION_CANCEL #define RSC_MIGRATE CRMD_ACTION_MIGRATE #define RSC_MIGRATED CRMD_ACTION_MIGRATED #define RSC_START CRMD_ACTION_START #define RSC_STARTED CRMD_ACTION_STARTED #define RSC_STOP CRMD_ACTION_STOP #define RSC_STOPPED CRMD_ACTION_STOPPED #define RSC_PROMOTE CRMD_ACTION_PROMOTE #define RSC_PROMOTED CRMD_ACTION_PROMOTED #define RSC_DEMOTE CRMD_ACTION_DEMOTE #define RSC_DEMOTED CRMD_ACTION_DEMOTED #define RSC_NOTIFY CRMD_ACTION_NOTIFY #define RSC_NOTIFIED CRMD_ACTION_NOTIFIED #define RSC_STATUS CRMD_ACTION_STATUS typedef GList* GListPtr; #define slist_destroy(child_type, child, parent, a) \ { \ GListPtr __crm_iter_head = parent; \ child_type *child = NULL; \ while(__crm_iter_head != NULL) { \ child = (child_type *) __crm_iter_head->data; \ __crm_iter_head = __crm_iter_head->next; \ { a; } \ } \ g_list_free(parent); \ } #define slist_iter(child, child_type, parent, counter, a) \ { \ GListPtr __crm_iter_head = parent; \ child_type *child = NULL; \ int counter = 0; \ for(; __crm_iter_head != NULL; counter++) { \ child = (child_type *) __crm_iter_head->data; \ __crm_iter_head = __crm_iter_head->next; \ { a; } \ } \ } #define LOG_DEBUG_2 LOG_DEBUG+1 #define LOG_DEBUG_3 LOG_DEBUG+2 #define LOG_DEBUG_4 LOG_DEBUG+3 #define LOG_DEBUG_5 LOG_DEBUG+4 #define LOG_DEBUG_6 LOG_DEBUG+5 #define LOG_MSG LOG_DEBUG_3 /* * Throughout the macros below, note the leading, pre-comma, space in the * various ' , ##args' occurences to aid portability across versions of 'gcc'. * http://gcc.gnu.org/onlinedocs/cpp/Variadic-Macros.html#Variadic-Macros */ #define do_crm_log(level, fmt, args...) do { \ if(crm_log_level < (level)) { \ continue; \ } else if((level) > LOG_DEBUG) { \ cl_log(LOG_DEBUG, "debug%d: %s: " fmt, \ level-LOG_INFO, __PRETTY_FUNCTION__ , ##args); \ } else { \ cl_log(level, "%s: " fmt, \ __PRETTY_FUNCTION__ , ##args); \ } \ } while(0) #define crm_crit(fmt, args...) do_crm_log(LOG_CRIT, fmt , ##args) #define crm_err(fmt, args...) do_crm_log(LOG_ERR, fmt , ##args) #define crm_warn(fmt, args...) do_crm_log(LOG_WARNING, fmt , ##args) #define crm_notice(fmt, args...) do_crm_log(LOG_NOTICE, fmt , ##args) #define crm_info(fmt, args...) do_crm_log(LOG_INFO, fmt , ##args) #define crm_debug(fmt, args...) do_crm_log(LOG_DEBUG, fmt , ##args) #define crm_debug_2(fmt, args...) do_crm_log(LOG_DEBUG_2, fmt , ##args) #define crm_debug_3(fmt, args...) do_crm_log(LOG_DEBUG_3, fmt , ##args) #define crm_debug_4(fmt, args...) do_crm_log(LOG_DEBUG_4, fmt , ##args) #define crm_debug_5(fmt, args...) do_crm_log(LOG_DEBUG_5, fmt , ##args) #define crm_debug_6(fmt, args...) do_crm_log(LOG_DEBUG_6, fmt , ##args) extern void crm_log_message_adv( int level, const char *alt_debugfile, const HA_Message *msg); #define crm_log_xml(level, text, xml) if(crm_log_level >= (level)) { \ print_xml_formatted(level, __PRETTY_FUNCTION__, xml, text); \ } #define crm_log_xml_crit(xml, text) crm_log_xml(LOG_CRIT, text, xml) #define crm_log_xml_err(xml, text) crm_log_xml(LOG_ERR, text, xml) #define crm_log_xml_warn(xml, text) crm_log_xml(LOG_WARNING, text, xml) #define crm_log_xml_notice(xml, text) crm_log_xml(LOG_NOTICE, text, xml) #define crm_log_xml_info(xml, text) crm_log_xml(LOG_INFO, text, xml) #define crm_log_xml_debug(xml, text) crm_log_xml(LOG_DEBUG, text, xml) #define crm_log_xml_debug_2(xml, text) crm_log_xml(LOG_DEBUG_2, text, xml) #define crm_log_xml_debug_3(xml, text) crm_log_xml(LOG_DEBUG_3, text, xml) #define crm_log_xml_debug_4(xml, text) crm_log_xml(LOG_DEBUG_4, text, xml) #define crm_log_xml_debug_5(xml, text) crm_log_xml(LOG_DEBUG_5, text, xml) #define crm_str(x) (const char*)(x?x:"") #if CRM_DEV_BUILD # define crm_malloc0(malloc_obj, length) do { \ if(malloc_obj) { \ crm_err("Potential memory leak:" \ " %s at %s:%d not NULL before alloc.", \ #malloc_obj, __FILE__, __LINE__); \ } \ malloc_obj = cl_malloc(length); \ if(malloc_obj == NULL) { \ crm_err("Failed allocation of %lu bytes", (unsigned long)length); \ CRM_ASSERT(malloc_obj != NULL); \ } \ memset(malloc_obj, 0, length); \ } while(0) /* it's not a memory leak to already have an object to realloc, that's * the usual case, however if it does have a value, it must have been * allocated by the same allocator! */ # define crm_realloc(realloc_obj, length) do { \ if (realloc_obj != NULL) { \ CRM_ASSERT(cl_is_allocated(realloc_obj) == 1); \ } \ realloc_obj = cl_realloc(realloc_obj, length); \ CRM_ASSERT(realloc_obj != NULL); \ } while(0) # define crm_free(free_obj) if(free_obj) { \ CRM_ASSERT(cl_is_allocated(free_obj) == 1); \ cl_free(free_obj); \ free_obj=NULL; \ } #else # define crm_malloc0(malloc_obj, length) do { \ malloc_obj = cl_malloc(length); \ if(malloc_obj == NULL) { \ crm_err("Failed allocation of %lu bytes", (unsigned long)length); \ CRM_ASSERT(malloc_obj != NULL); \ } \ memset(malloc_obj, 0, length); \ } while(0) # define crm_realloc(realloc_obj, length) do { \ realloc_obj = cl_realloc(realloc_obj, length); \ CRM_ASSERT(realloc_obj != NULL); \ } while(0) # define crm_free(free_obj) if(free_obj) { cl_free(free_obj); free_obj=NULL; } #endif #define crm_msg_del(msg) if(msg != NULL) { ha_msg_del(msg); msg = NULL; } #define crm_strdup(str) crm_strdup_fn(str, __FILE__, __PRETTY_FUNCTION__, __LINE__) #endif diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h index d0759d93b2..515930a2fe 100644 --- a/include/crm/pengine/status.h +++ b/include/crm/pengine/status.h @@ -1,234 +1,235 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef PENGINE_STATUS__H #define PENGINE_STATUS__H #include #include #include typedef struct node_s node_t; typedef struct action_s action_t; typedef struct resource_s resource_t; typedef enum no_quorum_policy_e { no_quorum_freeze, no_quorum_stop, no_quorum_ignore } no_quorum_policy_t; enum node_type { node_ping, node_member }; enum pe_restart { pe_restart_restart, pe_restart_ignore }; typedef struct pe_working_set_s { xmlNode *input; ha_time_t *now; /* options extracted from the input */ char *transition_idle_timeout; char *dc_uuid; node_t *dc_node; gboolean have_quorum; gboolean stonith_enabled; const char *stonith_action; gboolean symmetric_cluster; gboolean is_managed_default; gboolean start_failure_fatal; gboolean remove_after_stop; gboolean stop_rsc_orphans; gboolean stop_action_orphans; gboolean stop_everything; int default_failure_timeout; int default_migration_threshold; int default_resource_stickiness; no_quorum_policy_t no_quorum_policy; GHashTable *config_hash; GListPtr nodes; GListPtr resources; GListPtr placement_constraints; GListPtr ordering_constraints; GListPtr colocation_constraints; GListPtr actions; xmlNode *failed; /* stats */ int num_synapse; int max_valid_nodes; int order_id; int action_id; /* final output */ xmlNode *graph; } pe_working_set_t; struct node_shared_s { const char *id; const char *uname; gboolean online; gboolean standby; + gboolean pending; gboolean unclean; gboolean shutdown; gboolean expected_up; gboolean is_dc; int num_resources; GListPtr running_rsc; /* resource_t* */ GListPtr allocated_rsc; /* resource_t* */ GHashTable *attrs; /* char* => char* */ enum node_type type; }; struct node_s { int weight; gboolean fixed; int count; struct node_shared_s *details; }; #include #define pe_rsc_orphan 0x00000001ULL #define pe_rsc_managed 0x00000002ULL #define pe_rsc_notify 0x00000010ULL #define pe_rsc_unique 0x00000020ULL #define pe_rsc_can_migrate 0x00000040ULL #define pe_rsc_provisional 0x00000100ULL #define pe_rsc_allocating 0x00000200ULL #define pe_rsc_merging 0x00000400ULL #define pe_rsc_failed 0x00010000ULL #define pe_rsc_shutdown 0x00020000ULL #define pe_rsc_runnable 0x00040000ULL #define pe_rsc_start_pending 0x00080000ULL #define pe_rsc_starting 0x00100000ULL #define pe_rsc_stopping 0x00200000ULL struct resource_s { char *id; char *clone_name; char *long_name; xmlNode *xml; xmlNode *ops_xml; resource_t *parent; void *variant_opaque; enum pe_obj_types variant; resource_object_functions_t *fns; resource_alloc_functions_t *cmds; enum rsc_recovery_type recovery_type; enum pe_restart restart_type; int priority; int stickiness; int sort_index; int failure_timeout; int effective_priority; int migration_threshold; unsigned long long flags; GListPtr rsc_cons_lhs; /* rsc_colocation_t* */ GListPtr rsc_cons; /* rsc_colocation_t* */ GListPtr rsc_location; /* rsc_to_node_t* */ GListPtr actions; /* action_t* */ node_t *allocated_to; GListPtr running_on; /* node_t* */ GListPtr known_on; /* node_t* */ GListPtr allowed_nodes; /* node_t* */ enum rsc_role_e role; enum rsc_role_e next_role; GHashTable *meta; GHashTable *parameters; GListPtr children; /* resource_t* */ }; struct action_s { int id; int priority; resource_t *rsc; void *rsc_opaque; node_t *node; char *task; char *uuid; xmlNode *op_entry; gboolean pseudo; gboolean runnable; gboolean optional; gboolean print_always; gboolean failure_is_fatal; gboolean implied_by_stonith; gboolean allow_reload_conversion; enum rsc_start_requirement needs; enum action_fail_response on_fail; enum rsc_role_e fail_role; gboolean dumped; gboolean processed; action_t *pre_notify; action_t *pre_notified; action_t *post_notify; action_t *post_notified; int seen_count; GHashTable *meta; GHashTable *extra; GHashTable *notify_keys; /* do NOT free */ GListPtr actions_before; /* action_warpper_t* */ GListPtr actions_after; /* action_warpper_t* */ }; gboolean cluster_status(pe_working_set_t *data_set); extern void set_working_set_defaults(pe_working_set_t *data_set); extern void cleanup_calculations(pe_working_set_t *data_set); extern resource_t *pe_find_resource(GListPtr rsc_list, const char *id_rh); extern node_t *pe_find_node(GListPtr node_list, const char *uname); extern node_t *pe_find_node_id(GListPtr node_list, const char *id); extern GListPtr find_operations( const char *rsc, const char *node, gboolean active_filter, pe_working_set_t *data_set); #endif diff --git a/lib/crm/cib/cib_client.c b/lib/crm/cib/cib_client.c index c5bf446fc3..e1b40fc720 100644 --- a/lib/crm/cib/cib_client.c +++ b/lib/crm/cib/cib_client.c @@ -1,557 +1,568 @@ /* * Copyright (c) 2004 Andrew Beekhof * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include GHashTable *cib_op_callback_table = NULL; int cib_client_set_op_callback( cib_t *cib, void (*callback)(const xmlNode *msg, int call_id, int rc, xmlNode *output)); int cib_client_add_notify_callback( cib_t *cib, const char *event, void (*callback)( const char *event, xmlNode *msg)); int cib_client_del_notify_callback( cib_t *cib, const char *event, void (*callback)( const char *event, xmlNode *msg)); gint ciblib_GCompareFunc(gconstpointer a, gconstpointer b); #define op_common(cib) \ if(cib == NULL) { \ return cib_missing; \ } else if(cib->cmds->variant_op == NULL) { \ return cib_variant; \ } static int cib_client_noop(cib_t *cib, int call_options) { op_common(cib) return cib->cmds->variant_op( cib, CRM_OP_NOOP, NULL, NULL, NULL, NULL, call_options); } static int cib_client_ping(cib_t *cib, xmlNode **output_data, int call_options) { op_common(cib) return cib->cmds->variant_op( cib, CRM_OP_PING, NULL,NULL,NULL, output_data, call_options); } static int cib_client_query(cib_t *cib, const char *section, xmlNode **output_data, int call_options) { return cib->cmds->query_from( cib, NULL, section, output_data, call_options); } static int cib_client_query_from(cib_t *cib, const char *host, const char *section, xmlNode **output_data, int call_options) { op_common(cib) return cib->cmds->variant_op(cib, CIB_OP_QUERY, host, section, NULL, output_data, call_options); } static int cib_client_is_master(cib_t *cib) { op_common(cib) return cib->cmds->variant_op( cib, CIB_OP_ISMASTER, NULL, NULL,NULL,NULL, cib_scope_local|cib_sync_call); } static int cib_client_set_slave(cib_t *cib, int call_options) { op_common(cib) return cib->cmds->variant_op( cib, CIB_OP_SLAVE, NULL,NULL,NULL,NULL, call_options); } static int cib_client_set_slave_all(cib_t *cib, int call_options) { op_common(cib) return cib->cmds->variant_op( cib, CIB_OP_SLAVEALL, NULL,NULL,NULL,NULL, call_options); } static int cib_client_set_master(cib_t *cib, int call_options) { op_common(cib) crm_debug_3("Adding cib_scope_local to options"); return cib->cmds->variant_op( cib, CIB_OP_MASTER, NULL,NULL,NULL,NULL, call_options|cib_scope_local); } static int cib_client_bump_epoch(cib_t *cib, int call_options) { op_common(cib) return cib->cmds->variant_op( cib, CIB_OP_BUMP, NULL, NULL, NULL, NULL, call_options); } static int cib_client_sync(cib_t *cib, const char *section, int call_options) { return cib->cmds->sync_from(cib, NULL, section, call_options); } static int cib_client_sync_from( cib_t *cib, const char *host, const char *section, int call_options) { op_common(cib) return cib->cmds->variant_op( cib, CIB_OP_SYNC, host, section, NULL, NULL, call_options); } static int cib_client_create( cib_t *cib, const char *section, xmlNode *data, int call_options) { op_common(cib) return cib->cmds->variant_op( cib, CIB_OP_CREATE, NULL, section, data, NULL, call_options); } static int cib_client_modify( cib_t *cib, const char *section, xmlNode *data, int call_options) { op_common(cib) return cib->cmds->variant_op( cib, CIB_OP_MODIFY, NULL, section, data, NULL, call_options); } static int cib_client_update( cib_t *cib, const char *section, xmlNode *data, int call_options) { op_common(cib) return cib->cmds->variant_op( cib, CIB_OP_MODIFY, NULL, section, data, NULL, call_options); } static int cib_client_replace( cib_t *cib, const char *section, xmlNode *data, int call_options) { op_common(cib) return cib->cmds->variant_op( cib, CIB_OP_REPLACE, NULL, section, data, NULL, call_options); } static int cib_client_delete( cib_t *cib, const char *section, xmlNode *data, int call_options) { op_common(cib) return cib->cmds->variant_op( cib, CIB_OP_DELETE, NULL, section, data, NULL, call_options); } static int cib_client_delete_absolute( cib_t *cib, const char *section, xmlNode *data, int call_options) { op_common(cib) return cib->cmds->variant_op( cib, CIB_OP_DELETE_ALT, NULL, section, data, NULL, call_options); } static int cib_client_erase( cib_t *cib, xmlNode **output_data, int call_options) { op_common(cib) return cib->cmds->variant_op( cib, CIB_OP_ERASE, NULL, NULL, NULL, output_data, call_options); } static int cib_client_quit(cib_t *cib, int call_options) { op_common(cib) return cib->cmds->variant_op( cib, CRM_OP_QUIT, NULL, NULL, NULL, NULL, call_options); } static void cib_destroy_op_callback(gpointer data) { cib_callback_client_t *blob = data; if(blob->timer && blob->timer->ref > 0) { g_source_remove(blob->timer->ref); } crm_free(blob); } char *get_shadow_file(const char *name) { return crm_concat(WORKING_DIR"/shadow", name, '.'); } cib_t* cib_shadow_new(const char *shadow) { cib_t *new_cib = NULL; char *shadow_file = NULL; CRM_CHECK(shadow != NULL, return NULL); shadow_file = get_shadow_file(shadow); new_cib = cib_file_new(shadow_file); crm_free(shadow_file); return new_cib; } cib_t *cib_new_no_shadow(void) { unsetenv("CIB_shadow"); return cib_new(); } cib_t* cib_new(void) { const char *value = getenv("CIB_shadow"); if(value) { return cib_shadow_new(value); } value = getenv("CIB_file"); if(value) { return cib_file_new(value); } value = getenv("CIB_port"); if(value) { int port = crm_parse_int(value, NULL); const char *server = getenv("CIB_server"); const char *user = getenv("CIB_user"); const char *pass = getenv("CIB_passwd"); if(user == NULL) { user = getenv("USER"); crm_info("Defaulting to current user: %s", user); } if(server == NULL) { server = "localhost"; crm_info("Defaulting to localhost"); } return cib_remote_new(server, user, pass, port); } return cib_native_new(); } /* this is backwards... cib_*_new should call this not the other way around */ cib_t* cib_new_variant(void) { cib_t* new_cib = NULL; crm_malloc0(new_cib, sizeof(cib_t)); if(cib_op_callback_table != NULL) { g_hash_table_destroy(cib_op_callback_table); cib_op_callback_table = NULL; } if(cib_op_callback_table == NULL) { cib_op_callback_table = g_hash_table_new_full( g_direct_hash, g_direct_equal, NULL, cib_destroy_op_callback); } new_cib->call_id = 1; new_cib->variant = cib_undefined; new_cib->type = cib_none; new_cib->state = cib_disconnected; new_cib->op_callback = NULL; new_cib->variant_opaque = NULL; new_cib->notify_list = NULL; /* the rest will get filled in by the variant constructor */ crm_malloc0(new_cib->cmds, sizeof(cib_api_operations_t)); new_cib->cmds->set_op_callback = cib_client_set_op_callback; new_cib->cmds->add_notify_callback = cib_client_add_notify_callback; new_cib->cmds->del_notify_callback = cib_client_del_notify_callback; new_cib->cmds->register_callback = cib_client_register_callback; new_cib->cmds->noop = cib_client_noop; new_cib->cmds->ping = cib_client_ping; new_cib->cmds->query = cib_client_query; new_cib->cmds->sync = cib_client_sync; new_cib->cmds->query_from = cib_client_query_from; new_cib->cmds->sync_from = cib_client_sync_from; new_cib->cmds->is_master = cib_client_is_master; new_cib->cmds->set_master = cib_client_set_master; new_cib->cmds->set_slave = cib_client_set_slave; new_cib->cmds->set_slave_all = cib_client_set_slave_all; new_cib->cmds->bump_epoch = cib_client_bump_epoch; new_cib->cmds->create = cib_client_create; new_cib->cmds->modify = cib_client_modify; new_cib->cmds->update = cib_client_update; new_cib->cmds->replace = cib_client_replace; new_cib->cmds->delete = cib_client_delete; new_cib->cmds->erase = cib_client_erase; new_cib->cmds->quit = cib_client_quit; new_cib->cmds->delete_absolute = cib_client_delete_absolute; return new_cib; } void cib_delete(cib_t *cib) { GList *list = cib->notify_list; while(list != NULL) { cib_notify_client_t *client = g_list_nth_data(list, 0); list = g_list_remove(list, client); crm_free(client); } g_hash_table_destroy(cib_op_callback_table); cib->cmds->free(cib); cib = NULL; } int cib_client_set_op_callback( cib_t *cib, void (*callback)(const xmlNode *msg, int call_id, int rc, xmlNode *output)) { if(callback == NULL) { crm_info("Un-Setting operation callback"); } else { crm_debug_3("Setting operation callback"); } cib->op_callback = callback; return cib_ok; } int cib_client_add_notify_callback( cib_t *cib, const char *event, void (*callback)( const char *event, xmlNode *msg)) { GList *list_item = NULL; cib_notify_client_t *new_client = NULL; - if(cib->variant != cib_native) { - crm_err("Not supported"); + if(cib->variant != cib_native + && cib->variant != cib_remote) { return cib_NOTSUPPORTED; } crm_debug_2("Adding callback for %s events (%d)", event, g_list_length(cib->notify_list)); crm_malloc0(new_client, sizeof(cib_notify_client_t)); new_client->event = event; new_client->callback = callback; list_item = g_list_find_custom( cib->notify_list, new_client, ciblib_GCompareFunc); if(list_item != NULL) { crm_warn("Callback already present"); crm_free(new_client); + return cib_EXISTS; } else { cib->notify_list = g_list_append( cib->notify_list, new_client); cib->cmds->register_notification(cib, event, 1); crm_debug_3("Callback added (%d)", g_list_length(cib->notify_list)); } return cib_ok; } int cib_client_del_notify_callback( cib_t *cib, const char *event, void (*callback)( const char *event, xmlNode *msg)) { GList *list_item = NULL; cib_notify_client_t *new_client = NULL; - if(cib->variant != cib_native) { - crm_err("Not supported"); + if(cib->variant != cib_native + && cib->variant != cib_remote) { return cib_NOTSUPPORTED; } - + crm_debug("Removing callback for %s events", event); crm_malloc0(new_client, sizeof(cib_notify_client_t)); new_client->event = event; new_client->callback = callback; list_item = g_list_find_custom( cib->notify_list, new_client, ciblib_GCompareFunc); cib->cmds->register_notification(cib, event, 0); if(list_item != NULL) { cib_notify_client_t *list_client = list_item->data; cib->notify_list = g_list_remove(cib->notify_list, list_client); crm_free(list_client); crm_debug_3("Removed callback"); } else { crm_debug_3("Callback not present"); } crm_free(new_client); return cib_ok; } gint ciblib_GCompareFunc(gconstpointer a, gconstpointer b) { - const cib_notify_client_t *a_client = a; - const cib_notify_client_t *b_client = b; - if(a_client->callback == b_client->callback - && safe_str_neq(a_client->event, b_client->event)) { - return 0; + int rc = 0; + const cib_notify_client_t *a_client = a; + const cib_notify_client_t *b_client = b; + + CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0); + rc = strcmp(a_client->event, b_client->event); + if(rc == 0) { + if(a_client->callback == b_client->callback) { + return 0; } else if(((long)a_client->callback) < ((long)b_client->callback)) { - return -1; - } + crm_err("callbacks for %s are not equal: %p vs. %p", + a_client->event, a_client->callback, b_client->callback); + return -1; + } + crm_err("callbacks for %s are not equal: %p vs. %p", + a_client->event, a_client->callback, b_client->callback); return 1; + } + return rc; } static gboolean cib_async_timeout_handler(gpointer data) { struct timer_rec_s *timer = data; crm_debug("Async call %d timed out after %ds", timer->call_id, timer->timeout); cib_native_callback(timer->cib, NULL, timer->call_id, cib_remote_timeout); /* Always return TRUE, never remove the handler * We do that in remove_cib_op_callback() */ return TRUE; } gboolean cib_client_register_callback( cib_t *cib, int call_id, int timeout, gboolean only_success, void *user_data, const char *callback_name, void (*callback)(xmlNode*, int, int, xmlNode*,void*)) { cib_callback_client_t *blob = NULL; if(call_id < 0) { crm_warn("CIB call failed: %s", cib_error2string(call_id)); if(only_success == FALSE) { callback(NULL, call_id, call_id, NULL, user_data); } return FALSE; } crm_malloc0(blob, sizeof(cib_callback_client_t)); blob->id = callback_name; blob->only_success = only_success; blob->user_data = user_data; blob->callback = callback; if(timeout > 0) { struct timer_rec_s *async_timer = NULL; crm_malloc0(async_timer, sizeof(struct timer_rec_s)); blob->timer = async_timer; async_timer->cib = cib; async_timer->call_id = call_id; async_timer->timeout = timeout*1000; async_timer->ref = Gmain_timeout_add( async_timer->timeout, cib_async_timeout_handler, async_timer); } g_hash_table_insert(cib_op_callback_table, GINT_TO_POINTER(call_id), blob); return TRUE; } void remove_cib_op_callback(int call_id, gboolean all_callbacks) { if(all_callbacks) { if(cib_op_callback_table != NULL) { g_hash_table_destroy(cib_op_callback_table); } cib_op_callback_table = g_hash_table_new_full( g_direct_hash, g_direct_equal, NULL, cib_destroy_op_callback); } else { g_hash_table_remove(cib_op_callback_table, GINT_TO_POINTER(call_id)); } } int num_cib_op_callbacks(void) { if(cib_op_callback_table == NULL) { return 0; } return g_hash_table_size(cib_op_callback_table); } static void cib_dump_pending_op(gpointer key, gpointer value, gpointer user_data) { int call = GPOINTER_TO_INT(key); cib_callback_client_t *blob = value; crm_debug("Call %d (%s): pending", call, crm_str(blob->id)); } void cib_dump_pending_callbacks(void) { if(cib_op_callback_table == NULL) { return; } return g_hash_table_foreach(cib_op_callback_table, cib_dump_pending_op, NULL); } diff --git a/lib/crm/cib/cib_native.c b/lib/crm/cib/cib_native.c index d2074290f2..0f6364d585 100644 --- a/lib/crm/cib/cib_native.c +++ b/lib/crm/cib/cib_native.c @@ -1,650 +1,660 @@ /* * Copyright (c) 2004 International Business Machines * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include typedef struct cib_native_opaque_s { IPC_Channel *command_channel; IPC_Channel *callback_channel; GCHSource *callback_source; char *token; } cib_native_opaque_t; int cib_native_perform_op( cib_t *cib, const char *op, const char *host, const char *section, xmlNode *data, xmlNode **output_data, int call_options); int cib_native_free(cib_t* cib); int cib_native_signoff(cib_t* cib); int cib_native_signon(cib_t* cib, const char *name, enum cib_conn_type type); IPC_Channel *cib_native_channel(cib_t* cib); gboolean cib_native_msgready(cib_t* cib); gboolean cib_native_dispatch(IPC_Channel *channel, gpointer user_data); int cib_native_inputfd(cib_t* cib); int cib_native_rcvmsg(cib_t* cib, int blocking); int cib_native_set_connection_dnotify(cib_t *cib, void (*dnotify)(gpointer user_data)); cib_t* cib_native_new (void) { cib_native_opaque_t *native = NULL; cib_t *cib = cib_new_variant(); crm_malloc0(native, sizeof(cib_native_opaque_t)); cib->variant = cib_native; cib->variant_opaque = native; native->command_channel = NULL; native->callback_channel = NULL; /* assign variant specific ops*/ cib->cmds->variant_op = cib_native_perform_op; cib->cmds->signon = cib_native_signon; cib->cmds->signoff = cib_native_signoff; cib->cmds->free = cib_native_free; cib->cmds->inputfd = cib_native_inputfd; cib->cmds->register_notification = cib_native_register_notification; cib->cmds->set_connection_dnotify = cib_native_set_connection_dnotify; return cib; } int cib_native_signon(cib_t* cib, const char *name, enum cib_conn_type type) { int rc = cib_ok; xmlNode *hello = NULL; char *uuid_ticket = NULL; cib_native_opaque_t *native = cib->variant_opaque; crm_debug_4("Connecting command channel"); if(type == cib_command) { cib->state = cib_connected_command; native->command_channel = init_client_ipc_comms_nodispatch( cib_channel_rw); } else if(type == cib_query) { cib->state = cib_connected_query; native->command_channel = init_client_ipc_comms_nodispatch( cib_channel_ro); } else { return cib_not_connected; } if(native->command_channel == NULL) { crm_debug("Connection to command channel failed"); rc = cib_connection; } else if(native->command_channel->ch_status != IPC_CONNECT) { crm_err("Connection may have succeeded," " but authentication to command channel failed"); rc = cib_authentication; } if(rc == cib_ok) { if(rc == cib_ok) { rc = get_channel_token(native->command_channel, &uuid_ticket); native->token = uuid_ticket; } } if(rc == cib_ok) { crm_debug_4("Connecting callback channel"); native->callback_source = init_client_ipc_comms( cib_channel_callback, cib_native_dispatch, cib, &(native->callback_channel)); if(native->callback_channel == NULL) { crm_debug("Connection to callback channel failed"); rc = cib_connection; } else if(native->callback_channel->ch_status != IPC_CONNECT) { crm_err("Connection may have succeeded," " but authentication to callback channel failed"); rc = cib_authentication; } else if(native->callback_source == NULL) { crm_err("Callback source not recorded"); rc = cib_connection; } else { native->callback_channel->send_queue->max_qlen = 500; } } if(rc == cib_ok) { rc = get_channel_token(native->callback_channel, &uuid_ticket); if(rc == cib_ok) { crm_free(native->token); native->token = uuid_ticket; } } if(rc == cib_ok) { CRM_CHECK(native->token != NULL, ;); hello = cib_create_op(0, native->token, CRM_OP_REGISTER, NULL, NULL, NULL, 0); crm_xml_add(hello, F_CIB_CLIENTNAME, name); if(send_ipc_message(native->command_channel, hello) == FALSE) { rc = cib_callback_register; } free_xml(hello); } if(rc == cib_ok) { crm_debug("Connection to CIB successful"); return cib_ok; } crm_debug("Connection to CIB failed: %s", cib_error2string(rc)); cib_native_signoff(cib); return rc; } int cib_native_signoff(cib_t* cib) { cib_native_opaque_t *native = cib->variant_opaque; crm_debug("Signing out of the CIB Service"); /* close channels */ if (native->command_channel != NULL) { native->command_channel->ops->destroy( native->command_channel); native->command_channel = NULL; } if (native->callback_source != NULL) { G_main_del_IPC_Channel(native->callback_source); native->callback_source = NULL; } if (native->callback_channel != NULL) { #ifdef BUG native->callback_channel->ops->destroy( native->callback_channel); #endif native->callback_channel = NULL; } cib->state = cib_disconnected; cib->type = cib_none; return cib_ok; } int cib_native_free (cib_t* cib) { int rc = cib_ok; crm_warn("Freeing CIB"); if(cib->state != cib_disconnected) { rc = cib_native_signoff(cib); if(rc == cib_ok) { crm_free(cib->variant_opaque); crm_free(cib->cmds); crm_free(cib); } } return rc; } IPC_Channel * cib_native_channel(cib_t* cib) { cib_native_opaque_t *native = NULL; if(cib == NULL) { crm_err("Missing cib object"); return NULL; } native = cib->variant_opaque; if(native != NULL) { return native->callback_channel; } crm_err("couldnt find variant specific data in %p", cib); return NULL; } int cib_native_inputfd(cib_t* cib) { IPC_Channel *ch = cib_native_channel(cib); return ch->ops->get_recv_select_fd(ch); } static gboolean timer_expired = FALSE; static struct timer_rec_s *sync_timer = NULL; static gboolean cib_timeout_handler(gpointer data) { struct timer_rec_s *timer = data; timer_expired = TRUE; crm_err("Call %d timed out after %ds", timer->call_id, timer->timeout); /* Always return TRUE, never remove the handler * We do that after the while-loop in cib_native_perform_op() */ return TRUE; } int cib_native_perform_op( cib_t *cib, const char *op, const char *host, const char *section, xmlNode *data, xmlNode **output_data, int call_options) { int rc = HA_OK; xmlNode *op_msg = NULL; xmlNode *op_reply = NULL; cib_native_opaque_t *native = cib->variant_opaque; if(sync_timer == NULL) { crm_malloc0(sync_timer, sizeof(struct timer_rec_s)); } if(cib->state == cib_disconnected) { return cib_not_connected; } if(output_data != NULL) { *output_data = NULL; } if(op == NULL) { crm_err("No operation specified"); return cib_operation; } cib->call_id++; /* prevent call_id from being negative (or zero) and conflicting * with the cib_errors enum * use 2 because we use it as (cib->call_id - 1) below */ if(cib->call_id < 1) { cib->call_id = 1; } CRM_CHECK(native->token != NULL, ;); op_msg = cib_create_op( cib->call_id, native->token, op, host, section, data, call_options); if(op_msg == NULL) { return cib_create_msg; } crm_debug_3("Sending %s message to CIB service", op); if(send_ipc_message(native->command_channel, op_msg) == FALSE) { crm_err("Sending message to CIB service FAILED"); free_xml(op_msg); return cib_send_failed; } else { crm_debug_3("Message sent"); } free_xml(op_msg); if((call_options & cib_discard_reply)) { crm_debug_3("Discarding reply"); return cib_ok; } else if(!(call_options & cib_sync_call)) { crm_debug_3("Async call, returning"); CRM_CHECK(cib->call_id != 0, return cib_reply_failed); return cib->call_id; } rc = IPC_OK; crm_debug_3("Waiting for a syncronous reply"); if(cib->call_timeout > 0) { /* We need this, even with msgfromIPC_timeout(), because we might * get other/older replies that don't match the active request */ timer_expired = FALSE; sync_timer->call_id = cib->call_id; sync_timer->timeout = cib->call_timeout*1000; sync_timer->ref = Gmain_timeout_add( sync_timer->timeout, cib_timeout_handler, sync_timer); } while(timer_expired == FALSE && IPC_ISRCONN(native->command_channel)) { int reply_id = -1; int msg_id = cib->call_id; op_reply = xmlfromIPC(native->command_channel, cib->call_timeout); if(op_reply == NULL) { break; } crm_element_value_int(op_reply, F_CIB_CALLID, &reply_id); CRM_CHECK(reply_id > 0, crm_log_xml(LOG_ERR, "Invalid call id", op_reply); free_xml(op_reply); if(sync_timer->ref > 0) { g_source_remove(sync_timer->ref); sync_timer->ref = 0; } return cib_reply_failed); if(reply_id == msg_id) { break; } else if(reply_id < msg_id) { crm_debug("Recieved old reply: %d (wanted %d)", reply_id, msg_id); crm_log_xml(LOG_MSG, "Old reply", op_reply); } else if((reply_id - 10000) > msg_id) { /* wrap-around case */ crm_debug("Recieved old reply: %d (wanted %d)", reply_id, msg_id); crm_log_xml( LOG_MSG, "Old reply", op_reply); } else { crm_err("Received a __future__ reply:" " %d (wanted %d)", reply_id, msg_id); } free_xml(op_reply); op_reply = NULL; } if(sync_timer->ref > 0) { g_source_remove(sync_timer->ref); sync_timer->ref = 0; } if(timer_expired) { return cib_remote_timeout; } if(IPC_ISRCONN(native->command_channel) == FALSE) { crm_err("CIB disconnected: %d", native->command_channel->ch_status); cib->state = cib_disconnected; } if(op_reply == NULL) { crm_err("No reply message - empty - %d", rc); return cib_reply_failed; } crm_debug_3("Syncronous reply recieved"); rc = cib_ok; /* Start processing the reply... */ if(crm_element_value_int(op_reply, F_CIB_RC, &rc) != 0) { rc = cib_return_code; } if(rc == cib_diff_resync) { /* This is an internal value that clients do not and should not care about */ rc = cib_ok; } if(rc == cib_ok || rc == cib_not_master || rc == cib_master_timeout) { crm_log_xml(LOG_MSG, "passed", op_reply); } else { /* } else if(rc == cib_remote_timeout) { */ crm_err("Call failed: %s", cib_error2string(rc)); crm_log_xml(LOG_WARNING, "failed", op_reply); } if(output_data == NULL) { /* do nothing more */ } else if(!(call_options & cib_discard_reply)) { xmlNode *tmp = get_message_xml(op_reply, F_CIB_CALLDATA); if(tmp == NULL) { crm_debug_3("No output in reply to \"%s\" command %d", op, cib->call_id - 1); } else { *output_data = copy_xml(tmp); } } free_xml(op_reply); return rc; } gboolean cib_native_msgready(cib_t* cib) { cib_native_opaque_t *native = NULL; if (cib == NULL) { crm_err("No CIB!"); return FALSE; } native = cib->variant_opaque; if(native->command_channel != NULL) { /* drain the channel */ IPC_Channel *cmd_ch = native->command_channel; xmlNode *cmd_msg = NULL; while(cmd_ch->ch_status != IPC_DISCONNECT && cmd_ch->ops->is_message_pending(cmd_ch)) { /* this will happen when the CIB exited from beneath us */ cmd_msg = xmlfromIPC(cmd_ch, 0); free_xml(cmd_msg); } } else { crm_err("No command channel"); } if(native->callback_channel == NULL) { crm_err("No callback channel"); return FALSE; } else if(native->callback_channel->ch_status == IPC_DISCONNECT) { crm_info("Lost connection to the CIB service [%d].", native->callback_channel->farside_pid); return FALSE; } else if(native->callback_channel->ops->is_message_pending( native->callback_channel)) { crm_debug_4("Message pending on command channel [%d]", native->callback_channel->farside_pid); return TRUE; } crm_debug_3("No message pending"); return FALSE; } int cib_native_rcvmsg(cib_t* cib, int blocking) { const char *type = NULL; xmlNode* msg = NULL; cib_native_opaque_t *native = NULL; if (cib == NULL) { crm_err("No CIB!"); return FALSE; } native = cib->variant_opaque; /* if it is not blocking mode and no message in the channel, return */ if (blocking == 0 && cib_native_msgready(cib) == FALSE) { crm_debug_3("No message ready and non-blocking..."); return 0; } else if (cib_native_msgready(cib) == FALSE) { crm_debug("Waiting for message from CIB service..."); if(native->callback_channel == NULL) { return 0; } else if(native->callback_channel->ch_status != IPC_CONNECT) { return 0; } else if(native->command_channel && native->command_channel->ch_status != IPC_CONNECT){ return 0; } native->callback_channel->ops->waitin(native->callback_channel); } /* IPC_INTR is not a factor here */ msg = xmlfromIPC(native->callback_channel, 0); if (msg == NULL) { crm_warn("Received a NULL msg from CIB service."); return 0; } /* do callbacks */ type = crm_element_value(msg, F_TYPE); crm_debug_4("Activating %s callbacks...", type); if(safe_str_eq(type, T_CIB)) { cib_native_callback(cib, msg, 0, 0); } else if(safe_str_eq(type, T_CIB_NOTIFY)) { g_list_foreach(cib->notify_list, cib_native_notify, msg); } else { crm_err("Unknown message type: %s", type); } free_xml(msg); return 1; } gboolean cib_native_dispatch(IPC_Channel *channel, gpointer user_data) { int lpc = 0; cib_t *cib = user_data; cib_native_opaque_t *native = NULL; crm_debug_3("Received callback"); if(user_data == NULL){ crm_err("user_data field must contain the CIB struct"); return FALSE; } native = cib->variant_opaque; while(cib_native_msgready(cib)) { lpc++; /* invoke the callbacks but dont block */ if(cib_native_rcvmsg(cib, 0) < 1) { break; } } crm_debug_3("%d CIB messages dispatched", lpc); if(native->callback_channel && native->callback_channel->ch_status != IPC_CONNECT) { crm_crit("Lost connection to the CIB service [%d/callback].", channel->farside_pid); if(native->callback_source != NULL) { G_main_del_IPC_Channel(native->callback_source); native->callback_source = NULL; } return FALSE; } else if(native->command_channel && native->command_channel->ch_status != IPC_CONNECT) { crm_crit("Lost connection to the CIB service [%d/command].", channel->farside_pid); return FALSE; } return TRUE; } +static void +default_cib_connection_destroy(gpointer user_data) +{ + cib_t *cib = user_data; + cib->state = cib_disconnected; +} + int cib_native_set_connection_dnotify( cib_t *cib, void (*dnotify)(gpointer user_data)) { cib_native_opaque_t *native = NULL; if (cib == NULL) { crm_err("No CIB!"); return FALSE; } native = cib->variant_opaque; if(dnotify == NULL) { crm_warn("Setting dnotify back to default value"); - set_IPC_Channel_dnotify(native->callback_source, - default_ipc_connection_destroy); + set_IPC_Channel_dnotify( + native->callback_source, default_cib_connection_destroy); } else { crm_debug_3("Setting dnotify"); set_IPC_Channel_dnotify(native->callback_source, dnotify); } return cib_ok; } int cib_native_register_notification(cib_t* cib, const char *callback, int enabled) { xmlNode *notify_msg = create_xml_node(NULL, "cib-callback"); cib_native_opaque_t *native = cib->variant_opaque; - crm_xml_add(notify_msg, F_CIB_OPERATION, T_CIB_NOTIFY); - crm_xml_add(notify_msg, F_CIB_NOTIFY_TYPE, callback); - crm_xml_add_int(notify_msg, F_CIB_NOTIFY_ACTIVATE, enabled); - send_ipc_message(native->callback_channel, notify_msg); + if(cib->state != cib_disconnected) { + crm_xml_add(notify_msg, F_CIB_OPERATION, T_CIB_NOTIFY); + crm_xml_add(notify_msg, F_CIB_NOTIFY_TYPE, callback); + crm_xml_add_int(notify_msg, F_CIB_NOTIFY_ACTIVATE, enabled); + send_ipc_message(native->callback_channel, notify_msg); + } + free_xml(notify_msg); return cib_ok; } diff --git a/lib/crm/pengine/unpack.c b/lib/crm/pengine/unpack.c index 9a6fc5781c..38024480cc 100644 --- a/lib/crm/pengine/unpack.c +++ b/lib/crm/pengine/unpack.c @@ -1,1462 +1,1486 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include /* for ONLINESTATUS */ #include #include #include #include gboolean unpack_config(xmlNode *config, pe_working_set_t *data_set) { const char *name = NULL; const char *value = NULL; GHashTable *config_hash = g_hash_table_new_full( g_str_hash,g_str_equal, g_hash_destroy_str,g_hash_destroy_str); data_set->config_hash = config_hash; unpack_instance_attributes( config, XML_CIB_TAG_PROPSET, NULL, config_hash, CIB_OPTIONS_FIRST, FALSE, data_set->now); #if CRM_DEPRECATED_SINCE_2_0_1 xml_child_iter_filter( config, a_child, XML_CIB_TAG_NVPAIR, name = crm_element_value(a_child, XML_NVPAIR_ATTR_NAME); value = crm_element_value(a_child, XML_NVPAIR_ATTR_VALUE); if(g_hash_table_lookup(config_hash, name) == NULL) { g_hash_table_insert( config_hash,crm_strdup(name),crm_strdup(value)); } crm_config_err("Creating directly" "beneath has been depreciated since" " 2.0.1", ID(a_child), name); ); #else xml_child_iter_filter( config, a_child, XML_CIB_TAG_NVPAIR, name = crm_element_value(a_child, XML_NVPAIR_ATTR_NAME); crm_config_err("Creating directly" "beneath has been depreciated since" " 2.0.1 and is now disabled", ID(a_child), name); ); #endif verify_pe_options(data_set->config_hash); value = pe_pref(data_set->config_hash, "default-action-timeout"); data_set->transition_idle_timeout = crm_strdup(value); crm_debug("Default action timeout: %s", data_set->transition_idle_timeout); value = pe_pref(data_set->config_hash, "default-resource-stickiness"); data_set->default_resource_stickiness = char2score(value); crm_debug("Default stickiness: %d", data_set->default_resource_stickiness); value = pe_pref(data_set->config_hash, "stop-all-resources"); data_set->stop_everything = crm_is_true(value); crm_debug("Stop all active resources: %s", data_set->stop_everything?"true":"false"); value = pe_pref(data_set->config_hash, "default-failure-timeout"); data_set->default_failure_timeout = (crm_get_msec(value) / 1000); crm_debug("Default failure timeout: %d", data_set->default_failure_timeout); value = pe_pref(data_set->config_hash, "default-migration-threshold"); data_set->default_migration_threshold = char2score(value); crm_debug("Default migration threshold: %d", data_set->default_migration_threshold); value = pe_pref(data_set->config_hash, "stonith-enabled"); cl_str_to_boolean(value, &data_set->stonith_enabled); crm_debug("STONITH of failed nodes is %s", data_set->stonith_enabled?"enabled":"disabled"); data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action"); crm_debug_2("STONITH will %s nodes", data_set->stonith_action); value = pe_pref(data_set->config_hash, "symmetric-cluster"); cl_str_to_boolean(value, &data_set->symmetric_cluster); if(data_set->symmetric_cluster) { crm_debug("Cluster is symmetric" " - resources can run anywhere by default"); } value = pe_pref(data_set->config_hash, "no-quorum-policy"); if(safe_str_eq(value, "ignore")) { data_set->no_quorum_policy = no_quorum_ignore; } else if(safe_str_eq(value, "freeze")) { data_set->no_quorum_policy = no_quorum_freeze; } else { data_set->no_quorum_policy = no_quorum_stop; } switch (data_set->no_quorum_policy) { case no_quorum_freeze: crm_debug("On loss of CCM Quorum: Freeze resources"); break; case no_quorum_stop: crm_debug("On loss of CCM Quorum: Stop ALL resources"); break; case no_quorum_ignore: crm_notice("On loss of CCM Quorum: Ignore"); break; } value = pe_pref(data_set->config_hash, "stop-orphan-resources"); cl_str_to_boolean(value, &data_set->stop_rsc_orphans); crm_debug_2("Orphan resources are %s", data_set->stop_rsc_orphans?"stopped":"ignored"); value = pe_pref(data_set->config_hash, "stop-orphan-actions"); cl_str_to_boolean(value, &data_set->stop_action_orphans); crm_debug_2("Orphan resource actions are %s", data_set->stop_action_orphans?"stopped":"ignored"); value = pe_pref(data_set->config_hash, "remove-after-stop"); cl_str_to_boolean(value, &data_set->remove_after_stop); crm_debug_2("Stopped resources are removed from the status section: %s", data_set->remove_after_stop?"true":"false"); value = pe_pref(data_set->config_hash, "is-managed-default"); cl_str_to_boolean(value, &data_set->is_managed_default); crm_debug_2("By default resources are %smanaged", data_set->is_managed_default?"":"not "); value = pe_pref(data_set->config_hash, "start-failure-is-fatal"); cl_str_to_boolean(value, &data_set->start_failure_fatal); crm_debug_2("Start failures are %s", data_set->start_failure_fatal?"always fatal":"handled by failcount"); return TRUE; } gboolean unpack_nodes(xmlNode * xml_nodes, pe_working_set_t *data_set) { node_t *new_node = NULL; const char *id = NULL; const char *uname = NULL; const char *type = NULL; gboolean unseen_are_unclean = TRUE; const char *blind_faith = pe_pref( data_set->config_hash, "startup-fencing"); if(crm_is_true(blind_faith) == FALSE) { unseen_are_unclean = FALSE; crm_warn("Blind faith: not fencing unseen nodes"); } xml_child_iter_filter( xml_nodes, xml_obj, XML_CIB_TAG_NODE, new_node = NULL; id = crm_element_value(xml_obj, XML_ATTR_ID); uname = crm_element_value(xml_obj, XML_ATTR_UNAME); type = crm_element_value(xml_obj, XML_ATTR_TYPE); crm_debug_3("Processing node %s/%s", uname, id); if(id == NULL) { crm_config_err("Must specify id tag in "); continue; } if(type == NULL) { crm_config_err("Must specify type tag in "); continue; } if(pe_find_node(data_set->nodes, uname) != NULL) { crm_config_warn("Detected multiple node entries with uname=%s" " - this is rarely intended", uname); } crm_malloc0(new_node, sizeof(node_t)); if(new_node == NULL) { return FALSE; } new_node->weight = 0; new_node->fixed = FALSE; crm_malloc0(new_node->details, sizeof(struct node_shared_s)); if(new_node->details == NULL) { crm_free(new_node); return FALSE; } crm_debug_3("Creaing node for entry %s/%s", uname, id); new_node->details->id = id; new_node->details->uname = uname; new_node->details->type = node_ping; new_node->details->online = FALSE; new_node->details->shutdown = FALSE; new_node->details->running_rsc = NULL; new_node->details->attrs = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); /* if(data_set->have_quorum == FALSE */ /* && data_set->no_quorum_policy == no_quorum_stop) { */ /* /\* start shutting resources down *\/ */ /* new_node->weight = -INFINITY; */ /* } */ if(data_set->stonith_enabled == FALSE || unseen_are_unclean == FALSE) { /* blind faith... */ new_node->details->unclean = FALSE; } else { /* all nodes are unclean until we've seen their * status entry */ new_node->details->unclean = TRUE; } if(type == NULL || safe_str_eq(type, "member") || safe_str_eq(type, NORMALNODE)) { new_node->details->type = node_member; } add_node_attrs(xml_obj, new_node, FALSE, data_set); data_set->nodes = g_list_append(data_set->nodes, new_node); crm_debug_3("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME)); ); return TRUE; } gboolean unpack_resources(xmlNode * xml_resources, pe_working_set_t *data_set) { xml_child_iter( xml_resources, xml_obj, resource_t *new_rsc = NULL; crm_debug_3("Begining unpack... %s", xml_obj?crm_element_name(xml_obj):""); if(common_unpack(xml_obj, &new_rsc, NULL, data_set)) { data_set->resources = g_list_append( data_set->resources, new_rsc); print_resource(LOG_DEBUG_3, "Added", new_rsc, FALSE); } else { crm_config_err("Failed unpacking %s %s", crm_element_name(xml_obj), crm_element_value(xml_obj, XML_ATTR_ID)); if(new_rsc != NULL && new_rsc->fns != NULL) { new_rsc->fns->free(new_rsc); } } ); data_set->resources = g_list_sort( data_set->resources, sort_rsc_priority); return TRUE; } /* remove nodes that are down, stopping */ /* create +ve rsc_to_node constraints between resources and the nodes they are running on */ /* anything else? */ gboolean unpack_status(xmlNode * status, pe_working_set_t *data_set) { const char *id = NULL; const char *uname = NULL; xmlNode * lrm_rsc = NULL; xmlNode * attrs = NULL; node_t *this_node = NULL; crm_debug_3("Begining unpack"); xml_child_iter_filter( status, node_state, XML_CIB_TAG_STATE, id = crm_element_value(node_state, XML_ATTR_ID); uname = crm_element_value(node_state, XML_ATTR_UNAME); attrs = find_xml_node( node_state, XML_TAG_TRANSIENT_NODEATTRS, FALSE); lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE); lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE); crm_debug_3("Processing node %s", uname); this_node = pe_find_node_id(data_set->nodes, id); if(uname == NULL) { /* error */ continue; } else if(this_node == NULL) { crm_config_warn("Node %s in status section no longer exists", uname); continue; } /* Mark the node as provisionally clean * - at least we have seen it in the current cluster's lifetime */ this_node->details->unclean = FALSE; crm_debug_3("Adding runtime node attrs"); add_node_attrs(attrs, this_node, TRUE, data_set); if(crm_is_true(g_hash_table_lookup(this_node->details->attrs, "standby"))) { crm_info("Node %s is in standby-mode", this_node->details->uname); this_node->details->standby = TRUE; } crm_debug_3("determining node state"); determine_online_status(node_state, this_node, data_set); if(this_node->details->online || data_set->stonith_enabled) { /* offline nodes run no resources... * unless stonith is enabled in which case we need to * make sure rsc start events happen after the stonith */ crm_debug_3("Processing lrm resource entries"); unpack_lrm_resources(this_node, lrm_rsc, data_set); } ); return TRUE; } static gboolean determine_online_status_no_fencing(xmlNode * node_state, node_t *this_node) { gboolean online = FALSE; const char *join_state = crm_element_value(node_state, XML_CIB_ATTR_JOINSTATE); const char *crm_state = crm_element_value(node_state, XML_CIB_ATTR_CRMDSTATE); const char *ccm_state = crm_element_value(node_state, XML_CIB_ATTR_INCCM); const char *ha_state = crm_element_value(node_state, XML_CIB_ATTR_HASTATE); const char *exp_state = crm_element_value(node_state, XML_CIB_ATTR_EXPSTATE); if(ha_state == NULL) { ha_state = DEADSTATUS; } if(!crm_is_true(ccm_state) || safe_str_eq(ha_state, DEADSTATUS)){ crm_debug_2("Node is down: ha_state=%s, ccm_state=%s", crm_str(ha_state), crm_str(ccm_state)); } else if(!crm_is_true(ccm_state) || safe_str_eq(ha_state, DEADSTATUS)) { } else if(safe_str_eq(crm_state, ONLINESTATUS)) { if(safe_str_eq(join_state, CRMD_JOINSTATE_MEMBER)) { online = TRUE; } else { crm_debug("Node is not ready to run resources: %s", join_state); } } else if(this_node->details->expected_up == FALSE) { crm_debug_2("CRMd is down: ha_state=%s, ccm_state=%s", crm_str(ha_state), crm_str(ccm_state)); crm_debug_2("\tcrm_state=%s, join_state=%s, expected=%s", crm_str(crm_state), crm_str(join_state), crm_str(exp_state)); } else { /* mark it unclean */ this_node->details->unclean = TRUE; crm_warn("Node %s is partially & un-expectedly down", this_node->details->uname); crm_info("\tha_state=%s, ccm_state=%s," " crm_state=%s, join_state=%s, expected=%s", crm_str(ha_state), crm_str(ccm_state), crm_str(crm_state), crm_str(join_state), crm_str(exp_state)); } return online; } static gboolean determine_online_status_fencing(xmlNode * node_state, node_t *this_node) { gboolean online = FALSE; const char *join_state = crm_element_value(node_state, XML_CIB_ATTR_JOINSTATE); const char *crm_state = crm_element_value(node_state, XML_CIB_ATTR_CRMDSTATE); const char *ccm_state = crm_element_value(node_state, XML_CIB_ATTR_INCCM); const char *ha_state = crm_element_value(node_state, XML_CIB_ATTR_HASTATE); const char *exp_state = crm_element_value(node_state, XML_CIB_ATTR_EXPSTATE); if(ha_state == NULL) { ha_state = DEADSTATUS; } if(crm_is_true(ccm_state) && safe_str_eq(ha_state, ACTIVESTATUS) && safe_str_eq(crm_state, ONLINESTATUS)) { + + if(safe_str_eq(join_state, CRMD_JOINSTATE_MEMBER)) { online = TRUE; - if(safe_str_neq(join_state, CRMD_JOINSTATE_MEMBER)) { - crm_info("Node %s is not ready to run resources", - this_node->details->uname); - this_node->details->standby = TRUE; - } + + } else if(safe_str_eq(join_state, CRMD_JOINSTATE_PENDING)) { + crm_info("Node %s is not ready to run resources", + this_node->details->uname); + this_node->details->standby = TRUE; + this_node->details->pending = TRUE; + online = TRUE; + + } else if(safe_str_eq(join_state, CRMD_JOINSTATE_NACK)) { + crm_warn("Node %s is not part of the cluster", + this_node->details->uname); + this_node->details->standby = TRUE; + this_node->details->pending = TRUE; + online = TRUE; + + } else { + crm_warn("Node %s (%s) is un-expectedly down", + this_node->details->uname, this_node->details->id); + crm_info("\tha_state=%s, ccm_state=%s," + " crm_state=%s, join_state=%s, expected=%s", + crm_str(ha_state), crm_str(ccm_state), + crm_str(crm_state), crm_str(join_state), + crm_str(exp_state)); + this_node->details->unclean = TRUE; + + } } else if(crm_is_true(ccm_state) == FALSE && safe_str_eq(ha_state, DEADSTATUS) && safe_str_eq(crm_state, OFFLINESTATUS) && this_node->details->expected_up == FALSE) { crm_debug("Node %s is down: join_state=%s, expected=%s", this_node->details->uname, crm_str(join_state), crm_str(exp_state)); } else if(this_node->details->expected_up) { /* mark it unclean */ this_node->details->unclean = TRUE; crm_warn("Node %s (%s) is un-expectedly down", this_node->details->uname, this_node->details->id); crm_info("\tha_state=%s, ccm_state=%s," " crm_state=%s, join_state=%s, expected=%s", crm_str(ha_state), crm_str(ccm_state), crm_str(crm_state), crm_str(join_state), crm_str(exp_state)); } else { crm_info("Node %s is comming up", this_node->details->uname); crm_debug("\tha_state=%s, ccm_state=%s," " crm_state=%s, join_state=%s, expected=%s", crm_str(ha_state), crm_str(ccm_state), crm_str(crm_state), crm_str(join_state), crm_str(exp_state)); } return online; } gboolean determine_online_status( xmlNode * node_state, node_t *this_node, pe_working_set_t *data_set) { gboolean online = FALSE; const char *shutdown = crm_element_value(node_state, XML_CIB_ATTR_SHUTDOWN); const char *exp_state = crm_element_value(node_state, XML_CIB_ATTR_EXPSTATE); if(this_node == NULL) { crm_config_err("No node to check"); return online; } shutdown = crm_element_value(node_state, XML_CIB_ATTR_SHUTDOWN); this_node->details->expected_up = FALSE; if(safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) { this_node->details->expected_up = TRUE; } this_node->details->shutdown = FALSE; if(shutdown != NULL && safe_str_neq("0", shutdown)) { this_node->details->shutdown = TRUE; this_node->details->expected_up = FALSE; } if(data_set->stonith_enabled == FALSE) { online = determine_online_status_no_fencing( node_state, this_node); } else { online = determine_online_status_fencing( node_state, this_node); } if(online) { this_node->details->online = TRUE; } else { /* remove node from contention */ this_node->fixed = TRUE; this_node->weight = -INFINITY; } if(online && this_node->details->shutdown) { /* dont run resources here */ this_node->fixed = TRUE; this_node->weight = -INFINITY; } if(this_node->details->unclean) { pe_proc_warn("Node %s is unclean", this_node->details->uname); } else if(this_node->details->online) { crm_info("Node %s is %s", this_node->details->uname, this_node->details->shutdown?"shutting down": + this_node->details->pending?"pending": this_node->details->standby?"standby":"online"); } else { crm_debug_2("Node %s is offline", this_node->details->uname); } return online; } #define set_char(x) last_rsc_id[lpc] = x; complete = TRUE; static char * increment_clone(char *last_rsc_id) { int lpc = 0; int len = 0; char *tmp = NULL; gboolean complete = FALSE; CRM_CHECK(last_rsc_id != NULL, return NULL); if(last_rsc_id != NULL) { len = strlen(last_rsc_id); } lpc = len-1; while(complete == FALSE && lpc > 0) { switch (last_rsc_id[lpc]) { case 0: lpc--; break; case '0': set_char('1'); break; case '1': set_char('2'); break; case '2': set_char('3'); break; case '3': set_char('4'); break; case '4': set_char('5'); break; case '5': set_char('6'); break; case '6': set_char('7'); break; case '7': set_char('8'); break; case '8': set_char('9'); break; case '9': last_rsc_id[lpc] = '0'; lpc--; break; case ':': tmp = last_rsc_id; crm_malloc0(last_rsc_id, len + 2); memcpy(last_rsc_id, tmp, len); last_rsc_id[++lpc] = '1'; last_rsc_id[len] = '0'; last_rsc_id[len+1] = 0; complete = TRUE; crm_free(tmp); break; default: crm_err("Unexpected char: %c (%d)", last_rsc_id[lpc], lpc); break; } } return last_rsc_id; } static resource_t * create_fake_resource(const char *rsc_id, xmlNode *rsc_entry, pe_working_set_t *data_set) { resource_t *rsc = NULL; xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE); copy_in_properties(xml_rsc, rsc_entry); crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id); crm_log_xml_info(xml_rsc, "Orphan resource"); common_unpack(xml_rsc, &rsc, NULL, data_set); set_bit(rsc->flags, pe_rsc_orphan); data_set->resources = g_list_append(data_set->resources, rsc); return rsc; } static resource_t * unpack_find_resource( pe_working_set_t *data_set, node_t *node, const char *rsc_id, xmlNode *rsc_entry) { resource_t *rsc = NULL; gboolean is_duped_clone = FALSE; char *alt_rsc_id = crm_strdup(rsc_id); while(rsc == NULL) { crm_debug_3("looking for: %s", alt_rsc_id); rsc = pe_find_resource(data_set->resources, alt_rsc_id); /* no match */ if(rsc == NULL) { crm_debug_2("%s not found: %d", alt_rsc_id, is_duped_clone); if(is_duped_clone) { /* create one */ rsc = create_fake_resource(alt_rsc_id, rsc_entry, data_set); crm_info("Making sure orphan %s/%s is stopped on %s", rsc_id, rsc->id, node->details->uname); resource_location(rsc, NULL, -INFINITY, "__orphan_clone_dont_run__", data_set); } break; /* not running anywhere else */ } else if(rsc->running_on == NULL) { crm_debug_3("not active yet"); break; /* always unique */ } else if(is_set(rsc->flags, pe_rsc_unique)) { crm_debug_3("unique"); break; /* running somewhere already but we dont care * find another clone instead */ } else { crm_debug_3("find another one"); rsc = NULL; is_duped_clone = TRUE; alt_rsc_id = increment_clone(alt_rsc_id); } } crm_free(alt_rsc_id); if(rsc != NULL) { crm_free(rsc->clone_name); rsc->clone_name = NULL; if(is_duped_clone) { crm_info("Internally renamed %s on %s to %s", rsc_id, node->details->uname, rsc->id); rsc->clone_name = crm_strdup(rsc_id); } } return rsc; } static resource_t * process_orphan_resource(xmlNode *rsc_entry, node_t *node, pe_working_set_t *data_set) { resource_t *rsc = NULL; const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID); crm_log_xml_info(rsc_entry, "Orphan resource"); crm_config_warn("Nothing known about resource %s running on %s", rsc_id, node->details->uname); rsc = create_fake_resource(rsc_id, rsc_entry, data_set); if(data_set->stop_rsc_orphans == FALSE) { clear_bit(rsc->flags, pe_rsc_managed); } else { crm_info("Making sure orphan %s is stopped", rsc_id); print_resource(LOG_DEBUG_3, "Added orphan", rsc, FALSE); CRM_CHECK(rsc != NULL, return NULL); resource_location(rsc, NULL, -INFINITY, "__orphan_dont_run__", data_set); } return rsc; } static void process_rsc_state(resource_t *rsc, node_t *node, enum action_fail_response on_fail, xmlNode *migrate_op, pe_working_set_t *data_set) { if(on_fail == action_migrate_failure) { node_t *from = NULL; const char *uuid = NULL; uuid = crm_element_value(migrate_op, CRMD_ACTION_MIGRATED); from = pe_find_node_id(data_set->nodes, uuid); process_rsc_state(rsc, from, action_fail_recover,NULL,data_set); on_fail = action_fail_recover; } crm_debug_2("Resource %s is %s on %s", rsc->id, role2text(rsc->role), node->details->uname); /* process current state */ if(rsc->role != RSC_ROLE_UNKNOWN) { rsc->known_on = g_list_append(rsc->known_on, node); } if(rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) { if(on_fail != action_fail_ignore) { set_bit(rsc->flags, pe_rsc_failed); crm_debug_2("Force stop"); } native_add_running(rsc, node, data_set); if(is_set(rsc->flags, pe_rsc_managed) && rsc->stickiness != 0) { node_t *match = pe_find_node_id(rsc->allowed_nodes, node->details->id); if(match != NULL || data_set->symmetric_cluster) { resource_location(rsc, node, rsc->stickiness, "stickiness", data_set); crm_debug_2("Resource %s: preferring current location" " (node=%s, weight=%d)", rsc->id, node->details->uname, rsc->stickiness); } else { crm_debug("Ignoring stickiness for %s: the cluster is asymmetric and node %s is no longer explicitly allowed", rsc->id, node->details->uname); } } if(on_fail == action_fail_ignore) { /* nothing to do */ } else if(node->details->unclean) { stop_action(rsc, node, FALSE); } else if(on_fail == action_fail_fence) { /* treat it as if it is still running * but also mark the node as unclean */ node->details->unclean = TRUE; stop_action(rsc, node, FALSE); } else if(on_fail == action_fail_block) { /* is_managed == FALSE will prevent any * actions being sent for the resource */ clear_bit(rsc->flags, pe_rsc_managed); } else if(on_fail == action_fail_migrate) { stop_action(rsc, node, FALSE); /* make sure it comes up somewhere else * or not at all */ resource_location(rsc, node, -INFINITY, "__action_migration_auto__",data_set); } else { stop_action(rsc, node, FALSE); } } else if(rsc->clone_name) { crm_debug_2("Resetting clone_name %s for %s (stopped)", rsc->clone_name, rsc->id); crm_free(rsc->clone_name); rsc->clone_name = NULL; } else { char *key = stop_key(rsc); GListPtr possible_matches = find_actions(rsc->actions, key, node); slist_iter(stop, action_t, possible_matches, lpc, stop->optional = TRUE; ); crm_free(key); } } /* create active recurring operations as optional */ static void process_recurring(node_t *node, resource_t *rsc, int start_index, int stop_index, GListPtr sorted_op_list, pe_working_set_t *data_set) { const char *task = NULL; const char *status = NULL; crm_debug_3("%s: Start index %d, stop index = %d", rsc->id, start_index, stop_index); slist_iter(rsc_op, xmlNode, sorted_op_list, lpc, int interval = 0; char *key = NULL; const char *id = ID(rsc_op); const char *interval_s = NULL; if(node->details->online == FALSE) { crm_debug_4("Skipping %s/%s: node is offline", rsc->id, node->details->uname); break; } else if(start_index < stop_index) { crm_debug_4("Skipping %s/%s: not active", rsc->id, node->details->uname); break; } else if(lpc <= start_index) { crm_debug_4("Skipping %s/%s: old", id, node->details->uname); continue; } interval_s = crm_element_value(rsc_op,XML_LRM_ATTR_INTERVAL); interval = crm_parse_int(interval_s, "0"); if(interval == 0) { crm_debug_4("Skipping %s/%s: non-recurring", id, node->details->uname); continue; } status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS); if(safe_str_eq(status, "-1")) { crm_debug_4("Skipping %s/%s: status", id, node->details->uname); continue; } task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK); /* create the action */ key = generate_op_key(rsc->id, task, interval); crm_debug_3("Creating %s/%s", key, node->details->uname); custom_action(rsc, key, task, node, TRUE, TRUE, data_set); ); } void calculate_active_ops(GListPtr sorted_op_list, int *start_index, int *stop_index) { const char *task = NULL; const char *status = NULL; *stop_index = -1; *start_index = -1; slist_iter( rsc_op, xmlNode, sorted_op_list, lpc, task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK); status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS); if(safe_str_eq(task, CRMD_ACTION_STOP) && safe_str_eq(status, "0")) { *stop_index = lpc; } else if(safe_str_eq(task, CRMD_ACTION_START)) { *start_index = lpc; } else if(*start_index <= *stop_index && safe_str_eq(task, CRMD_ACTION_STATUS)) { const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC); if(safe_str_eq(rc, "0") || safe_str_eq(rc, "8")) { *start_index = lpc; } } ); } static void unpack_lrm_rsc_state( node_t *node, xmlNode * rsc_entry, pe_working_set_t *data_set) { int stop_index = -1; int start_index = -1; int max_call_id = -1; const char *task = NULL; const char *value = NULL; const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID); resource_t *rsc = NULL; GListPtr op_list = NULL; GListPtr sorted_op_list = NULL; xmlNode *migrate_op = NULL; enum action_fail_response on_fail = FALSE; enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN; crm_debug_3("[%s] Processing %s on %s", crm_element_name(rsc_entry), rsc_id, node->details->uname); /* extract operations */ op_list = NULL; sorted_op_list = NULL; xml_child_iter_filter( rsc_entry, rsc_op, XML_LRM_TAG_RSC_OP, op_list = g_list_append(op_list, rsc_op); ); if(op_list == NULL) { /* if there are no operations, there is nothing to do */ return; } /* find the resource */ rsc = unpack_find_resource(data_set, node, rsc_id, rsc_entry); if(rsc == NULL) { rsc = process_orphan_resource(rsc_entry, node, data_set); } CRM_ASSERT(rsc != NULL); /* process operations */ max_call_id = -1; saved_role = rsc->role; on_fail = action_fail_ignore; rsc->role = RSC_ROLE_UNKNOWN; sorted_op_list = g_list_sort(op_list, sort_op_by_callid); slist_iter( rsc_op, xmlNode, sorted_op_list, lpc, task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK); if(safe_str_eq(task, CRMD_ACTION_MIGRATED)) { migrate_op = rsc_op; } unpack_rsc_op(rsc, node, rsc_op, &max_call_id, &on_fail, data_set); ); /* create active recurring operations as optional */ calculate_active_ops(sorted_op_list, &start_index, &stop_index); process_recurring(node, rsc, start_index, stop_index, sorted_op_list, data_set); /* no need to free the contents */ g_list_free(sorted_op_list); process_rsc_state(rsc, node, on_fail, migrate_op, data_set); value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE); if(value != NULL && safe_str_neq("default", value)) { enum rsc_role_e req_role = text2role(value); if(req_role != RSC_ROLE_UNKNOWN && req_role != rsc->next_role){ if(rsc->next_role != RSC_ROLE_UNKNOWN) { crm_debug("%s: Overwriting calculated next role %s" " with requested next role %s", rsc->id, role2text(rsc->next_role), role2text(req_role)); } rsc->next_role = req_role; } } if(saved_role > rsc->role) { rsc->role = saved_role; } } gboolean unpack_lrm_resources(node_t *node, xmlNode * lrm_rsc_list, pe_working_set_t *data_set) { CRM_CHECK(node != NULL, return FALSE); crm_debug_3("Unpacking resources on %s", node->details->uname); xml_child_iter_filter( lrm_rsc_list, rsc_entry, XML_LRM_TAG_RESOURCE, unpack_lrm_rsc_state(node, rsc_entry, data_set); ); return TRUE; } gboolean unpack_rsc_op(resource_t *rsc, node_t *node, xmlNode *xml_op, int *max_call_id, enum action_fail_response *on_fail, pe_working_set_t *data_set) { const char *id = NULL; const char *task = NULL; const char *magic = NULL; const char *task_id = NULL; const char *actual_rc = NULL; /* const char *target_rc = NULL; */ const char *task_status = NULL; const char *interval_s = NULL; const char *op_digest = NULL; const char *op_version = NULL; int interval = 0; int task_id_i = -1; int task_status_i = -2; int actual_rc_i = 0; action_t *action = NULL; node_t *effective_node = NULL; gboolean expired = FALSE; gboolean is_probe = FALSE; gboolean is_stop_action = FALSE; CRM_CHECK(rsc != NULL, return FALSE); CRM_CHECK(node != NULL, return FALSE); CRM_CHECK(xml_op != NULL, return FALSE); id = ID(xml_op); task = crm_element_value(xml_op, XML_LRM_ATTR_TASK); task_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID); task_status = crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS); op_digest = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST); op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION); magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC); CRM_CHECK(id != NULL, return FALSE); CRM_CHECK(task != NULL, return FALSE); CRM_CHECK(task_status != NULL, return FALSE); task_status_i = crm_parse_int(task_status, NULL); CRM_CHECK(task_status_i <= LRM_OP_ERROR, return FALSE); CRM_CHECK(task_status_i >= LRM_OP_PENDING, return FALSE); if(safe_str_eq(task, CRMD_ACTION_NOTIFY)) { /* safe to ignore these */ return TRUE; } if(rsc->failure_timeout > 0) { int last_run = 0; if(crm_element_value_int(xml_op, "last-run", &last_run) == 0) { /* int last_change = crm_element_value_int(xml_op, "last_rc_change"); */ time_t now = get_timet_now(data_set); if(now > (last_run + rsc->failure_timeout)) { expired = TRUE; } } } crm_debug_2("Unpacking task %s/%s (call_id=%s, status=%s) on %s (role=%s)", id, task, task_id, task_status, node->details->uname, role2text(rsc->role)); interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL); interval = crm_parse_int(interval_s, "0"); if(interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) { is_probe = TRUE; } if(safe_str_eq(task, CRMD_ACTION_STOP)) { is_stop_action = TRUE; } if(task_status_i != LRM_OP_PENDING) { task_id_i = crm_parse_int(task_id, "-1"); CRM_CHECK(task_id != NULL, return FALSE); CRM_CHECK(task_id_i >= 0, return FALSE); CRM_CHECK(task_id_i > *max_call_id, return FALSE); } if(*max_call_id < task_id_i) { *max_call_id = task_id_i; } if(node->details->unclean) { crm_debug_2("Node %s (where %s is running) is unclean." " Further action depends on the value of the stop's on-fail attribue", node->details->uname, rsc->id); } actual_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC); CRM_CHECK(actual_rc != NULL, return FALSE); actual_rc_i = crm_parse_int(actual_rc, NULL); if(task_status_i == LRM_OP_NOTSUPPORTED) { actual_rc_i = EXECRA_UNIMPLEMENT_FEATURE; } if(expired && actual_rc_i != EXECRA_NOT_RUNNING && actual_rc_i != EXECRA_RUNNING_MASTER && actual_rc_i != EXECRA_OK) { crm_notice("Ignoring expired failure %s (rc=%d, magic=%s) on %s", id, actual_rc_i, magic, node->details->uname); goto done; } switch(actual_rc_i) { case EXECRA_NOT_RUNNING: if(is_probe) { /* treat these like stops */ is_stop_action = TRUE; } if(is_stop_action) { task_status_i = LRM_OP_DONE; } else { task_status_i = LRM_OP_ERROR; } break; case EXECRA_RUNNING_MASTER: if(is_probe || (rsc->role == RSC_ROLE_MASTER && safe_str_eq(task, CRMD_ACTION_STATUS))) { task_status_i = LRM_OP_DONE; } else { task_status_i = LRM_OP_ERROR; if(rsc->role != RSC_ROLE_MASTER) { /* this wil happen normally if the PE is invoked after * a resource is demoted and re-promoted but before the * 'master' monitor has been re-initiated * * The monitor will occur before the promote and appear * to be an error (the error status is be cleared by a * successful demote) */ crm_warn("%s reported %s in master mode on %s", id, rsc->id, node->details->uname); } } rsc->role = RSC_ROLE_MASTER; break; case EXECRA_FAILED_MASTER: rsc->role = RSC_ROLE_MASTER; task_status_i = LRM_OP_ERROR; break; case EXECRA_UNIMPLEMENT_FEATURE: if(interval > 0) { task_status_i = LRM_OP_ERROR; break; } /* else: fall through */ case EXECRA_INSUFFICIENT_PRIV: case EXECRA_NOT_INSTALLED: effective_node = node; /* fall through */ case EXECRA_NOT_CONFIGURED: case EXECRA_INVALID_PARAM: crm_err("Hard error - %s failed with rc=%d: Preventing %s from re-starting %s %s", id, actual_rc_i, rsc->id, effective_node?"on":"anywhere", effective_node?effective_node->details->uname:"in the cluster"); resource_location(rsc, effective_node, -INFINITY, "hard-error", data_set); if(is_probe) { /* treat these like stops */ is_stop_action = TRUE; task_status_i = LRM_OP_DONE; actual_rc_i = EXECRA_NOT_RUNNING; } else { task_status_i = LRM_OP_ERROR; } break; case EXECRA_OK: if(interval > 0 && rsc->role == RSC_ROLE_MASTER) { /* catch status ops that return 0 instead of 8 while they * are supposed to be in master mode */ task_status_i = LRM_OP_ERROR; } break; default: if(task_status_i == LRM_OP_DONE) { crm_info("Remapping %s (rc=%d) on %s to an ERROR", id, actual_rc_i, node->details->uname); task_status_i = LRM_OP_ERROR; } } if(task_status_i == LRM_OP_ERROR || task_status_i == LRM_OP_TIMEOUT || task_status_i == LRM_OP_NOTSUPPORTED) { action = custom_action(rsc, crm_strdup(id), task, NULL, TRUE, FALSE, data_set); if(expired) { crm_notice("Ignoring expired failure (calculated) %s (rc=%d, magic=%s) on %s", id, actual_rc_i, magic, node->details->uname); goto done; } else if(action->on_fail == action_fail_ignore) { crm_warn("Remapping %s (rc=%d) on %s to DONE: ignore", id, actual_rc_i, node->details->uname); task_status_i = LRM_OP_DONE; } } switch(task_status_i) { case LRM_OP_PENDING: if(safe_str_eq(task, CRMD_ACTION_START)) { set_bit(rsc->flags, pe_rsc_start_pending); rsc->role = RSC_ROLE_STARTED; } else if(safe_str_eq(task, CRMD_ACTION_PROMOTE)) { rsc->role = RSC_ROLE_MASTER; } break; case LRM_OP_DONE: crm_debug_3("%s/%s completed on %s", rsc->id, task, node->details->uname); if(is_stop_action) { rsc->role = RSC_ROLE_STOPPED; /* clear any previous failure actions */ *on_fail = action_fail_ignore; rsc->next_role = RSC_ROLE_UNKNOWN; } else if(safe_str_eq(task, CRMD_ACTION_PROMOTE)) { rsc->role = RSC_ROLE_MASTER; } else if(safe_str_eq(task, CRMD_ACTION_DEMOTE)) { rsc->role = RSC_ROLE_SLAVE; } else if(rsc->role < RSC_ROLE_STARTED) { crm_debug_3("%s active on %s", rsc->id, node->details->uname); rsc->role = RSC_ROLE_STARTED; } break; case LRM_OP_ERROR: case LRM_OP_TIMEOUT: case LRM_OP_NOTSUPPORTED: crm_warn("Processing failed op %s on %s: %s", id, node->details->uname, op_status2text(task_status_i)); crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname); add_node_copy(data_set->failed, xml_op); if(*on_fail < action->on_fail) { *on_fail = action->on_fail; } if(is_stop_action) { resource_location( rsc, node, -INFINITY, "__stop_fail__", data_set); } else if((data_set->start_failure_fatal || compare_version("2.0", op_version) > 0) && safe_str_eq(task, CRMD_ACTION_START)) { crm_warn("Compatability handling for failed op %s on %s", id, node->details->uname); resource_location( rsc, node, -INFINITY, "__legacy_start__", data_set); } if(safe_str_eq(task, CRMD_ACTION_PROMOTE)) { rsc->role = RSC_ROLE_MASTER; } else if(safe_str_eq(task, CRMD_ACTION_DEMOTE)) { /* * staying in role=master ends up putting the PE/TE into a loop * setting role=slave is not dangerous because no master will be * promoted until the failed resource has been fully stopped */ crm_warn("Forcing %s to stop after a failed demote action", rsc->id); rsc->next_role = RSC_ROLE_STOPPED; rsc->role = RSC_ROLE_SLAVE; } else if(rsc->role < RSC_ROLE_STARTED) { rsc->role = RSC_ROLE_STARTED; } crm_debug_2("Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s", rsc->id, role2text(rsc->role), node->details->unclean?"true":"false", fail2text(action->on_fail), role2text(action->fail_role)); if(action->fail_role != RSC_ROLE_STARTED && rsc->next_role < action->fail_role) { rsc->next_role = action->fail_role; } if(action->fail_role == RSC_ROLE_STOPPED) { crm_err("Making sure %s doesn't come up again", rsc->id); /* make sure it doesnt come up again */ pe_free_shallow_adv(rsc->allowed_nodes, TRUE); rsc->allowed_nodes = node_list_dup( data_set->nodes, FALSE, FALSE); slist_iter( node, node_t, rsc->allowed_nodes, lpc, node->weight = -INFINITY; ); } pe_free_action(action); action = NULL; break; case LRM_OP_CANCELLED: /* do nothing?? */ pe_err("Dont know what to do for cancelled ops yet"); break; } done: crm_debug_3("Resource %s after %s: role=%s", rsc->id, task, role2text(rsc->role)); pe_free_action(action); return TRUE; } gboolean add_node_attrs(xmlNode *xml_obj, node_t *node, gboolean overwrite, pe_working_set_t *data_set) { g_hash_table_insert(node->details->attrs, crm_strdup("#"XML_ATTR_UNAME), crm_strdup(node->details->uname)); g_hash_table_insert(node->details->attrs, crm_strdup("#"XML_ATTR_ID), crm_strdup(node->details->id)); if(safe_str_eq(node->details->id, data_set->dc_uuid)) { data_set->dc_node = node; node->details->is_dc = TRUE; g_hash_table_insert(node->details->attrs, crm_strdup("#"XML_ATTR_DC), crm_strdup(XML_BOOLEAN_TRUE)); } else { g_hash_table_insert(node->details->attrs, crm_strdup("#"XML_ATTR_DC), crm_strdup(XML_BOOLEAN_FALSE)); } unpack_instance_attributes( xml_obj, XML_TAG_ATTR_SETS, NULL, node->details->attrs, NULL, overwrite, data_set->now); return TRUE; } static GListPtr extract_operations(const char *node, const char *rsc, xmlNode *rsc_entry, gboolean active_filter) { int stop_index = -1; int start_index = -1; GListPtr op_list = NULL; GListPtr sorted_op_list = NULL; /* extract operations */ op_list = NULL; sorted_op_list = NULL; xml_child_iter_filter( rsc_entry, rsc_op, XML_LRM_TAG_RSC_OP, crm_xml_add(rsc_op, "resource", rsc); crm_xml_add(rsc_op, XML_ATTR_UNAME, node); op_list = g_list_append(op_list, rsc_op); ); if(op_list == NULL) { /* if there are no operations, there is nothing to do */ return NULL; } sorted_op_list = g_list_sort(op_list, sort_op_by_callid); /* create active recurring operations as optional */ if(active_filter == FALSE) { return sorted_op_list; } op_list = NULL; calculate_active_ops(sorted_op_list, &start_index, &stop_index); slist_iter(rsc_op, xmlNode, sorted_op_list, lpc, if(start_index < stop_index) { crm_debug_4("Skipping %s: not active", ID(rsc_entry)); break; } else if(lpc < start_index) { crm_debug_4("Skipping %s: old", ID(rsc_op)); continue; } op_list = g_list_append(op_list, rsc_op); ); g_list_free(sorted_op_list); return op_list; } GListPtr find_operations( const char *rsc, const char *node, gboolean active_filter, pe_working_set_t *data_set) { GListPtr output = NULL; GListPtr intermediate = NULL; xmlNode *tmp = NULL; xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE); const char *uname = NULL; node_t *this_node = NULL; xml_child_iter_filter( status, node_state, XML_CIB_TAG_STATE, uname = crm_element_value(node_state, XML_ATTR_UNAME); if(node != NULL && safe_str_neq(uname, node)) { continue; } this_node = pe_find_node(data_set->nodes, uname); CRM_CHECK(this_node != NULL, continue); determine_online_status(node_state, this_node, data_set); if(this_node->details->online || data_set->stonith_enabled) { /* offline nodes run no resources... * unless stonith is enabled in which case we need to * make sure rsc start events happen after the stonith */ tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE); tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE); xml_child_iter_filter( tmp, lrm_rsc, XML_LRM_TAG_RESOURCE, const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID); if(rsc != NULL && safe_str_neq(rsc_id, rsc)) { continue; } intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter); output = g_list_concat(output, intermediate); ); } ); return output; } diff --git a/lib/crm/pengine/utils.c b/lib/crm/pengine/utils.c index 7aa4d8c8fc..bbf4a99d09 100644 --- a/lib/crm/pengine/utils.c +++ b/lib/crm/pengine/utils.c @@ -1,1302 +1,1307 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include void print_str_str(gpointer key, gpointer value, gpointer user_data); gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data); void unpack_operation( action_t *action, xmlNode *xml_obj, pe_working_set_t* data_set); void pe_free_shallow(GListPtr alist) { pe_free_shallow_adv(alist, TRUE); } void pe_free_shallow_adv(GListPtr alist, gboolean with_data) { GListPtr item; GListPtr item_next = alist; if(with_data == FALSE && alist != NULL) { g_list_free(alist); return; } while(item_next != NULL) { item = item_next; item_next = item_next->next; if(with_data) { /* crm_debug_5("freeing %p", item->data); */ crm_free(item->data); } item->data = NULL; item->next = NULL; g_list_free_1(item); } } node_t * node_copy(node_t *this_node) { node_t *new_node = NULL; CRM_CHECK(this_node != NULL, return NULL); crm_malloc0(new_node, sizeof(node_t)); CRM_ASSERT(new_node != NULL); crm_debug_5("Copying %p (%s) to %p", this_node, this_node->details->uname, new_node); new_node->weight = this_node->weight; new_node->fixed = this_node->fixed; new_node->details = this_node->details; return new_node; } /* are the contents of list1 and list2 equal * nodes with weight < 0 are ignored if filter == TRUE * * slow but linear * */ gboolean node_list_eq(GListPtr list1, GListPtr list2, gboolean filter) { node_t *other_node; GListPtr lhs = list1; GListPtr rhs = list2; slist_iter( node, node_t, lhs, lpc, if(node == NULL || (filter && node->weight < 0)) { continue; } other_node = (node_t*) pe_find_node_id(rhs, node->details->id); if(other_node == NULL || other_node->weight < 0) { return FALSE; } ); lhs = list2; rhs = list1; slist_iter( node, node_t, lhs, lpc, if(node == NULL || (filter && node->weight < 0)) { continue; } other_node = (node_t*) pe_find_node_id(rhs, node->details->id); if(other_node == NULL || other_node->weight < 0) { return FALSE; } ); return TRUE; } /* any node in list1 or list2 and not in the other gets a score of -INFINITY */ GListPtr node_list_exclude(GListPtr list1, GListPtr list2) { node_t *other_node = NULL; GListPtr result = NULL; result = node_list_dup(list1, FALSE, FALSE); slist_iter( node, node_t, result, lpc, other_node = pe_find_node_id(list2, node->details->id); if(other_node == NULL) { node->weight = -INFINITY; } else { node->weight = merge_weights(node->weight, other_node->weight); } ); slist_iter( node, node_t, list2, lpc, other_node = pe_find_node_id(result, node->details->id); if(other_node == NULL) { node_t *new_node = node_copy(node); new_node->weight = -INFINITY; result = g_list_append(result, new_node); } ); return result; } /* the intersection of list1 and list2 */ GListPtr node_list_and(GListPtr list1, GListPtr list2, gboolean filter) { GListPtr result = NULL; unsigned lpc = 0; for(lpc = 0; lpc < g_list_length(list1); lpc++) { node_t *node = (node_t*)g_list_nth_data(list1, lpc); node_t *other_node = pe_find_node_id(list2, node->details->id); node_t *new_node = NULL; if(other_node != NULL) { new_node = node_copy(node); } if(new_node != NULL) { crm_debug_4("%s: %d + %d", node->details->uname, other_node->weight, new_node->weight); new_node->weight = merge_weights( new_node->weight, other_node->weight); crm_debug_3("New node weight for %s: %d", new_node->details->uname, new_node->weight); if(filter && new_node->weight < 0) { crm_free(new_node); new_node = NULL; } } if(new_node != NULL) { result = g_list_append(result, new_node); } } return result; } /* list1 - list2 */ GListPtr node_list_minus(GListPtr list1, GListPtr list2, gboolean filter) { GListPtr result = NULL; slist_iter( node, node_t, list1, lpc, node_t *other_node = pe_find_node_id(list2, node->details->id); node_t *new_node = NULL; if(node == NULL || other_node != NULL || (filter && node->weight < 0)) { continue; } new_node = node_copy(node); result = g_list_append(result, new_node); ); crm_debug_3("Minus result len: %d", g_list_length(result)); return result; } /* list1 + list2 - (intersection of list1 and list2) */ GListPtr node_list_xor(GListPtr list1, GListPtr list2, gboolean filter) { GListPtr result = NULL; slist_iter( node, node_t, list1, lpc, node_t *new_node = NULL; node_t *other_node = (node_t*) pe_find_node_id(list2, node->details->id); if(node == NULL || other_node != NULL || (filter && node->weight < 0)) { continue; } new_node = node_copy(node); result = g_list_append(result, new_node); ); slist_iter( node, node_t, list2, lpc, node_t *new_node = NULL; node_t *other_node = (node_t*) pe_find_node_id(list1, node->details->id); if(node == NULL || other_node != NULL || (filter && node->weight < 0)) { continue; } new_node = node_copy(node); result = g_list_append(result, new_node); ); crm_debug_3("Xor result len: %d", g_list_length(result)); return result; } GListPtr node_list_or(GListPtr list1, GListPtr list2, gboolean filter) { node_t *other_node = NULL; GListPtr result = NULL; gboolean needs_filter = FALSE; result = node_list_dup(list1, FALSE, filter); slist_iter( node, node_t, list2, lpc, if(node == NULL) { continue; } other_node = (node_t*)pe_find_node_id( result, node->details->id); if(other_node != NULL) { crm_debug_4("%s + %s: %d + %d", node->details->uname, other_node->details->uname, node->weight, other_node->weight); other_node->weight = merge_weights( other_node->weight, node->weight); if(filter && node->weight < 0) { needs_filter = TRUE; } } else if(filter == FALSE || node->weight >= 0) { node_t *new_node = node_copy(node); result = g_list_append(result, new_node); } ); /* not the neatest way, but the most expedient for now */ if(filter && needs_filter) { GListPtr old_result = result; result = node_list_dup(old_result, FALSE, filter); pe_free_shallow_adv(old_result, TRUE); } return result; } GListPtr node_list_dup(GListPtr list1, gboolean reset, gboolean filter) { GListPtr result = NULL; slist_iter( this_node, node_t, list1, lpc, node_t *new_node = NULL; if(filter && this_node->weight < 0) { continue; } new_node = node_copy(this_node); if(reset) { new_node->weight = 0; } if(new_node != NULL) { result = g_list_append(result, new_node); } ); return result; } void dump_node_scores(int level, resource_t *rsc, const char *comment, GListPtr nodes) { GListPtr list = nodes; if(rsc) { list = rsc->allowed_nodes; } slist_iter( node, node_t, list, lpc, if(rsc) { do_crm_log(level, "%s: %s allocation score on %s: %d", comment, rsc->id, node->details->uname, node->weight); } else { do_crm_log(level, "%s: %s = %d", comment, node->details->uname, node->weight); } ); if(rsc && rsc->children) { slist_iter( child, resource_t, rsc->children, lpc, dump_node_scores(level, child, comment, nodes); ); } } gint sort_rsc_index(gconstpointer a, gconstpointer b) { const resource_t *resource1 = (const resource_t*)a; const resource_t *resource2 = (const resource_t*)b; if(a == NULL && b == NULL) { return 0; } if(a == NULL) { return 1; } if(b == NULL) { return -1; } if(resource1->sort_index > resource2->sort_index) { return -1; } if(resource1->sort_index < resource2->sort_index) { return 1; } return 0; } gint sort_rsc_priority(gconstpointer a, gconstpointer b) { const resource_t *resource1 = (const resource_t*)a; const resource_t *resource2 = (const resource_t*)b; if(a == NULL && b == NULL) { return 0; } if(a == NULL) { return 1; } if(b == NULL) { return -1; } if(resource1->priority > resource2->priority) { return -1; } if(resource1->priority < resource2->priority) { return 1; } return 0; } action_t * custom_action(resource_t *rsc, char *key, const char *task, node_t *on_node, gboolean optional, gboolean save_action, pe_working_set_t *data_set) { action_t *action = NULL; GListPtr possible_matches = NULL; CRM_CHECK(key != NULL, return NULL); CRM_CHECK(task != NULL, return NULL); if(save_action && rsc != NULL) { possible_matches = find_actions(rsc->actions, key, on_node); } if(possible_matches != NULL) { crm_free(key); if(g_list_length(possible_matches) > 1) { pe_warn("Action %s for %s on %s exists %d times", task, rsc?rsc->id:"", on_node?on_node->details->uname:"", g_list_length(possible_matches)); } action = g_list_nth_data(possible_matches, 0); crm_debug_4("Found existing action (%d) %s for %s on %s", action->id, task, rsc?rsc->id:"", on_node?on_node->details->uname:""); g_list_free(possible_matches); } if(action == NULL) { if(save_action) { crm_debug_4("Creating%s action %d: %s for %s on %s", optional?"":" manditory", data_set->action_id, key, rsc?rsc->id:"", on_node?on_node->details->uname:""); } crm_malloc0(action, sizeof(action_t)); if(save_action) { action->id = data_set->action_id++; } else { action->id = 0; } action->rsc = rsc; CRM_ASSERT(task != NULL); action->task = crm_strdup(task); action->node = on_node; action->uuid = key; action->actions_before = NULL; action->actions_after = NULL; action->failure_is_fatal = TRUE; action->pseudo = FALSE; action->dumped = FALSE; action->runnable = TRUE; action->processed = FALSE; action->optional = optional; action->seen_count = 0; action->extra = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); action->meta = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); if(save_action) { data_set->actions = g_list_append( data_set->actions, action); } if(rsc != NULL) { action->op_entry = find_rsc_op_entry(rsc, key); unpack_operation( action, action->op_entry, data_set); if(save_action) { rsc->actions = g_list_append( rsc->actions, action); } } if(save_action) { crm_debug_4("Action %d created", action->id); } } if(optional == FALSE && action->optional) { crm_debug_2("Action %d (%s) marked manditory", action->id, action->uuid); action->optional = FALSE; } if(rsc != NULL) { enum action_tasks a_task = text2task(action->task); int warn_level = LOG_DEBUG_3; if(save_action) { warn_level = LOG_WARNING; } if(action->node != NULL && action->op_entry != NULL) { unpack_instance_attributes( action->op_entry, XML_TAG_ATTR_SETS, action->node->details->attrs, action->extra, NULL, FALSE, data_set->now); } if(action->pseudo) { /* leave untouched */ } else if(action->node == NULL) { action->runnable = FALSE; } else if(is_not_set(rsc->flags, pe_rsc_managed)) { do_crm_log(warn_level, "Action %s (unmanaged)", action->uuid); action->optional = TRUE; /* action->runnable = FALSE; */ } else if(action->node->details->online == FALSE) { action->runnable = FALSE; do_crm_log(warn_level, "Action %s on %s is unrunnable (offline)", action->uuid, action->node->details->uname); if(is_set(action->rsc->flags, pe_rsc_managed) && save_action && a_task == stop_rsc) { do_crm_log(warn_level, "Marking node %s unclean", action->node->details->uname); action->node->details->unclean = TRUE; } + } else if(action->node->details->pending) { + action->runnable = FALSE; + do_crm_log(warn_level, "Action %s on %s is unrunnable (pending)", + action->uuid, action->node->details->uname); + } else if(action->needs == rsc_req_nothing) { crm_debug_3("Action %s doesnt require anything", action->uuid); action->runnable = TRUE; #if 0 /* * No point checking this * - if we dont have quorum we cant stonith anyway */ } else if(action->needs == rsc_req_stonith) { crm_debug_3("Action %s requires only stonith", action->uuid); action->runnable = TRUE; #endif } else if(data_set->have_quorum == FALSE && data_set->no_quorum_policy == no_quorum_stop) { action->runnable = FALSE; crm_debug("%s\t%s (cancelled : quorum)", action->node->details->uname, action->uuid); } else if(data_set->have_quorum == FALSE && data_set->no_quorum_policy == no_quorum_freeze) { crm_debug_3("Check resource is already active"); if(rsc->fns->active(rsc, TRUE) == FALSE) { action->runnable = FALSE; crm_debug("%s\t%s (cancelled : quorum freeze)", action->node->details->uname, action->uuid); } } else { crm_debug_3("Action %s is runnable", action->uuid); action->runnable = TRUE; } if(save_action) { switch(a_task) { case stop_rsc: set_bit(rsc->flags, pe_rsc_stopping); break; case start_rsc: clear_bit(rsc->flags, pe_rsc_starting); if(action->runnable) { set_bit(rsc->flags, pe_rsc_starting); } break; default: break; } } } return action; } void unpack_operation( action_t *action, xmlNode *xml_obj, pe_working_set_t* data_set) { int value_i = 0; int start_delay = 0; char *value_ms = NULL; const char *class = NULL; const char *value = NULL; const char *field = NULL; CRM_CHECK(action->rsc != NULL, return); class = g_hash_table_lookup(action->rsc->meta, "class"); if(xml_obj != NULL) { value = crm_element_value(xml_obj, "prereq"); } if(value == NULL && safe_str_neq(action->task, CRMD_ACTION_START)) { /* todo: integrate stop as an option? */ action->needs = rsc_req_nothing; value = "nothing (default)"; } else if(safe_str_eq(value, "nothing")) { action->needs = rsc_req_nothing; } else if(safe_str_eq(value, "quorum")) { action->needs = rsc_req_quorum; } else if(safe_str_eq(value, "fencing")) { action->needs = rsc_req_stonith; } else if(data_set->no_quorum_policy == no_quorum_ignore || safe_str_eq(class, "stonith")) { action->needs = rsc_req_nothing; value = "nothing (default)"; } else if(data_set->no_quorum_policy == no_quorum_freeze && data_set->stonith_enabled) { action->needs = rsc_req_stonith; value = "fencing (default)"; } else { action->needs = rsc_req_quorum; value = "quorum (default)"; } if(safe_str_eq(class, "stonith")) { if(action->needs == rsc_req_stonith) { crm_config_err("Stonith resources (eg. %s) cannot require" " fencing to start", action->rsc->id); } action->needs = rsc_req_nothing; value = "nothing (fencing override)"; } crm_debug_3("\tAction %s requires: %s", action->task, value); value = NULL; if(xml_obj != NULL) { value = crm_element_value(xml_obj, XML_OP_ATTR_ON_FAIL); } if(value == NULL) { } else if(safe_str_eq(value, "block")) { action->on_fail = action_fail_block; } else if(safe_str_eq(value, "fence")) { action->on_fail = action_fail_fence; value = "node fencing"; if(data_set->stonith_enabled == FALSE) { crm_config_err("Specifying on_fail=fence and" " stonith-enabled=false makes no sense"); action->on_fail = action_fail_stop; action->fail_role = RSC_ROLE_STOPPED; value = "stop resource"; } } else if(safe_str_eq(value, "ignore") || safe_str_eq(value, "nothing")) { action->on_fail = action_fail_ignore; value = "ignore"; } else if(safe_str_eq(value, "migrate")) { action->on_fail = action_fail_migrate; value = "force migration"; } else if(safe_str_eq(value, "stop")) { action->on_fail = action_fail_stop; action->fail_role = RSC_ROLE_STOPPED; value = "stop resource"; } else if(safe_str_eq(value, "restart")) { action->on_fail = action_fail_recover; value = "restart (and possibly migrate)"; } else { pe_err("Resource %s: Unknown failure type (%s)", action->rsc->id, value); value = NULL; } /* defaults */ if(value == NULL && safe_str_eq(action->task, CRMD_ACTION_STOP)) { if(data_set->stonith_enabled) { action->on_fail = action_fail_fence; value = "resource fence (default)"; } else { action->on_fail = action_fail_block; value = "resource block (default)"; } } else if(value == NULL && safe_str_eq(action->task, CRMD_ACTION_MIGRATED)) { action->on_fail = action_migrate_failure; value = "atomic migration recovery (default)"; } else if(value == NULL) { action->on_fail = action_fail_recover; value = "restart (and possibly migrate) (default)"; } crm_debug_3("\t%s failure handling: %s", action->task, value); value = NULL; if(xml_obj != NULL) { value = crm_element_value(xml_obj, "role_after_failure"); } if(value != NULL && action->fail_role == RSC_ROLE_UNKNOWN) { action->fail_role = text2role(value); } /* defaults */ if(action->fail_role == RSC_ROLE_UNKNOWN) { if(safe_str_eq(action->task, CRMD_ACTION_PROMOTE)) { action->fail_role = RSC_ROLE_SLAVE; } else { action->fail_role = RSC_ROLE_STARTED; } } crm_debug_3("\t%s failure results in: %s", action->task, role2text(action->fail_role)); if(xml_obj != NULL) { xml_prop_iter(xml_obj, p_name, p_value, if(p_value != NULL) { g_hash_table_insert(action->meta, crm_strdup(p_name), crm_strdup(p_value)); } ); g_hash_table_remove(action->meta, "id"); unpack_instance_attributes(xml_obj, XML_TAG_META_SETS, NULL, action->meta, NULL, FALSE, data_set->now); unpack_instance_attributes(xml_obj, XML_TAG_ATTR_SETS, NULL, action->meta, NULL, FALSE, data_set->now); } field = XML_LRM_ATTR_INTERVAL; value = g_hash_table_lookup(action->meta, field); if(value != NULL) { value_i = crm_get_msec(value); CRM_CHECK(value_i >= 0, value_i = 0); value_ms = crm_itoa(value_i); if(value_i > 0) { g_hash_table_replace(action->meta, crm_strdup(field), value_ms); } else { g_hash_table_remove(action->meta, field); } } field = XML_OP_ATTR_START_DELAY; value = g_hash_table_lookup(action->meta, field); if(value != NULL) { value_i = crm_get_msec(value); if(value_i < 0) { value_i = 0; } start_delay = value_i; value_ms = crm_itoa(value_i); g_hash_table_replace(action->meta, crm_strdup(field), value_ms); } field = XML_ATTR_TIMEOUT; value = g_hash_table_lookup(action->meta, field); if(value == NULL) { value = pe_pref( data_set->config_hash, "default-action-timeout"); } value_i = crm_get_msec(value); if(value_i < 0) { value_i = 0; } value_i += start_delay; value_ms = crm_itoa(value_i); g_hash_table_replace(action->meta, crm_strdup(field), value_ms); } xmlNode * find_rsc_op_entry(resource_t *rsc, const char *key) { int number = 0; const char *name = NULL; const char *value = NULL; const char *interval = NULL; char *match_key = NULL; xmlNode *op = NULL; xml_child_iter_filter( rsc->ops_xml, operation, "op", name = crm_element_value(operation, "name"); interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL); value = crm_element_value(operation, "disabled"); if(crm_is_true(value)) { crm_debug_2("%s disabled", ID(operation)); continue; } number = crm_get_msec(interval); if(number < 0) { continue; } match_key = generate_op_key(rsc->id, name, number); if(safe_str_eq(key, match_key)) { op = operation; } crm_free(match_key); if(op != NULL) { return op; } ); crm_debug_3("No match for %s", key); return op; } void print_node(const char *pre_text, node_t *node, gboolean details) { if(node == NULL) { crm_debug_4("%s%s: ", pre_text==NULL?"":pre_text, pre_text==NULL?"":": "); return; } crm_debug_4("%s%s%sNode %s: (weight=%d, fixed=%s)", pre_text==NULL?"":pre_text, pre_text==NULL?"":": ", node->details==NULL?"error ":node->details->online?"":"Unavailable/Unclean ", node->details->uname, node->weight, node->fixed?"True":"False"); if(details && node != NULL && node->details != NULL) { char *pe_mutable = crm_strdup("\t\t"); crm_debug_4("\t\t===Node Attributes"); g_hash_table_foreach(node->details->attrs, print_str_str, pe_mutable); crm_free(pe_mutable); crm_debug_4("\t\t=== Resources"); slist_iter( rsc, resource_t, node->details->running_rsc, lpc, print_resource(LOG_DEBUG_4, "\t\t", rsc, FALSE); ); } } /* * Used by the HashTable for-loop */ void print_str_str(gpointer key, gpointer value, gpointer user_data) { crm_debug_4("%s%s %s ==> %s", user_data==NULL?"":(char*)user_data, user_data==NULL?"":": ", (char*)key, (char*)value); } void print_resource( int log_level, const char *pre_text, resource_t *rsc, gboolean details) { long options = pe_print_log; if(rsc == NULL) { do_crm_log(log_level-1, "%s%s: ", pre_text==NULL?"":pre_text, pre_text==NULL?"":": "); return; } if(details) { options |= pe_print_details; } rsc->fns->print(rsc, pre_text, options, &log_level); } void pe_free_action(action_t *action) { if(action == NULL) { return; } pe_free_shallow(action->actions_before);/* action_warpper_t* */ pe_free_shallow(action->actions_after); /* action_warpper_t* */ g_hash_table_destroy(action->extra); g_hash_table_destroy(action->meta); crm_free(action->task); crm_free(action->uuid); crm_free(action); } GListPtr find_recurring_actions(GListPtr input, node_t *not_on_node) { const char *value = NULL; GListPtr result = NULL; CRM_CHECK(input != NULL, return NULL); slist_iter( action, action_t, input, lpc, value = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL); if(value == NULL) { /* skip */ } else if(safe_str_eq(value, "0")) { /* skip */ } else if(safe_str_eq(CRMD_ACTION_CANCEL, action->task)) { /* skip */ } else if(not_on_node == NULL) { crm_debug_5("(null) Found: %s", action->uuid); result = g_list_append(result, action); } else if(action->node == NULL) { /* skip */ } else if(action->node->details != not_on_node->details) { crm_debug_5("Found: %s", action->uuid); result = g_list_append(result, action); } ); return result; } GListPtr find_actions(GListPtr input, const char *key, node_t *on_node) { GListPtr result = NULL; CRM_CHECK(key != NULL, return NULL); slist_iter( action, action_t, input, lpc, crm_debug_5("Matching %s against %s", key, action->uuid); if(safe_str_neq(key, action->uuid)) { continue; } else if(on_node == NULL) { result = g_list_append(result, action); } else if(action->node == NULL) { /* skip */ crm_debug_2("While looking for %s action on %s, " "found an unallocated one. Assigning" " it to the requested node...", key, on_node->details->uname); action->node = on_node; result = g_list_append(result, action); } else if(safe_str_eq(on_node->details->id, action->node->details->id)) { result = g_list_append(result, action); } ); return result; } GListPtr find_actions_exact(GListPtr input, const char *key, node_t *on_node) { GListPtr result = NULL; CRM_CHECK(key != NULL, return NULL); slist_iter( action, action_t, input, lpc, crm_debug_5("Matching %s against %s", key, action->uuid); if(safe_str_neq(key, action->uuid)) { crm_debug_3("Key mismatch: %s vs. %s", key, action->uuid); continue; } else if(on_node == NULL || action->node == NULL) { crm_debug_3("on_node=%p, action->node=%p", on_node, action->node); continue; } else if(safe_str_eq(on_node->details->id, action->node->details->id)) { result = g_list_append(result, action); } crm_debug_2("Node mismatch: %s vs. %s", on_node->details->id, action->node->details->id); ); return result; } void set_id(xmlNode * xml_obj, const char *prefix, int child) { int id_len = 0; gboolean use_prefix = TRUE; gboolean use_child = TRUE; char *new_id = NULL; const char *id = crm_element_value(xml_obj, XML_ATTR_ID); id_len = 1 + strlen(id); if(child > 999) { pe_err("Are you insane?!?" " The CRM does not support > 1000 children per resource"); return; } else if(child < 0) { use_child = FALSE; } else { id_len += 4; /* child */ } if(prefix == NULL || safe_str_eq(id, prefix)) { use_prefix = FALSE; } else { id_len += (1 + strlen(prefix)); } crm_malloc0(new_id, id_len); if(use_child) { snprintf(new_id, id_len, "%s%s%s:%d", use_prefix?prefix:"", use_prefix?":":"", id, child); } else { snprintf(new_id, id_len, "%s%s%s", use_prefix?prefix:"", use_prefix?":":"", id); } crm_xml_add(xml_obj, XML_ATTR_ID, new_id); crm_free(new_id); } static void resource_node_score(resource_t *rsc, node_t *node, int score, const char *tag) { node_t *match = NULL; if(rsc->children) { slist_iter( child_rsc, resource_t, rsc->children, lpc, resource_node_score(child_rsc, node, score, tag); ); } crm_debug_2("Setting %s for %s on %s: %d", tag, rsc->id, node->details->uname, score); match = pe_find_node_id(rsc->allowed_nodes, node->details->id); if(match == NULL) { match = node_copy(node); match->weight = 0; rsc->allowed_nodes = g_list_append(rsc->allowed_nodes, match); } match->weight = merge_weights(match->weight, score); } void resource_location(resource_t *rsc, node_t *node, int score, const char *tag, pe_working_set_t *data_set) { if(node != NULL) { resource_node_score(rsc, node, score, tag); } else if(data_set != NULL) { slist_iter( node, node_t, data_set->nodes, lpc, resource_node_score(rsc, node, score, tag); ); } else { slist_iter( node, node_t, rsc->allowed_nodes, lpc, resource_node_score(rsc, node, score, tag); ); } if(node == NULL && score == -INFINITY) { if(rsc->allocated_to) { crm_info("Deallocating %s from %s", rsc->id, rsc->allocated_to->details->uname); crm_free(rsc->allocated_to); rsc->allocated_to = NULL; } } } #define sort_return(an_int) crm_free(a_uuid); crm_free(b_uuid); return an_int gint sort_op_by_callid(gconstpointer a, gconstpointer b) { char *a_uuid = NULL; char *b_uuid = NULL; const xmlNode *xml_a = a; const xmlNode *xml_b = b; const char *a_xml_id = crm_element_value_const(xml_a, XML_ATTR_ID); const char *b_xml_id = crm_element_value_const(xml_b, XML_ATTR_ID); const char *a_task_id = crm_element_value_const(xml_a, XML_LRM_ATTR_CALLID); const char *b_task_id = crm_element_value_const(xml_b, XML_LRM_ATTR_CALLID); const char *a_key = crm_element_value_const(xml_a, XML_ATTR_TRANSITION_MAGIC); const char *b_key = crm_element_value_const(xml_b, XML_ATTR_TRANSITION_MAGIC); int dummy = -1; int a_id = -1; int b_id = -1; int a_rc = -1; int b_rc = -1; int a_status = -1; int b_status = -1; int a_call_id = -1; int b_call_id = -1; if(safe_str_eq(a_xml_id, b_xml_id)) { /* We have duplicate lrm_rsc_op entries in the status * section which is unliklely to be a good thing * - we can handle it easily enough, but we need to get * to the bottom of why its happening. */ pe_err("Duplicate lrm_rsc_op entries named %s", a_xml_id); sort_return(0); } CRM_CHECK(a_task_id != NULL && b_task_id != NULL, crm_err("a: %s, b: %s", crm_str(a_xml_id), crm_str(b_xml_id)); sort_return(0)); a_call_id = crm_parse_int(a_task_id, NULL); b_call_id = crm_parse_int(b_task_id, NULL); if(a_call_id == -1 && b_call_id == -1) { /* both are pending ops so it doesnt matter since * stops are never pending */ sort_return(0); } else if(a_call_id >= 0 && a_call_id < b_call_id) { crm_debug_4("%s (%d) < %s (%d) : call id", a_xml_id, a_call_id, b_xml_id, b_call_id); sort_return(-1); } else if(b_call_id >= 0 && a_call_id > b_call_id) { crm_debug_4("%s (%d) > %s (%d) : call id", a_xml_id, a_call_id, b_xml_id, b_call_id); sort_return(1); } crm_debug_5("%s (%d) == %s (%d) : continuing", a_xml_id, a_call_id, b_xml_id, b_call_id); /* now process pending ops */ CRM_CHECK(a_key != NULL && b_key != NULL, sort_return(0)); CRM_CHECK(decode_transition_magic( a_key, &a_uuid, &a_id, &dummy, &a_status, &a_rc, &dummy), sort_return(0)); CRM_CHECK(decode_transition_magic( b_key, &b_uuid, &b_id, &dummy, &b_status, &b_rc, &dummy), sort_return(0)); /* try and determin the relative age of the operation... * some pending operations (ie. a start) may have been supuerceeded * by a subsequent stop * * [a|b]_id == -1 means its a shutdown operation and _always_ comes last */ if(safe_str_neq(a_uuid, b_uuid) || a_id == b_id) { /* * some of the logic in here may be redundant... * * if the UUID from the TE doesnt match then one better * be a pending operation. * pending operations dont survive between elections and joins * because we query the LRM directly */ CRM_CHECK(a_call_id == -1 || b_call_id == -1, crm_err("a: %s=%d, b: %s=%d", crm_str(a_xml_id), a_call_id, crm_str(b_xml_id), b_call_id); sort_return(0)); CRM_CHECK(a_call_id >= 0 || b_call_id >= 0, sort_return(0)); if(b_call_id == -1) { crm_debug_2("%s (%d) < %s (%d) : transition + call id", a_xml_id, a_call_id, b_xml_id, b_call_id); sort_return(-1); } if(a_call_id == -1) { crm_debug_2("%s (%d) > %s (%d) : transition + call id", a_xml_id, a_call_id, b_xml_id, b_call_id); sort_return(1); } } else if((a_id >= 0 && a_id < b_id) || b_id == -1) { crm_debug_3("%s (%d) < %s (%d) : transition", a_xml_id, a_id, b_xml_id, b_id); sort_return(-1); } else if((b_id >= 0 && a_id > b_id) || a_id == -1) { crm_debug_3("%s (%d) > %s (%d) : transition", a_xml_id, a_id, b_xml_id, b_id); sort_return(1); } /* we should never end up here */ crm_err("%s (%d:%d:%s) ?? %s (%d:%d:%s) : default", a_xml_id, a_call_id, a_id, a_uuid, b_xml_id, b_call_id, b_id, b_uuid); CRM_CHECK(FALSE, sort_return(0)); } time_t get_timet_now(pe_working_set_t *data_set) { time_t now = 0; if(data_set && data_set->now) { now = data_set->now->tm_now; } if(now == 0) { /* eventually we should convert data_set->now into time_tm * for now, its only triggered by PE regression tests */ now = time(NULL); crm_crit("Defaulting to 'now'"); if(data_set && data_set->now) { data_set->now->tm_now = now; } } return now; } int get_failcount(node_t *node, resource_t *rsc, int *last_failure, pe_working_set_t *data_set) { int last = 0; int fail_count = 0; resource_t *failed = rsc; char *fail_attr = crm_concat("fail-count", rsc->id, '-'); const char *value = g_hash_table_lookup(node->details->attrs, fail_attr); if(is_not_set(rsc->flags, pe_rsc_unique)) { failed = uber_parent(rsc); } if(value != NULL) { fail_count = char2score(value); crm_info("%s has failed %d times on %s", rsc->id, fail_count, node->details->uname); } crm_free(fail_attr); fail_attr = crm_concat("last-failure", rsc->id, '-'); value = g_hash_table_lookup(node->details->attrs, fail_attr); if(value != NULL && rsc->failure_timeout) { last = crm_parse_int(value, NULL); if(last_failure) { *last_failure = last; } if(last > 0) { time_t now = get_timet_now(data_set); if(now > (last + rsc->failure_timeout)) { crm_notice("Failcount for %s on %s has expired (limit was %ds)", failed->id, node->details->uname, rsc->failure_timeout); fail_count = 0; } } } crm_free(fail_attr); return fail_count; }