diff --git a/cib/cibio.h b/cib/cibio.h index e082bfdcc5..2c53abdf04 100644 --- a/cib/cibio.h +++ b/cib/cibio.h @@ -1,52 +1,55 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef CIB_IO__H # define CIB_IO__H # include # include # include # include # include # include # include +# include extern gboolean initialized; extern xmlNode *the_cib; extern xmlNode *node_search; extern xmlNode *resource_search; extern xmlNode *constraint_search; extern xmlNode *status_search; extern xmlNode *get_the_CIB(void); extern int initializeCib(xmlNode * cib); extern gboolean uninitializeCib(void); extern xmlNode *createEmptyCib(void); extern gboolean verifyCibXml(xmlNode * cib); extern xmlNode *readCibXml(char *buffer); extern xmlNode *readCibXmlFile(const char *dir, const char *file, gboolean discard_status); extern int activateCibBuffer(char *buffer, const char *filename); extern int activateCibXml(xmlNode * doc, gboolean to_disk, const char *op); +extern crm_trigger_t *cib_writer; +extern gboolean cib_writes_enabled; /* extern xmlNode *server_get_cib_copy(void); */ #endif diff --git a/cib/io.c b/cib/io.c index db2872b120..1d3fab0d08 100644 --- a/cib/io.c +++ b/cib/io.c @@ -1,696 +1,727 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include +#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define CIB_SERIES "cib" extern const char *cib_root; static int cib_wrap = 100; #define CIB_WRITE_PARANOIA 0 const char *local_resource_path[] = { XML_CIB_TAG_STATUS, }; const char *resource_path[] = { XML_CIB_TAG_RESOURCES, }; const char *node_path[] = { XML_CIB_TAG_NODES, }; const char *constraint_path[] = { XML_CIB_TAG_CONSTRAINTS, }; +crm_trigger_t *cib_writer = NULL; gboolean initialized = FALSE; xmlNode *node_search = NULL; xmlNode *resource_search = NULL; xmlNode *constraint_search = NULL; xmlNode *status_search = NULL; -extern gboolean cib_writes_enabled; -extern GTRIGSource *cib_writer; extern enum cib_errors cib_status; int set_connected_peers(xmlNode * xml_obj); void GHFunc_count_peers(gpointer key, gpointer value, gpointer user_data); int write_cib_contents(gpointer p); extern void cib_cleanup(void); static gboolean validate_cib_digest(xmlNode * local_cib, const char *sigfile) { int s_res = -1; struct stat buf; char *digest = NULL; char *expected = NULL; gboolean passed = FALSE; FILE *expected_strm = NULL; int start = 0, length = 0, read_len = 0; CRM_ASSERT(sigfile != NULL); s_res = stat(sigfile, &buf); if (s_res != 0) { crm_warn("No on-disk digest present"); return TRUE; } if (local_cib != NULL) { digest = calculate_on_disk_digest(local_cib); } expected_strm = fopen(sigfile, "r"); if (expected_strm == NULL) { crm_perror(LOG_ERR, "Could not open signature file %s for reading", sigfile); goto bail; } start = ftell(expected_strm); fseek(expected_strm, 0L, SEEK_END); length = ftell(expected_strm); fseek(expected_strm, 0L, start); CRM_ASSERT(length >= 0); CRM_ASSERT(start == ftell(expected_strm)); if (length > 0) { crm_trace("Reading %d bytes from file", length); crm_malloc0(expected, (length + 1)); read_len = fread(expected, 1, length, expected_strm); /* Coverity: False positive */ CRM_ASSERT(read_len == length); } fclose(expected_strm); bail: if (expected == NULL) { crm_err("On-disk digest is empty"); } else if (safe_str_eq(expected, digest)) { crm_trace("Digest comparision passed: %s", digest); passed = TRUE; } else { crm_err("Digest comparision failed: expected %s (%s), calculated %s", expected, sigfile, digest); } crm_free(digest); crm_free(expected); return passed; } static int write_cib_digest(xmlNode * local_cib, const char *digest_file, char *digest) { int rc = 0; char *local_digest = NULL; FILE *digest_strm = fopen(digest_file, "w"); if (digest_strm == NULL) { crm_perror(LOG_ERR, "Cannot open signature file %s for writing", digest_file); return -1; } if (digest == NULL) { local_digest = calculate_on_disk_digest(local_cib); CRM_ASSERT(digest != NULL); digest = local_digest; } rc = fprintf(digest_strm, "%s", digest); if (rc < 0) { crm_perror(LOG_ERR, "Cannot write to signature file %s", digest_file); } CRM_ASSERT(digest_strm != NULL); if (fflush(digest_strm) != 0) { crm_perror(LOG_ERR, "Couldnt flush the contents of %s", digest_file); rc = -1; } if (fsync(fileno(digest_strm)) < 0) { crm_perror(LOG_ERR, "Couldnt sync the contents of %s", digest_file); rc = -1; } fclose(digest_strm); crm_free(local_digest); return rc; } static gboolean validate_on_disk_cib(const char *filename, xmlNode ** on_disk_cib) { int s_res = -1; struct stat buf; gboolean passed = TRUE; xmlNode *root = NULL; CRM_ASSERT(filename != NULL); s_res = stat(filename, &buf); if (s_res == 0) { char *sigfile = NULL; size_t fnsize; crm_trace("Reading cluster configuration from: %s", filename); root = filename2xml(filename); fnsize = strlen(filename) + 5; crm_malloc0(sigfile, fnsize); snprintf(sigfile, fnsize, "%s.sig", filename); if (validate_cib_digest(root, sigfile) == FALSE) { passed = FALSE; } crm_free(sigfile); } if (on_disk_cib != NULL) { *on_disk_cib = root; } else { free_xml(root); } return passed; } static int cib_rename(const char *old, const char *new) { int rc = 0; char *automatic = NULL; if (new == NULL) { automatic = crm_concat(cib_root, "cib.auto.XXXXXX", '/'); automatic = mktemp(automatic); new = automatic; crm_err("Archiving corrupt or unusable file %s as %s", old, automatic); } rc = rename(old, new); if (rc < 0) { crm_perror(LOG_ERR, "Couldn't rename %s as %s - Disabling disk writes and continuing", old, new); cib_writes_enabled = FALSE; } crm_free(automatic); return rc; } /* * It is the callers responsibility to free the output of this function */ static xmlNode * retrieveCib(const char *filename, const char *sigfile, gboolean archive_invalid) { struct stat buf; xmlNode *root = NULL; crm_info("Reading cluster configuration from: %s (digest: %s)", filename, sigfile); if (stat(filename, &buf) != 0) { crm_warn("Cluster configuration not found: %s", filename); return NULL; } root = filename2xml(filename); if (root == NULL) { crm_err("%s exists but does NOT contain valid XML. ", filename); crm_warn("Continuing but %s will NOT used.", filename); } else if (validate_cib_digest(root, sigfile) == FALSE) { crm_err("Checksum of %s failed! Configuration contents ignored!", filename); crm_err("Usually this is caused by manual changes, " "please refer to http://clusterlabs.org/wiki/FAQ#cib_changes_detected"); crm_warn("Continuing but %s will NOT used.", filename); free_xml(root); root = NULL; if (archive_invalid) { /* Archive the original files so the contents are not lost */ cib_rename(filename, NULL); cib_rename(sigfile, NULL); } } return root; } xmlNode * readCibXmlFile(const char *dir, const char *file, gboolean discard_status) { int seq = 0; char *backup_file = NULL; char *filename = NULL, *sigfile = NULL; const char *name = NULL; const char *value = NULL; const char *validation = NULL; const char *use_valgrind = getenv("HA_VALGRIND_ENABLED"); xmlNode *root = NULL; xmlNode *status = NULL; if (!crm_is_writable(dir, file, CRM_DAEMON_USER, NULL, FALSE)) { cib_status = cib_bad_permissions; return NULL; } filename = crm_concat(dir, file, '/'); sigfile = crm_concat(filename, "sig", '.'); cib_status = cib_ok; root = retrieveCib(filename, sigfile, TRUE); if (root == NULL) { crm_warn("Primary configuration corrupt or unusable, trying backup..."); seq = get_last_sequence(cib_root, CIB_SERIES); } while (root == NULL) { struct stat buf; crm_free(sigfile); if (seq == 0) { seq += cib_wrap; /* unwrap */ } backup_file = generate_series_filename(cib_root, CIB_SERIES, seq - 1, FALSE); sigfile = crm_concat(filename, "sig", '.'); if (stat(backup_file, &buf) != 0) { crm_debug("Backup file %s not found", backup_file); break; } crm_warn("Attempting to load: %s", backup_file); root = retrieveCib(backup_file, sigfile, FALSE); seq--; } crm_free(backup_file); if (root == NULL) { root = createEmptyCib(); crm_xml_add(root, XML_ATTR_GENERATION, "0"); crm_xml_add(root, XML_ATTR_NUMUPDATES, "0"); crm_xml_add(root, XML_ATTR_GENERATION_ADMIN, "0"); crm_xml_add(root, XML_ATTR_VALIDATION, LATEST_SCHEMA_VERSION); crm_warn("Continuing with an empty configuration."); } if (cib_writes_enabled && use_valgrind) { if (crm_is_true(use_valgrind) || strstr(use_valgrind, "cib")) { cib_writes_enabled = FALSE; crm_err("*********************************************************"); crm_err("*** Disabling disk writes to avoid confusing Valgrind ***"); crm_err("*********************************************************"); } } status = find_xml_node(root, XML_CIB_TAG_STATUS, FALSE); if (discard_status && status != NULL) { /* strip out the status section if there is one */ free_xml_from_parent(root, status); status = NULL; } if (status == NULL) { create_xml_node(root, XML_CIB_TAG_STATUS); } /* Do this before DTD validation happens */ /* fill in some defaults */ name = XML_ATTR_GENERATION_ADMIN; value = crm_element_value(root, name); if (value == NULL) { crm_warn("No value for %s was specified in the configuration.", name); crm_warn("The reccomended course of action is to shutdown," " run crm_verify and fix any errors it reports."); crm_warn("We will default to zero and continue but may get" " confused about which configuration to use if" " multiple nodes are powered up at the same time."); crm_xml_add_int(root, name, 0); } name = XML_ATTR_GENERATION; value = crm_element_value(root, name); if (value == NULL) { crm_xml_add_int(root, name, 0); } name = XML_ATTR_NUMUPDATES; value = crm_element_value(root, name); if (value == NULL) { crm_xml_add_int(root, name, 0); } /* unset these and require the DC/CCM to update as needed */ xml_remove_prop(root, XML_ATTR_DC_UUID); if (discard_status) { crm_log_xml_trace(root, "[on-disk]"); } validation = crm_element_value(root, XML_ATTR_VALIDATION); if (validate_xml(root, NULL, TRUE) == FALSE) { crm_err("CIB does not validate with %s", crm_str(validation)); cib_status = cib_dtd_validation; } else if (validation == NULL) { int version = 0; update_validation(&root, &version, FALSE, FALSE); if (version > 0) { crm_notice("Enabling %s validation on" " the existing (sane) configuration", get_schema_name(version)); } else { crm_err("CIB does not validate with any known DTD or schema"); cib_status = cib_dtd_validation; } } crm_free(filename); crm_free(sigfile); return root; } /* * The caller should never free the return value */ xmlNode * get_the_CIB(void) { return the_cib; } gboolean uninitializeCib(void) { xmlNode *tmp_cib = the_cib; if (tmp_cib == NULL) { crm_debug("The CIB has already been deallocated."); return FALSE; } initialized = FALSE; the_cib = NULL; node_search = NULL; resource_search = NULL; constraint_search = NULL; status_search = NULL; crm_debug("Deallocating the CIB."); free_xml(tmp_cib); crm_debug("The CIB has been deallocated."); return TRUE; } /* * This method will not free the old CIB pointer or the new one. * We rely on the caller to have saved a pointer to the old CIB * and to free the old/bad one depending on what is appropriate. */ gboolean initializeCib(xmlNode * new_cib) { if (new_cib == NULL) { return FALSE; } the_cib = new_cib; initialized = TRUE; return TRUE; } static void sync_directory(const char *name) { int fd = 0; DIR *directory = NULL; directory = opendir(name); if (directory == NULL) { crm_perror(LOG_ERR, "Could not open %s for syncing", name); return; } fd = dirfd(directory); if (fd < 0) { crm_perror(LOG_ERR, "Could not obtain file descriptor for %s", name); } else if (fsync(fd) < 0) { crm_perror(LOG_ERR, "Could not sync %s", name); } if (closedir(directory) < 0) { crm_perror(LOG_ERR, "Could not close %s after fsync", name); } } /* * This method will free the old CIB pointer on success and the new one * on failure. */ int activateCibXml(xmlNode * new_cib, gboolean to_disk, const char *op) { xmlNode *saved_cib = the_cib; CRM_ASSERT(new_cib != saved_cib); if (initializeCib(new_cib) == FALSE) { free_xml(new_cib); crm_err("Ignoring invalid or NULL CIB"); if (saved_cib != NULL) { crm_warn("Reverting to last known CIB"); if (initializeCib(saved_cib) == FALSE) { /* oh we are so dead */ crm_crit("Couldn't re-initialize the old CIB!"); cl_flush_logs(); exit(1); } } else { crm_crit("Could not write out new CIB and no saved" " version to revert to"); } return cib_ACTIVATION; } free_xml(saved_cib); if (cib_writes_enabled && cib_status == cib_ok && to_disk) { crm_debug("Triggering CIB write for %s op", op); - G_main_set_trigger(cib_writer); + mainloop_set_trigger(cib_writer); } return cib_ok; } +static void cib_diskwrite_complete(GPid pid, gint status, gpointer user_data) +{ + int exitcode = -1; + + if(WIFSIGNALED(status)) { + int signo = WTERMSIG(status); + int core = WCOREDUMP(status); + crm_notice("Disk write process terminated with signal %d (pid=%d, core=%d)", signo, pid, core); + + } else if(WIFEXITED(status)) { + exitcode = WEXITSTATUS(status); + do_crm_log(exitcode == 0 ? LOG_TRACE : LOG_ERR, "Disk write process exited (pid=%d, rc=%d)", pid, exitcode); + } + + if(exitcode != 0 && cib_writes_enabled) { + crm_err("Disabling disk writes after write failure"); + cib_writes_enabled = FALSE; + } +} + int write_cib_contents(gpointer p) { int exit_rc = LSB_EXIT_OK; gboolean need_archive = FALSE; struct stat buf; char *digest = NULL; xmlNode *cib_status_root = NULL; xmlNode *local_cib = NULL; char *tmp1 = NULL; char *tmp2 = NULL; char *digest_file = NULL; char *primary_file = NULL; char *backup_file = NULL; char *backup_digest = NULL; const char *epoch = NULL; const char *admin_epoch = NULL; if (p) { /* Synchronous write out */ local_cib = copy_xml(p); } else { + int pid = fork(); + if (pid < 0) { + return FALSE; + } + + if (pid) { + /* Parent */ + g_child_watch_add(pid, cib_diskwrite_complete, NULL); + return TRUE; + } + /* A-synchronous write out after a fork() */ /* Don't log anything unless strictly necessary */ set_crm_log_level(LOG_ERR); /* In theory we can scribble on "the_cib" here and not affect the parent * But lets be safe anyway */ local_cib = copy_xml(the_cib); } epoch = crm_element_value(local_cib, XML_ATTR_GENERATION); admin_epoch = crm_element_value(local_cib, XML_ATTR_GENERATION_ADMIN); tmp1 = crm_concat(cib_root, "cib.XXXXXX", '/'); tmp2 = crm_concat(cib_root, "cib.XXXXXX", '/'); primary_file = crm_concat(cib_root, "cib.xml", '/'); digest_file = crm_concat(primary_file, "sig", '.'); /* Always write out with num_updates=0 */ crm_xml_add(local_cib, XML_ATTR_NUMUPDATES, "0"); need_archive = (stat(primary_file, &buf) == 0); if (need_archive) { int rc = 0; int seq = get_last_sequence(cib_root, CIB_SERIES); /* check the admin didnt modify it underneath us */ if (validate_on_disk_cib(primary_file, NULL) == FALSE) { crm_err("%s was manually modified while the cluster was active!", primary_file); exit_rc = 1; goto cleanup; } backup_file = generate_series_filename(cib_root, CIB_SERIES, seq, FALSE); backup_digest = crm_concat(backup_file, "sig", '.'); unlink(backup_file); unlink(backup_digest); rc = link(primary_file, backup_file); if(rc < 0) { exit_rc = 4; crm_perror(LOG_ERR, "Cannot link %s to %s", primary_file, backup_file); goto cleanup; } rc = stat(digest_file, &buf); if (rc == 0) { rc = link(digest_file, backup_digest); if(rc < 0) { exit_rc = 5; crm_perror(LOG_ERR, "Cannot link %s to %s", digest_file, backup_digest); goto cleanup; } } write_last_sequence(cib_root, CIB_SERIES, seq + 1, cib_wrap); sync_directory(cib_root); crm_info("Archived previous version as %s", backup_file); } /* Given that we discard the status section on startup * there is no point writing it out in the first place * since users just get confused by it * * So delete the status section before we write it out */ crm_debug("Writing CIB to disk"); if (p == NULL) { cib_status_root = find_xml_node(local_cib, XML_CIB_TAG_STATUS, TRUE); CRM_LOG_ASSERT(cib_status_root != NULL); if (cib_status_root != NULL) { free_xml_from_parent(local_cib, cib_status_root); } } tmp1 = mktemp(tmp1); /* cib */ tmp2 = mktemp(tmp2); /* digest */ if (write_xml_file(local_cib, tmp1, FALSE) <= 0) { crm_err("Changes couldn't be written to %s", tmp1); exit_rc = 2; goto cleanup; } /* Must calculate the digest after writing as write_xml_file() updates the last-written field */ digest = calculate_on_disk_digest(local_cib); crm_info("Wrote version %s.%s.0 of the CIB to disk (digest: %s)", admin_epoch ? admin_epoch : "0", epoch ? epoch : "0", digest); if (write_cib_digest(local_cib, tmp2, digest) <= 0) { crm_err("Digest couldn't be written to %s", tmp2); exit_rc = 3; goto cleanup; } crm_debug("Wrote digest %s to disk", digest); CRM_ASSERT(retrieveCib(tmp1, tmp2, FALSE) != NULL); sync_directory(cib_root); crm_debug("Activating %s", tmp1); cib_rename(tmp1, primary_file); cib_rename(tmp2, digest_file); sync_directory(cib_root); cleanup: crm_free(backup_digest); crm_free(primary_file); crm_free(backup_file); crm_free(digest_file); crm_free(digest); crm_free(tmp2); crm_free(tmp1); free_xml(local_cib); if (p == NULL) { /* exit() could potentially affect the parent by closing things it shouldn't * Use _exit instead */ _exit(exit_rc); } return exit_rc; } void GHFunc_count_peers(gpointer key, gpointer value, gpointer user_data) { int *active = user_data; if (safe_str_eq(value, ONLINESTATUS)) { (*active)++; } else if (safe_str_eq(value, JOINSTATUS)) { (*active)++; } } diff --git a/cib/main.c b/cib/main.c index ed135a0f2c..f7a9db5d16 100644 --- a/cib/main.c +++ b/cib/main.c @@ -1,622 +1,600 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if HAVE_LIBXML2 # include #endif #ifdef HAVE_GETOPT_H # include #endif #if HAVE_BZLIB_H # include #endif extern int init_remote_listener(int port, gboolean encrypted); extern gboolean stand_alone; gboolean cib_shutdown_flag = FALSE; enum cib_errors cib_status = cib_ok; #if SUPPORT_HEARTBEAT oc_ev_t *cib_ev_token; ll_cluster_t *hb_conn = NULL; extern void oc_ev_special(const oc_ev_t *, oc_ev_class_t, int); gboolean cib_register_ha(ll_cluster_t * hb_cluster, const char *client_name); #endif extern void terminate_cib(const char *caller, gboolean fast); GMainLoop *mainloop = NULL; const char *cib_root = CRM_CONFIG_DIR; char *cib_our_uname = NULL; gboolean preserve_status = FALSE; gboolean cib_writes_enabled = TRUE; int remote_fd = 0; int remote_tls_fd = 0; void usage(const char *cmd, int exit_status); int cib_init(void); void cib_shutdown(int nsig); gboolean startCib(const char *filename); extern int write_cib_contents(gpointer p); -GTRIGSource *cib_writer = NULL; GHashTable *client_list = NULL; GHashTable *config_hash = NULL; char *channel1 = NULL; char *channel2 = NULL; char *channel3 = NULL; char *channel4 = NULL; char *channel5 = NULL; #define OPTARGS "maswr:V?" void cib_cleanup(void); static void cib_enable_writes(int nsig) { crm_info("(Re)enabling disk writes"); cib_writes_enabled = TRUE; } -static void -cib_diskwrite_complete(gpointer userdata, int status, int signo, int exitcode) -{ - if (exitcode != LSB_EXIT_OK || signo != 0 || status != 0) { - crm_err("Disk write failed: status=%d, signo=%d, exitcode=%d", status, signo, exitcode); - - if (cib_writes_enabled) { - crm_err("Disabling disk writes after write failure"); - cib_writes_enabled = FALSE; - } - - } else { - crm_trace("Disk write passed"); - } -} - static void log_cib_client(gpointer key, gpointer value, gpointer user_data) { cib_client_t *a_client = value; crm_info("Client %s", crm_str(a_client->name)); } int main(int argc, char **argv) { int flag; int rc = 0; int argerr = 0; #ifdef HAVE_GETOPT_H int option_index = 0; /* *INDENT-OFF* */ static struct option long_options[] = { {"per-action-cib", 0, 0, 'a'}, {"stand-alone", 0, 0, 's'}, {"disk-writes", 0, 0, 'w'}, {"cib-root", 1, 0, 'r'}, {"verbose", 0, 0, 'V'}, {"help", 0, 0, '?'}, {"metadata", 0, 0, 'm'}, {0, 0, 0, 0} }; /* *INDENT-ON* */ #endif struct passwd *pwentry = NULL; crm_log_init("cib", LOG_INFO, TRUE, FALSE, 0, NULL); mainloop_add_signal(SIGTERM, cib_shutdown); mainloop_add_signal(SIGPIPE, cib_enable_writes); - cib_writer = - G_main_add_tempproc_trigger(G_PRIORITY_LOW, write_cib_contents, "write_cib_contents", NULL, - NULL, NULL, cib_diskwrite_complete); - - /* EnableProcLogging(); */ - set_sigchld_proctrack(G_PRIORITY_HIGH, DEFAULT_MAXDISPATCHTIME); + cib_writer = mainloop_add_trigger(G_PRIORITY_LOW, write_cib_contents, NULL); crm_peer_init(); client_list = g_hash_table_new(crm_str_hash, g_str_equal); while (1) { #ifdef HAVE_GETOPT_H flag = getopt_long(argc, argv, OPTARGS, long_options, &option_index); #else flag = getopt(argc, argv, OPTARGS); #endif if (flag == -1) break; switch (flag) { case 'V': crm_bump_log_level(); break; case 's': stand_alone = TRUE; preserve_status = TRUE; cib_writes_enabled = FALSE; pwentry = getpwnam(CRM_DAEMON_USER); CRM_CHECK(pwentry != NULL, crm_perror(LOG_ERR, "Invalid uid (%s) specified", CRM_DAEMON_USER); return 100); rc = setgid(pwentry->pw_gid); if (rc < 0) { crm_perror(LOG_ERR, "Could not set group to %d", pwentry->pw_gid); return 100; } rc = setuid(pwentry->pw_uid); if (rc < 0) { crm_perror(LOG_ERR, "Could not set user to %d", pwentry->pw_uid); return 100; } break; case '?': /* Help message */ usage(crm_system_name, LSB_EXIT_OK); break; case 'w': cib_writes_enabled = TRUE; break; case 'r': cib_root = optarg; break; case 'm': cib_metadata(); return 0; default: ++argerr; break; } } if (argc - optind == 1 && safe_str_eq("metadata", argv[optind])) { cib_metadata(); return 0; } if (optind > argc) { ++argerr; } if (argerr) { usage(crm_system_name, LSB_EXIT_GENERIC); } if (crm_is_writable(cib_root, NULL, CRM_DAEMON_USER, CRM_DAEMON_GROUP, FALSE) == FALSE) { crm_err("Bad permissions on %s. Terminating", cib_root); fprintf(stderr, "ERROR: Bad permissions on %s. See logs for details\n", cib_root); fflush(stderr); return 100; } /* read local config file */ rc = cib_init(); CRM_CHECK(g_hash_table_size(client_list) == 0, crm_warn("Not all clients gone at exit")); g_hash_table_foreach(client_list, log_cib_client, NULL); cib_cleanup(); #if SUPPORT_HEARTBEAT if (hb_conn) { hb_conn->llc_ops->delete(hb_conn); } #endif crm_info("Done"); return rc; } void cib_cleanup(void) { crm_peer_destroy(); g_hash_table_destroy(config_hash); g_hash_table_destroy(client_list); crm_free(cib_our_uname); #if HAVE_LIBXML2 crm_xml_cleanup(); #endif crm_free(channel1); crm_free(channel2); crm_free(channel3); crm_free(channel4); crm_free(channel5); } unsigned long cib_num_ops = 0; const char *cib_stat_interval = "10min"; unsigned long cib_num_local = 0, cib_num_updates = 0, cib_num_fail = 0; unsigned long cib_bad_connects = 0, cib_num_timeouts = 0; #if SUPPORT_HEARTBEAT gboolean ccm_connect(void); static void ccm_connection_destroy(gpointer user_data) { crm_err("CCM connection failed... blocking while we reconnect"); CRM_ASSERT(ccm_connect()); return; } static void *ccm_library = NULL; gboolean ccm_connect(void) { gboolean did_fail = TRUE; int num_ccm_fails = 0; int max_ccm_fails = 30; int ret; int cib_ev_fd; int (*ccm_api_register) (oc_ev_t ** token) = find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_register"); int (*ccm_api_set_callback) (const oc_ev_t * token, oc_ev_class_t class, oc_ev_callback_t * fn, oc_ev_callback_t ** prev_fn) = find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_set_callback"); void (*ccm_api_special) (const oc_ev_t *, oc_ev_class_t, int) = find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_special"); int (*ccm_api_activate) (const oc_ev_t * token, int *fd) = find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_activate"); int (*ccm_api_unregister) (oc_ev_t * token) = find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_unregister"); static struct mainloop_fd_callbacks ccm_fd_callbacks = { .dispatch = cib_ccm_dispatch, .destroy = ccm_connection_destroy, }; while (did_fail) { did_fail = FALSE; crm_info("Registering with CCM..."); ret = (*ccm_api_register) (&cib_ev_token); if (ret != 0) { did_fail = TRUE; } if (did_fail == FALSE) { crm_trace("Setting up CCM callbacks"); ret = (*ccm_api_set_callback) (cib_ev_token, OC_EV_MEMB_CLASS, cib_ccm_msg_callback, NULL); if (ret != 0) { crm_warn("CCM callback not set"); did_fail = TRUE; } } if (did_fail == FALSE) { (*ccm_api_special) (cib_ev_token, OC_EV_MEMB_CLASS, 0); crm_trace("Activating CCM token"); ret = (*ccm_api_activate) (cib_ev_token, &cib_ev_fd); if (ret != 0) { crm_warn("CCM Activation failed"); did_fail = TRUE; } } if (did_fail) { num_ccm_fails++; (*ccm_api_unregister) (cib_ev_token); if (num_ccm_fails < max_ccm_fails) { crm_warn("CCM Connection failed %d times (%d max)", num_ccm_fails, max_ccm_fails); sleep(3); } else { crm_err("CCM Activation failed %d (max) times", num_ccm_fails); return FALSE; } } } crm_debug("CCM Activation passed... all set to go!"); mainloop_add_fd("heartbeat-ccm", cib_ev_fd, cib_ev_token, &ccm_fd_callbacks); return TRUE; } #endif #if SUPPORT_COROSYNC static gboolean cib_ais_dispatch(AIS_Message * wrapper, char *data, int sender) { xmlNode *xml = NULL; if (wrapper->header.id == crm_class_cluster) { xml = string2xml(data); if (xml == NULL) { goto bail; } crm_xml_add(xml, F_ORIG, wrapper->sender.uname); crm_xml_add_int(xml, F_SEQ, wrapper->id); cib_peer_callback(xml, NULL); } free_xml(xml); return TRUE; bail: crm_err("Invalid XML: '%.120s'", data); return TRUE; } static void cib_ais_destroy(gpointer user_data) { if (cib_shutdown_flag) { crm_info("Corosync disconnection complete"); } else { crm_err("Corosync connection lost! Exiting."); terminate_cib(__FUNCTION__, TRUE); } } #endif static void cib_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data) { #if 0 /* crm_active_peers(crm_proc_cib) appears to give the wrong answer * sometimes, this might help figure out why */ if(type == crm_status_nstate) { crm_info("status: %s is now %s (was %s)", node->uname, node->state, (const char *)data); if (safe_str_eq(CRMD_STATE_ACTIVE, node->state)) { return; } } else if(type == crm_status_processes) { uint32_t old = 0; if (data) { old = *(const uint32_t *)data; } if ((node->processes ^ old) & crm_proc_cib) { crm_info("status: cib process on %s is now %sactive", node->uname, is_set(node->processes, crm_proc_cib)?"":"in"); } else { return; } } else { return; } #endif if(cib_shutdown_flag && crm_active_peers() < 2 && g_hash_table_size(client_list) == 0) { crm_info("No more peers"); terminate_cib(__FUNCTION__, FALSE); } } #if SUPPORT_HEARTBEAT static void cib_ha_connection_destroy(gpointer user_data) { if (cib_shutdown_flag) { crm_info("Heartbeat disconnection complete... exiting"); terminate_cib(__FUNCTION__, FALSE); } else { crm_err("Heartbeat connection lost! Exiting."); terminate_cib(__FUNCTION__, TRUE); } } #endif int cib_init(void) { gboolean was_error = FALSE; config_hash = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); if (startCib("cib.xml") == FALSE) { crm_crit("Cannot start CIB... terminating"); exit(1); } if (stand_alone == FALSE) { void *dispatch = NULL; void *destroy = NULL; if (is_openais_cluster()) { #if SUPPORT_COROSYNC destroy = cib_ais_destroy; dispatch = cib_ais_dispatch; #endif } else if(is_heartbeat_cluster()) { #if SUPPORT_HEARTBEAT dispatch = cib_ha_peer_callback; destroy = cib_ha_connection_destroy; #endif } if (crm_cluster_connect(&cib_our_uname, NULL, dispatch, destroy, #if SUPPORT_HEARTBEAT &hb_conn #else NULL #endif ) == FALSE) { crm_crit("Cannot sign in to the cluster... terminating"); exit(100); } if (is_openais_cluster()) { crm_set_status_callback(&cib_peer_update_callback); } #if SUPPORT_HEARTBEAT if (is_heartbeat_cluster()) { if (was_error == FALSE) { if (HA_OK != hb_conn->llc_ops->set_cstatus_callback(hb_conn, cib_client_status_callback, hb_conn)) { crm_err("Cannot set cstatus callback: %s", hb_conn->llc_ops->errmsg(hb_conn)); was_error = TRUE; } } if (was_error == FALSE) { was_error = (ccm_connect() == FALSE); } if (was_error == FALSE) { /* Async get client status information in the cluster */ crm_info("Requesting the list of configured nodes"); hb_conn->llc_ops->client_status(hb_conn, NULL, CRM_SYSTEM_CIB, -1); } } #endif } else { cib_our_uname = crm_strdup("localhost"); } ipcs_ro = mainloop_add_ipc_server(cib_channel_ro, QB_IPC_NATIVE, &ipc_ro_callbacks); ipcs_rw = mainloop_add_ipc_server(cib_channel_rw, QB_IPC_NATIVE, &ipc_rw_callbacks); ipcs_shm = mainloop_add_ipc_server(cib_channel_shm, QB_IPC_SHM, &ipc_rw_callbacks); if (stand_alone) { if (was_error) { crm_err("Couldnt start"); return 1; } cib_is_master = TRUE; /* Create the mainloop and run it... */ mainloop = g_main_new(FALSE); crm_info("Starting %s mainloop", crm_system_name); g_main_run(mainloop); return 0; } if (was_error == FALSE) { /* Create the mainloop and run it... */ mainloop = g_main_new(FALSE); crm_info("Starting %s mainloop", crm_system_name); g_main_run(mainloop); } else { crm_err("Couldnt start all communication channels, exiting."); } return 0; } void usage(const char *cmd, int exit_status) { FILE *stream; stream = exit_status ? stderr : stdout; fprintf(stream, "usage: %s [-%s]\n", cmd, OPTARGS); fprintf(stream, "\t--%s (-%c)\t\tTurn on debug info." " Additional instances increase verbosity\n", "verbose", 'V'); fprintf(stream, "\t--%s (-%c)\t\tThis help message\n", "help", '?'); fprintf(stream, "\t--%s (-%c)\t\tShow configurable cib options\n", "metadata", 'm'); fprintf(stream, "\t--%s (-%c)\tAdvanced use only\n", "per-action-cib", 'a'); fprintf(stream, "\t--%s (-%c)\tAdvanced use only\n", "stand-alone", 's'); fprintf(stream, "\t--%s (-%c)\tAdvanced use only\n", "disk-writes", 'w'); fprintf(stream, "\t--%s (-%c)\t\tAdvanced use only\n", "cib-root", 'r'); fflush(stream); exit(exit_status); } gboolean startCib(const char *filename) { gboolean active = FALSE; xmlNode *cib = readCibXmlFile(cib_root, filename, !preserve_status); CRM_ASSERT(cib != NULL); if (activateCibXml(cib, TRUE, "start") == 0) { int port = 0; const char *port_s = NULL; active = TRUE; cib_read_config(config_hash, cib); port_s = crm_element_value(cib, "remote-tls-port"); if (port_s) { port = crm_parse_int(port_s, "0"); remote_tls_fd = init_remote_listener(port, TRUE); } port_s = crm_element_value(cib, "remote-clear-port"); if (port_s) { port = crm_parse_int(port_s, "0"); remote_fd = init_remote_listener(port, FALSE); } crm_info("CIB Initialization completed successfully"); } return active; }