Page MenuHomeClusterLabs Projects
Files F3686868

No OneTemporary
Actions

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.
diff --git a/daemons/based/based_io.c b/daemons/based/based_io.c
index 7c73776fc0..f252ac1564 100644
--- a/daemons/based/based_io.c
+++ b/daemons/based/based_io.c
@@ -1,479 +1,479 @@
/*
* Copyright 2004-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <errno.h>
#include <fcntl.h>
#include <dirent.h>
#include <sys/param.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <glib.h>
#include <libxml/tree.h>
#include <crm/crm.h>
#include <crm/cib.h>
#include <crm/common/util.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <crm/cib/internal.h>
#include <crm/cluster.h>
#include <pacemaker-based.h>
crm_trigger_t *cib_writer = NULL;
int write_cib_contents(gpointer p);
static void
cib_rename(const char *old)
{
int new_fd;
char *new = crm_strdup_printf("%s/cib.auto.XXXXXX", cib_root);
umask(S_IWGRP | S_IWOTH | S_IROTH);
new_fd = mkstemp(new);
if ((new_fd < 0) || (rename(old, new) < 0)) {
crm_err("Couldn't archive unusable file %s (disabling disk writes and continuing)",
old);
cib_writes_enabled = FALSE;
} else {
crm_err("Archived unusable file %s as %s", old, new);
}
if (new_fd > 0) {
close(new_fd);
}
free(new);
}
/*
* It is the callers responsibility to free the output of this function
*/
static xmlNode *
retrieveCib(const char *filename, const char *sigfile)
{
xmlNode *root = NULL;
crm_info("Reading cluster configuration file %s (digest: %s)",
filename, sigfile);
switch (cib_file_read_and_verify(filename, sigfile, &root)) {
case -pcmk_err_cib_corrupt:
crm_warn("Continuing but %s will NOT be used.", filename);
break;
case -pcmk_err_cib_modified:
/* Archive the original files so the contents are not lost */
crm_warn("Continuing but %s will NOT be used.", filename);
cib_rename(filename);
cib_rename(sigfile);
break;
}
return root;
}
/*
* for OSs without support for direntry->d_type, like Solaris
*/
#ifndef DT_UNKNOWN
# define DT_UNKNOWN 0
# define DT_FIFO 1
# define DT_CHR 2
# define DT_DIR 4
# define DT_BLK 6
# define DT_REG 8
# define DT_LNK 10
# define DT_SOCK 12
# define DT_WHT 14
#endif /*DT_UNKNOWN*/
static int cib_archive_filter(const struct dirent * a)
{
int rc = 0;
/* Looking for regular files (d_type = 8) starting with 'cib-' and not ending in .sig */
struct stat s;
char *a_path = crm_strdup_printf("%s/%s", cib_root, a->d_name);
if(stat(a_path, &s) != 0) {
rc = errno;
crm_trace("%s - stat failed: %s (%d)", a->d_name, pcmk_rc_str(rc), rc);
rc = 0;
} else if ((s.st_mode & S_IFREG) != S_IFREG) {
unsigned char dtype;
#ifdef HAVE_STRUCT_DIRENT_D_TYPE
dtype = a->d_type;
#else
switch (s.st_mode & S_IFMT) {
case S_IFREG: dtype = DT_REG; break;
case S_IFDIR: dtype = DT_DIR; break;
case S_IFCHR: dtype = DT_CHR; break;
case S_IFBLK: dtype = DT_BLK; break;
case S_IFLNK: dtype = DT_LNK; break;
case S_IFIFO: dtype = DT_FIFO; break;
case S_IFSOCK: dtype = DT_SOCK; break;
default: dtype = DT_UNKNOWN; break;
}
#endif
crm_trace("%s - wrong type (%d)", a->d_name, dtype);
} else if(strstr(a->d_name, "cib-") != a->d_name) {
crm_trace("%s - wrong prefix", a->d_name);
} else if (pcmk__ends_with_ext(a->d_name, ".sig")) {
crm_trace("%s - wrong suffix", a->d_name);
} else {
crm_debug("%s - candidate", a->d_name);
rc = 1;
}
free(a_path);
return rc;
}
static int cib_archive_sort(const struct dirent ** a, const struct dirent **b)
{
/* Order by creation date - most recently created file first */
int rc = 0;
struct stat buf;
time_t a_age = 0;
time_t b_age = 0;
char *a_path = crm_strdup_printf("%s/%s", cib_root, a[0]->d_name);
char *b_path = crm_strdup_printf("%s/%s", cib_root, b[0]->d_name);
if(stat(a_path, &buf) == 0) {
a_age = buf.st_ctime;
}
if(stat(b_path, &buf) == 0) {
b_age = buf.st_ctime;
}
free(a_path);
free(b_path);
if(a_age > b_age) {
rc = 1;
} else if(a_age < b_age) {
rc = -1;
}
crm_trace("%s (%lu) vs. %s (%lu) : %d",
a[0]->d_name, (unsigned long)a_age,
b[0]->d_name, (unsigned long)b_age, rc);
return rc;
}
xmlNode *
readCibXmlFile(const char *dir, const char *file, gboolean discard_status)
{
struct dirent **namelist = NULL;
int lpc = 0;
char *sigfile = NULL;
char *sigfilepath = NULL;
char *filename = NULL;
const char *name = NULL;
const char *value = NULL;
const char *validation = NULL;
- const char *use_valgrind = getenv("PCMK_valgrind_enabled");
+ const char *use_valgrind = pcmk__env_option(PCMK__ENV_VALGRIND_ENABLED);
xmlNode *root = NULL;
xmlNode *status = NULL;
sigfile = crm_strdup_printf("%s.sig", file);
if (pcmk__daemon_can_write(dir, file) == FALSE
|| pcmk__daemon_can_write(dir, sigfile) == FALSE) {
cib_status = -EACCES;
return NULL;
}
filename = crm_strdup_printf("%s/%s", dir, file);
sigfilepath = crm_strdup_printf("%s/%s", dir, sigfile);
free(sigfile);
cib_status = pcmk_ok;
root = retrieveCib(filename, sigfilepath);
free(filename);
free(sigfilepath);
if (root == NULL) {
crm_warn("Primary configuration corrupt or unusable, trying backups in %s", cib_root);
lpc = scandir(cib_root, &namelist, cib_archive_filter, cib_archive_sort);
if (lpc < 0) {
crm_err("scandir(%s) failed: %s", cib_root, pcmk_rc_str(errno));
}
}
while (root == NULL && lpc > 1) {
crm_debug("Testing %d candidates", lpc);
lpc--;
filename = crm_strdup_printf("%s/%s", cib_root, namelist[lpc]->d_name);
sigfile = crm_strdup_printf("%s.sig", filename);
crm_info("Reading cluster configuration file %s (digest: %s)",
filename, sigfile);
if (cib_file_read_and_verify(filename, sigfile, &root) < 0) {
crm_warn("Continuing but %s will NOT be used.", filename);
} else {
crm_notice("Continuing with last valid configuration archive: %s", filename);
}
free(namelist[lpc]);
free(filename);
free(sigfile);
}
free(namelist);
if (root == NULL) {
root = createEmptyCib(0);
crm_warn("Continuing with an empty configuration.");
}
if (cib_writes_enabled && use_valgrind &&
(crm_is_true(use_valgrind) || strstr(use_valgrind, "pacemaker-based"))) {
cib_writes_enabled = FALSE;
crm_err("*** Disabling disk writes to avoid confusing Valgrind ***");
}
status = find_xml_node(root, XML_CIB_TAG_STATUS, FALSE);
if (discard_status && status != NULL) {
/* strip out the status section if there is one */
free_xml(status);
status = NULL;
}
if (status == NULL) {
create_xml_node(root, XML_CIB_TAG_STATUS);
}
/* Do this before schema validation happens */
/* fill in some defaults */
name = XML_ATTR_GENERATION_ADMIN;
value = crm_element_value(root, name);
if (value == NULL) {
crm_warn("No value for %s was specified in the configuration.", name);
crm_warn("The recommended course of action is to shutdown,"
" run crm_verify and fix any errors it reports.");
crm_warn("We will default to zero and continue but may get"
" confused about which configuration to use if"
" multiple nodes are powered up at the same time.");
crm_xml_add_int(root, name, 0);
}
name = XML_ATTR_GENERATION;
value = crm_element_value(root, name);
if (value == NULL) {
crm_xml_add_int(root, name, 0);
}
name = XML_ATTR_NUMUPDATES;
value = crm_element_value(root, name);
if (value == NULL) {
crm_xml_add_int(root, name, 0);
}
// Unset (DC should set appropriate value)
xml_remove_prop(root, XML_ATTR_DC_UUID);
if (discard_status) {
crm_log_xml_trace(root, "[on-disk]");
}
validation = crm_element_value(root, XML_ATTR_VALIDATION);
if (validate_xml(root, NULL, TRUE) == FALSE) {
crm_err("CIB does not validate with %s",
pcmk__s(validation, "no schema specified"));
cib_status = -pcmk_err_schema_validation;
} else if (validation == NULL) {
int version = 0;
update_validation(&root, &version, 0, FALSE, FALSE);
if (version > 0) {
crm_notice("Enabling %s validation on"
" the existing (sane) configuration", get_schema_name(version));
} else {
crm_err("CIB does not validate with any known schema");
cib_status = -pcmk_err_schema_validation;
}
}
return root;
}
gboolean
uninitializeCib(void)
{
xmlNode *tmp_cib = the_cib;
if (tmp_cib == NULL) {
crm_debug("The CIB has already been deallocated.");
return FALSE;
}
the_cib = NULL;
crm_debug("Deallocating the CIB.");
free_xml(tmp_cib);
crm_debug("The CIB has been deallocated.");
return TRUE;
}
/*
* This method will free the old CIB pointer on success and the new one
* on failure.
*/
int
activateCibXml(xmlNode * new_cib, gboolean to_disk, const char *op)
{
if (new_cib) {
xmlNode *saved_cib = the_cib;
CRM_ASSERT(new_cib != saved_cib);
the_cib = new_cib;
free_xml(saved_cib);
if (cib_writes_enabled && cib_status == pcmk_ok && to_disk) {
crm_debug("Triggering CIB write for %s op", op);
mainloop_set_trigger(cib_writer);
}
return pcmk_ok;
}
crm_err("Ignoring invalid CIB");
if (the_cib) {
crm_warn("Reverting to last known CIB");
} else {
crm_crit("Could not write out new CIB and no saved version to revert to");
}
return -ENODATA;
}
static void
cib_diskwrite_complete(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode)
{
const char *errmsg = "Could not write CIB to disk";
if ((exitcode != 0) && cib_writes_enabled) {
cib_writes_enabled = FALSE;
errmsg = "Disabling CIB disk writes after failure";
}
if ((signo == 0) && (exitcode == 0)) {
crm_trace("Disk write [%d] succeeded", (int) pid);
} else if (signo == 0) {
crm_err("%s: process %d exited %d", errmsg, (int) pid, exitcode);
} else {
crm_err("%s: process %d terminated with signal %d (%s)%s",
errmsg, (int) pid, signo, strsignal(signo),
(core? " and dumped core" : ""));
}
mainloop_trigger_complete(cib_writer);
}
int
write_cib_contents(gpointer p)
{
int exit_rc = pcmk_ok;
xmlNode *cib_local = NULL;
/* Make a copy of the CIB to write (possibly in a forked child) */
if (p) {
/* Synchronous write out */
cib_local = copy_xml(p);
} else {
int pid = 0;
int bb_state = qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_STATE_GET, 0);
/* Turn it off before the fork() to avoid:
* - 2 processes writing to the same shared mem
* - the child needing to disable it
* (which would close it from underneath the parent)
* This way, the shared mem files are already closed
*/
qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_FALSE);
pid = fork();
if (pid < 0) {
crm_err("Disabling disk writes after fork failure: %s", pcmk_rc_str(errno));
cib_writes_enabled = FALSE;
return FALSE;
}
if (pid) {
/* Parent */
mainloop_child_add(pid, 0, "disk-writer", NULL, cib_diskwrite_complete);
if (bb_state == QB_LOG_STATE_ENABLED) {
/* Re-enable now that it it safe */
qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_TRUE);
}
return -1; /* -1 means 'still work to do' */
}
/* Asynchronous write-out after a fork() */
/* In theory, we can scribble on the_cib here and not affect the parent,
* but let's be safe anyway.
*/
cib_local = copy_xml(the_cib);
}
/* Write the CIB */
exit_rc = cib_file_write_with_digest(cib_local, cib_root, "cib.xml");
/* A nonzero exit code will cause further writes to be disabled */
free_xml(cib_local);
if (p == NULL) {
crm_exit_t exit_code = CRM_EX_OK;
switch (exit_rc) {
case pcmk_ok:
exit_code = CRM_EX_OK;
break;
case pcmk_err_cib_modified:
exit_code = CRM_EX_DIGEST; // Existing CIB doesn't match digest
break;
case pcmk_err_cib_backup: // Existing CIB couldn't be backed up
case pcmk_err_cib_save: // New CIB couldn't be saved
exit_code = CRM_EX_CANTCREAT;
break;
default:
exit_code = CRM_EX_ERROR;
break;
}
/* Use _exit() because exit() could affect the parent adversely */
_exit(exit_code);
}
return exit_rc;
}
diff --git a/daemons/controld/controld_cib.c b/daemons/controld/controld_cib.c
index a6ac5e6fb6..822c491977 100644
--- a/daemons/controld/controld_cib.c
+++ b/daemons/controld/controld_cib.c
@@ -1,1069 +1,1069 @@
/*
* Copyright 2004-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <unistd.h> /* sleep */
#include <crm/common/alerts_internal.h>
#include <crm/common/xml.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/lrmd_internal.h>
#include <pacemaker-controld.h>
// Call ID of the most recent in-progress CIB resource update (or 0 if none)
static int pending_rsc_update = 0;
/*!
* \internal
* \brief Respond to a dropped CIB connection
*
* \param[in] user_data CIB connection that dropped
*/
static void
handle_cib_disconnect(gpointer user_data)
{
CRM_LOG_ASSERT(user_data == controld_globals.cib_conn);
controld_trigger_fsa();
controld_globals.cib_conn->state = cib_disconnected;
if (pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) {
// @TODO This should trigger a reconnect, not a shutdown
crm_crit("Lost connection to the CIB manager, shutting down");
register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
controld_clear_fsa_input_flags(R_CIB_CONNECTED);
} else { // Expected
crm_info("Disconnected from the CIB manager");
}
}
static void
do_cib_updated(const char *event, xmlNode * msg)
{
const xmlNode *patchset = NULL;
const char *client_name = NULL;
crm_debug("Received CIB diff notification: DC=%s", pcmk__btoa(AM_I_DC));
if (cib__get_notify_patchset(msg, &patchset) != pcmk_rc_ok) {
return;
}
if (cib__element_in_patchset(patchset, XML_CIB_TAG_ALERTS)
|| cib__element_in_patchset(patchset, XML_CIB_TAG_CRMCONFIG)) {
controld_trigger_config();
}
if (!AM_I_DC) {
// We're not in control of the join sequence
return;
}
client_name = crm_element_value(msg, F_CIB_CLIENTNAME);
if (!cib__client_triggers_refresh(client_name)) {
// The CIB is still accurate
return;
}
if (cib__element_in_patchset(patchset, XML_CIB_TAG_NODES)
|| cib__element_in_patchset(patchset, XML_CIB_TAG_STATUS)) {
/* An unsafe client modified the nodes or status section. Ensure the
* node list is up-to-date, and start the join process again so we get
* everyone's current resource history.
*/
if (client_name == NULL) {
client_name = crm_element_value(msg, F_CIB_CLIENTID);
}
crm_notice("Populating nodes and starting an election after %s event "
"triggered by %s",
event, pcmk__s(client_name, "(unidentified client)"));
populate_cib_nodes(node_update_quick|node_update_all, __func__);
register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
}
}
void
controld_disconnect_cib_manager(void)
{
cib_t *cib_conn = controld_globals.cib_conn;
CRM_ASSERT(cib_conn != NULL);
crm_debug("Disconnecting from the CIB manager");
controld_clear_fsa_input_flags(R_CIB_CONNECTED);
cib_conn->cmds->del_notify_callback(cib_conn, T_CIB_DIFF_NOTIFY,
do_cib_updated);
cib_free_callbacks(cib_conn);
if (cib_conn->state != cib_disconnected) {
cib_conn->cmds->set_secondary(cib_conn,
cib_scope_local|cib_discard_reply);
cib_conn->cmds->signoff(cib_conn);
}
}
/* A_CIB_STOP, A_CIB_START, O_CIB_RESTART */
void
do_cib_control(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
static int cib_retries = 0;
cib_t *cib_conn = controld_globals.cib_conn;
void (*dnotify_fn) (gpointer user_data) = handle_cib_disconnect;
void (*update_cb) (const char *event, xmlNodePtr msg) = do_cib_updated;
int rc = pcmk_ok;
CRM_ASSERT(cib_conn != NULL);
if (pcmk_is_set(action, A_CIB_STOP)) {
if ((cib_conn->state != cib_disconnected)
&& (pending_rsc_update != 0)) {
crm_info("Waiting for resource update %d to complete",
pending_rsc_update);
crmd_fsa_stall(FALSE);
return;
}
controld_disconnect_cib_manager();
}
if (!pcmk_is_set(action, A_CIB_START)) {
return;
}
if (cur_state == S_STOPPING) {
crm_err("Ignoring request to connect to the CIB manager after "
"shutdown");
return;
}
rc = cib_conn->cmds->signon(cib_conn, CRM_SYSTEM_CRMD,
cib_command_nonblocking);
if (rc != pcmk_ok) {
// A short wait that usually avoids stalling the FSA
sleep(1);
rc = cib_conn->cmds->signon(cib_conn, CRM_SYSTEM_CRMD,
cib_command_nonblocking);
}
if (rc != pcmk_ok) {
crm_info("Could not connect to the CIB manager: %s", pcmk_strerror(rc));
} else if (cib_conn->cmds->set_connection_dnotify(cib_conn,
dnotify_fn) != pcmk_ok) {
crm_err("Could not set dnotify callback");
} else if (cib_conn->cmds->add_notify_callback(cib_conn,
T_CIB_DIFF_NOTIFY,
update_cb) != pcmk_ok) {
crm_err("Could not set CIB notification callback (update)");
} else {
controld_set_fsa_input_flags(R_CIB_CONNECTED);
cib_retries = 0;
}
if (!pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) {
cib_retries++;
if (cib_retries < 30) {
crm_warn("Couldn't complete CIB registration %d times... "
"pause and retry", cib_retries);
controld_start_wait_timer();
crmd_fsa_stall(FALSE);
} else {
crm_err("Could not complete CIB registration %d times... "
"hard error", cib_retries);
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
}
}
#define MIN_CIB_OP_TIMEOUT (30)
/*!
* \internal
* \brief Get the timeout (in seconds) that should be used with CIB operations
*
* \return The maximum of 30 seconds, the value of the PCMK_cib_timeout
* environment variable, or 10 seconds times one more than the number of
* nodes in the cluster.
*/
unsigned int
cib_op_timeout(void)
{
static int env_timeout = -1;
unsigned int calculated_timeout = 0;
if (env_timeout == -1) {
- const char *env = getenv("PCMK_cib_timeout");
+ const char *env = pcmk__env_option(PCMK__ENV_CIB_TIMEOUT);
pcmk__scan_min_int(env, &env_timeout, MIN_CIB_OP_TIMEOUT);
crm_trace("Minimum CIB op timeout: %ds (environment: %s)",
env_timeout, (env? env : "none"));
}
calculated_timeout = 1 + crm_active_peers();
if (crm_remote_peer_cache) {
calculated_timeout += g_hash_table_size(crm_remote_peer_cache);
}
calculated_timeout *= 10;
calculated_timeout = QB_MAX(calculated_timeout, env_timeout);
crm_trace("Calculated timeout: %us", calculated_timeout);
if (controld_globals.cib_conn) {
controld_globals.cib_conn->call_timeout = calculated_timeout;
}
return calculated_timeout;
}
/*!
* \internal
* \brief Get CIB call options to use local scope if primary is unavailable
*
* \return CIB call options
*/
int
crmd_cib_smart_opt(void)
{
int call_opt = cib_none;
if ((controld_globals.fsa_state == S_ELECTION)
|| (controld_globals.fsa_state == S_PENDING)) {
crm_info("Sending update to local CIB in state: %s",
fsa_state2string(controld_globals.fsa_state));
cib__set_call_options(call_opt, "update", cib_scope_local);
}
return call_opt;
}
static void
cib_delete_callback(xmlNode *msg, int call_id, int rc, xmlNode *output,
void *user_data)
{
char *desc = user_data;
if (rc == 0) {
crm_debug("Deletion of %s (via CIB call %d) succeeded", desc, call_id);
} else {
crm_warn("Deletion of %s (via CIB call %d) failed: %s " CRM_XS " rc=%d",
desc, call_id, pcmk_strerror(rc), rc);
}
}
// Searches for various portions of node_state to delete
// Match a particular node's node_state (takes node name 1x)
#define XPATH_NODE_STATE "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"
// Node's lrm section (name 1x)
#define XPATH_NODE_LRM XPATH_NODE_STATE "/" XML_CIB_TAG_LRM
/* Node's lrm_rsc_op entries and lrm_resource entries without unexpired lock
* (name 2x, (seconds_since_epoch - XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT) 1x)
*/
#define XPATH_NODE_LRM_UNLOCKED XPATH_NODE_STATE "//" XML_LRM_TAG_RSC_OP \
"|" XPATH_NODE_STATE \
"//" XML_LRM_TAG_RESOURCE \
"[not(@" XML_CONFIG_ATTR_SHUTDOWN_LOCK ") " \
"or " XML_CONFIG_ATTR_SHUTDOWN_LOCK "<%lld]"
// Node's transient_attributes section (name 1x)
#define XPATH_NODE_ATTRS XPATH_NODE_STATE "/" XML_TAG_TRANSIENT_NODEATTRS
// Everything under node_state (name 1x)
#define XPATH_NODE_ALL XPATH_NODE_STATE "/*"
/* Unlocked history + transient attributes
* (name 2x, (seconds_since_epoch - XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT) 1x,
* name 1x)
*/
#define XPATH_NODE_ALL_UNLOCKED XPATH_NODE_LRM_UNLOCKED "|" XPATH_NODE_ATTRS
/*!
* \internal
* \brief Get the XPath and description of a node state section to be deleted
*
* \param[in] uname Desired node
* \param[in] section Subsection of node_state to be deleted
* \param[out] xpath Where to store XPath of \p section
* \param[out] desc If not \c NULL, where to store description of \p section
*/
void
controld_node_state_deletion_strings(const char *uname,
enum controld_section_e section,
char **xpath, char **desc)
{
const char *desc_pre = NULL;
// Shutdown locks that started before this time are expired
long long expire = (long long) time(NULL)
- controld_globals.shutdown_lock_limit;
switch (section) {
case controld_section_lrm:
*xpath = crm_strdup_printf(XPATH_NODE_LRM, uname);
desc_pre = "resource history";
break;
case controld_section_lrm_unlocked:
*xpath = crm_strdup_printf(XPATH_NODE_LRM_UNLOCKED,
uname, uname, expire);
desc_pre = "resource history (other than shutdown locks)";
break;
case controld_section_attrs:
*xpath = crm_strdup_printf(XPATH_NODE_ATTRS, uname);
desc_pre = "transient attributes";
break;
case controld_section_all:
*xpath = crm_strdup_printf(XPATH_NODE_ALL, uname);
desc_pre = "all state";
break;
case controld_section_all_unlocked:
*xpath = crm_strdup_printf(XPATH_NODE_ALL_UNLOCKED,
uname, uname, expire, uname);
desc_pre = "all state (other than shutdown locks)";
break;
default:
// We called this function incorrectly
CRM_ASSERT(false);
break;
}
if (desc != NULL) {
*desc = crm_strdup_printf("%s for node %s", desc_pre, uname);
}
}
/*!
* \internal
* \brief Delete subsection of a node's CIB node_state
*
* \param[in] uname Desired node
* \param[in] section Subsection of node_state to delete
* \param[in] options CIB call options to use
*/
void
controld_delete_node_state(const char *uname, enum controld_section_e section,
int options)
{
cib_t *cib = controld_globals.cib_conn;
char *xpath = NULL;
char *desc = NULL;
int cib_rc = pcmk_ok;
CRM_ASSERT((uname != NULL) && (cib != NULL));
controld_node_state_deletion_strings(uname, section, &xpath, &desc);
cib__set_call_options(options, "node state deletion",
cib_xpath|cib_multiple);
cib_rc = cib->cmds->remove(cib, xpath, NULL, options);
fsa_register_cib_callback(cib_rc, desc, cib_delete_callback);
crm_info("Deleting %s (via CIB call %d) " CRM_XS " xpath=%s",
desc, cib_rc, xpath);
// CIB library handles freeing desc
free(xpath);
}
// Takes node name and resource ID
#define XPATH_RESOURCE_HISTORY "//" XML_CIB_TAG_STATE \
"[@" XML_ATTR_UNAME "='%s']/" \
XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
"/" XML_LRM_TAG_RESOURCE \
"[@" XML_ATTR_ID "='%s']"
// @TODO could add "and @XML_CONFIG_ATTR_SHUTDOWN_LOCK" to limit to locks
/*!
* \internal
* \brief Clear resource history from CIB for a given resource and node
*
* \param[in] rsc_id ID of resource to be cleared
* \param[in] node Node whose resource history should be cleared
* \param[in] user_name ACL user name to use
* \param[in] call_options CIB call options
*
* \return Standard Pacemaker return code
*/
int
controld_delete_resource_history(const char *rsc_id, const char *node,
const char *user_name, int call_options)
{
char *desc = NULL;
char *xpath = NULL;
int rc = pcmk_rc_ok;
cib_t *cib = controld_globals.cib_conn;
CRM_CHECK((rsc_id != NULL) && (node != NULL), return EINVAL);
desc = crm_strdup_printf("resource history for %s on %s", rsc_id, node);
if (cib == NULL) {
crm_err("Unable to clear %s: no CIB connection", desc);
free(desc);
return ENOTCONN;
}
// Ask CIB to delete the entry
xpath = crm_strdup_printf(XPATH_RESOURCE_HISTORY, node, rsc_id);
cib->cmds->set_user(cib, user_name);
rc = cib->cmds->remove(cib, xpath, NULL, call_options|cib_xpath);
cib->cmds->set_user(cib, NULL);
if (rc < 0) {
rc = pcmk_legacy2rc(rc);
crm_err("Could not delete resource status of %s on %s%s%s: %s "
CRM_XS " rc=%d", rsc_id, node,
(user_name? " for user " : ""), (user_name? user_name : ""),
pcmk_rc_str(rc), rc);
free(desc);
free(xpath);
return rc;
}
if (pcmk_is_set(call_options, cib_sync_call)) {
if (pcmk_is_set(call_options, cib_dryrun)) {
crm_debug("Deletion of %s would succeed", desc);
} else {
crm_debug("Deletion of %s succeeded", desc);
}
free(desc);
} else {
crm_info("Clearing %s (via CIB call %d) " CRM_XS " xpath=%s",
desc, rc, xpath);
fsa_register_cib_callback(rc, desc, cib_delete_callback);
// CIB library handles freeing desc
}
free(xpath);
return pcmk_rc_ok;
}
/*!
* \internal
* \brief Build XML and string of parameters meeting some criteria, for digest
*
* \param[in] op Executor event with parameter table to use
* \param[in] metadata Parsed meta-data for executed resource agent
* \param[in] param_type Flag used for selection criteria
* \param[out] result Will be set to newly created XML with selected
* parameters as attributes
*
* \return Newly allocated space-separated string of parameter names
* \note Selection criteria varies by param_type: for the restart digest, we
* want parameters that are *not* marked reloadable (OCF 1.1) or that
* *are* marked unique (pre-1.1), for both string and XML results; for the
* secure digest, we want parameters that *are* marked private for the
* string, but parameters that are *not* marked private for the XML.
* \note It is the caller's responsibility to free the string return value with
* \p g_string_free() and the XML result with \p free_xml().
*/
static GString *
build_parameter_list(const lrmd_event_data_t *op,
const struct ra_metadata_s *metadata,
enum ra_param_flags_e param_type, xmlNode **result)
{
GString *list = NULL;
*result = create_xml_node(NULL, XML_TAG_PARAMS);
/* Consider all parameters only except private ones to be consistent with
* what scheduler does with calculate_secure_digest().
*/
if (param_type == ra_param_private
&& compare_version(controld_globals.dc_version, "3.16.0") >= 0) {
g_hash_table_foreach(op->params, hash2field, *result);
pcmk__filter_op_for_digest(*result);
}
for (GList *iter = metadata->ra_params; iter != NULL; iter = iter->next) {
struct ra_param_s *param = (struct ra_param_s *) iter->data;
bool accept_for_list = false;
bool accept_for_xml = false;
switch (param_type) {
case ra_param_reloadable:
accept_for_list = !pcmk_is_set(param->rap_flags, param_type);
accept_for_xml = accept_for_list;
break;
case ra_param_unique:
accept_for_list = pcmk_is_set(param->rap_flags, param_type);
accept_for_xml = accept_for_list;
break;
case ra_param_private:
accept_for_list = pcmk_is_set(param->rap_flags, param_type);
accept_for_xml = !accept_for_list;
break;
}
if (accept_for_list) {
crm_trace("Attr %s is %s", param->rap_name, ra_param_flag2text(param_type));
if (list == NULL) {
// We will later search for " WORD ", so start list with a space
pcmk__add_word(&list, 256, " ");
}
pcmk__add_word(&list, 0, param->rap_name);
} else {
crm_trace("Rejecting %s for %s", param->rap_name, ra_param_flag2text(param_type));
}
if (accept_for_xml) {
const char *v = g_hash_table_lookup(op->params, param->rap_name);
if (v != NULL) {
crm_trace("Adding attr %s=%s to the xml result", param->rap_name, v);
crm_xml_add(*result, param->rap_name, v);
}
} else {
crm_trace("Removing attr %s from the xml result", param->rap_name);
xml_remove_prop(*result, param->rap_name);
}
}
if (list != NULL) {
// We will later search for " WORD ", so end list with a space
pcmk__add_word(&list, 0, " ");
}
return list;
}
static void
append_restart_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
xmlNode *update, const char *version)
{
GString *list = NULL;
char *digest = NULL;
xmlNode *restart = NULL;
CRM_LOG_ASSERT(op->params != NULL);
if (op->interval_ms > 0) {
/* monitors are not reloadable */
return;
}
if (pcmk_is_set(metadata->ra_flags, ra_supports_reload_agent)) {
// Add parameters not marked reloadable to the "op-force-restart" list
list = build_parameter_list(op, metadata, ra_param_reloadable,
&restart);
} else if (pcmk_is_set(metadata->ra_flags, ra_supports_legacy_reload)) {
/* @COMPAT pre-OCF-1.1 resource agents
*
* Before OCF 1.1, Pacemaker abused "unique=0" to indicate
* reloadability. Add any parameters with unique="1" to the
* "op-force-restart" list.
*/
list = build_parameter_list(op, metadata, ra_param_unique, &restart);
} else {
// Resource does not support agent reloads
return;
}
digest = calculate_operation_digest(restart, version);
/* Add "op-force-restart" and "op-restart-digest" to indicate the resource supports reload,
* no matter if it actually supports any parameters with unique="1"). */
crm_xml_add(update, XML_LRM_ATTR_OP_RESTART,
(list == NULL)? "" : (const char *) list->str);
crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);
if ((list != NULL) && (list->len > 0)) {
crm_trace("%s: %s, %s", op->rsc_id, digest, (const char *) list->str);
} else {
crm_trace("%s: %s", op->rsc_id, digest);
}
if (list != NULL) {
g_string_free(list, TRUE);
}
free_xml(restart);
free(digest);
}
static void
append_secure_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
xmlNode *update, const char *version)
{
GString *list = NULL;
char *digest = NULL;
xmlNode *secure = NULL;
CRM_LOG_ASSERT(op->params != NULL);
/*
* To keep XML_LRM_ATTR_OP_SECURE short, we want it to contain the
* secure parameters but XML_LRM_ATTR_SECURE_DIGEST to be based on
* the insecure ones
*/
list = build_parameter_list(op, metadata, ra_param_private, &secure);
if (list != NULL) {
digest = calculate_operation_digest(secure, version);
crm_xml_add(update, XML_LRM_ATTR_OP_SECURE, (const char *) list->str);
crm_xml_add(update, XML_LRM_ATTR_SECURE_DIGEST, digest);
crm_trace("%s: %s, %s", op->rsc_id, digest, (const char *) list->str);
g_string_free(list, TRUE);
} else {
crm_trace("%s: no secure parameters", op->rsc_id);
}
free_xml(secure);
free(digest);
}
/*!
* \internal
* \brief Create XML for a resource history entry
*
* \param[in] func Function name of caller
* \param[in,out] parent XML to add entry to
* \param[in] rsc Affected resource
* \param[in,out] op Action to add an entry for (or NULL to do nothing)
* \param[in] node_name Node where action occurred
*/
void
controld_add_resource_history_xml_as(const char *func, xmlNode *parent,
const lrmd_rsc_info_t *rsc,
lrmd_event_data_t *op,
const char *node_name)
{
int target_rc = 0;
xmlNode *xml_op = NULL;
struct ra_metadata_s *metadata = NULL;
const char *caller_version = NULL;
lrm_state_t *lrm_state = NULL;
if (op == NULL) {
return;
}
target_rc = rsc_op_expected_rc(op);
caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION);
CRM_CHECK(caller_version != NULL, caller_version = CRM_FEATURE_SET);
xml_op = pcmk__create_history_xml(parent, op, caller_version, target_rc,
controld_globals.our_nodename, func);
if (xml_op == NULL) {
return;
}
if ((rsc == NULL) || (op->params == NULL)
|| !crm_op_needs_metadata(rsc->standard, op->op_type)) {
crm_trace("No digests needed for %s action on %s (params=%p rsc=%p)",
op->op_type, op->rsc_id, op->params, rsc);
return;
}
lrm_state = lrm_state_find(node_name);
if (lrm_state == NULL) {
crm_warn("Cannot calculate digests for operation " PCMK__OP_FMT
" because we have no connection to executor for %s",
op->rsc_id, op->op_type, op->interval_ms, node_name);
return;
}
/* Ideally the metadata is cached, and the agent is just a fallback.
*
* @TODO Go through all callers and ensure they get metadata asynchronously
* first.
*/
metadata = controld_get_rsc_metadata(lrm_state, rsc,
controld_metadata_from_agent
|controld_metadata_from_cache);
if (metadata == NULL) {
return;
}
crm_trace("Including additional digests for %s:%s:%s",
rsc->standard, rsc->provider, rsc->type);
append_restart_list(op, metadata, xml_op, caller_version);
append_secure_list(op, metadata, xml_op, caller_version);
return;
}
/*!
* \internal
* \brief Record an action as pending in the CIB, if appropriate
*
* \param[in] node_name Node where the action is pending
* \param[in] rsc Resource that action is for
* \param[in,out] op Pending action
*
* \return true if action was recorded in CIB, otherwise false
*/
bool
controld_record_pending_op(const char *node_name, const lrmd_rsc_info_t *rsc,
lrmd_event_data_t *op)
{
const char *record_pending = NULL;
CRM_CHECK((node_name != NULL) && (rsc != NULL) && (op != NULL),
return false);
// Never record certain operation types as pending
if ((op->op_type == NULL) || (op->params == NULL)
|| !controld_action_is_recordable(op->op_type)) {
return false;
}
// Check action's record-pending meta-attribute (defaults to true)
record_pending = crm_meta_value(op->params, XML_OP_ATTR_PENDING);
if ((record_pending != NULL) && !crm_is_true(record_pending)) {
return false;
}
op->call_id = -1;
op->t_run = time(NULL);
op->t_rcchange = op->t_run;
lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
crm_debug("Recording pending %s-interval %s for %s on %s in the CIB",
pcmk__readable_interval(op->interval_ms), op->op_type, op->rsc_id,
node_name);
controld_update_resource_history(node_name, rsc, op, 0);
return true;
}
static void
cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
switch (rc) {
case pcmk_ok:
case -pcmk_err_diff_failed:
case -pcmk_err_diff_resync:
crm_trace("Resource history update completed (call=%d rc=%d)",
call_id, rc);
break;
default:
if (call_id > 0) {
crm_warn("Resource history update %d failed: %s "
CRM_XS " rc=%d", call_id, pcmk_strerror(rc), rc);
} else {
crm_warn("Resource history update failed: %s " CRM_XS " rc=%d",
pcmk_strerror(rc), rc);
}
}
if (call_id == pending_rsc_update) {
pending_rsc_update = 0;
controld_trigger_fsa();
}
}
/* Only successful stops, and probes that found the resource inactive, get locks
* recorded in the history. This ensures the resource stays locked to the node
* until it is active there again after the node comes back up.
*/
static bool
should_preserve_lock(lrmd_event_data_t *op)
{
if (!pcmk_is_set(controld_globals.flags, controld_shutdown_lock_enabled)) {
return false;
}
if (!strcmp(op->op_type, PCMK_ACTION_STOP) && (op->rc == PCMK_OCF_OK)) {
return true;
}
if (!strcmp(op->op_type, PCMK_ACTION_MONITOR)
&& (op->rc == PCMK_OCF_NOT_RUNNING)) {
return true;
}
return false;
}
/*!
* \internal
* \brief Request a CIB update
*
* \param[in] section Section of CIB to update
* \param[in] data New XML of CIB section to update
* \param[in] options CIB call options
* \param[in] callback If not \c NULL, set this as the operation callback
*
* \return Standard Pacemaker return code
*
* \note If \p callback is \p cib_rsc_callback(), the CIB update's call ID is
* stored in \p pending_rsc_update on success.
*/
int
controld_update_cib(const char *section, xmlNode *data, int options,
void (*callback)(xmlNode *, int, int, xmlNode *, void *))
{
cib_t *cib = controld_globals.cib_conn;
int cib_rc = -ENOTCONN;
CRM_ASSERT(data != NULL);
if (cib != NULL) {
cib_rc = cib->cmds->modify(cib, section, data, options);
if (cib_rc >= 0) {
crm_debug("Submitted CIB update %d for %s section",
cib_rc, section);
}
}
if (callback == NULL) {
if (cib_rc < 0) {
crm_err("Failed to update CIB %s section: %s",
section, pcmk_rc_str(pcmk_legacy2rc(cib_rc)));
}
} else {
if ((cib_rc >= 0) && (callback == cib_rsc_callback)) {
/* Checking for a particular callback is a little hacky, but it
* didn't seem worth adding an output argument for cib_rc for just
* one use case.
*/
pending_rsc_update = cib_rc;
}
fsa_register_cib_callback(cib_rc, NULL, callback);
}
return (cib_rc >= 0)? pcmk_rc_ok : pcmk_legacy2rc(cib_rc);
}
/*!
* \internal
* \brief Update resource history entry in CIB
*
* \param[in] node_name Node where action occurred
* \param[in] rsc Resource that action is for
* \param[in,out] op Action to record
* \param[in] lock_time If nonzero, when resource was locked to node
*
* \note On success, the CIB update's call ID will be stored in
* pending_rsc_update.
*/
void
controld_update_resource_history(const char *node_name,
const lrmd_rsc_info_t *rsc,
lrmd_event_data_t *op, time_t lock_time)
{
xmlNode *update = NULL;
xmlNode *xml = NULL;
int call_opt = crmd_cib_smart_opt();
const char *node_id = NULL;
const char *container = NULL;
CRM_CHECK((node_name != NULL) && (op != NULL), return);
if (rsc == NULL) {
crm_warn("Resource %s no longer exists in the executor", op->rsc_id);
controld_ack_event_directly(NULL, NULL, rsc, op, op->rsc_id);
return;
}
// <status>
update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
// <node_state ...>
xml = create_xml_node(update, XML_CIB_TAG_STATE);
if (pcmk__str_eq(node_name, controld_globals.our_nodename,
pcmk__str_casei)) {
node_id = controld_globals.our_uuid;
} else {
node_id = node_name;
pcmk__xe_set_bool_attr(xml, XML_NODE_IS_REMOTE, true);
}
crm_xml_add(xml, XML_ATTR_ID, node_id);
crm_xml_add(xml, XML_ATTR_UNAME, node_name);
crm_xml_add(xml, XML_ATTR_ORIGIN, __func__);
// <lrm ...>
xml = create_xml_node(xml, XML_CIB_TAG_LRM);
crm_xml_add(xml, XML_ATTR_ID, node_id);
// <lrm_resources>
xml = create_xml_node(xml, XML_LRM_TAG_RESOURCES);
// <lrm_resource ...>
xml = create_xml_node(xml, XML_LRM_TAG_RESOURCE);
crm_xml_add(xml, XML_ATTR_ID, op->rsc_id);
crm_xml_add(xml, XML_AGENT_ATTR_CLASS, rsc->standard);
crm_xml_add(xml, XML_AGENT_ATTR_PROVIDER, rsc->provider);
crm_xml_add(xml, XML_ATTR_TYPE, rsc->type);
if (lock_time != 0) {
/* Actions on a locked resource should either preserve the lock by
* recording it with the action result, or clear it.
*/
if (!should_preserve_lock(op)) {
lock_time = 0;
}
crm_xml_add_ll(xml, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
(long long) lock_time);
}
if (op->params != NULL) {
container = g_hash_table_lookup(op->params,
CRM_META "_" XML_RSC_ATTR_CONTAINER);
if (container != NULL) {
crm_trace("Resource %s is a part of container resource %s",
op->rsc_id, container);
crm_xml_add(xml, XML_RSC_ATTR_CONTAINER, container);
}
}
// <lrm_resource_op ...> (possibly more than one)
controld_add_resource_history_xml(xml, rsc, op, node_name);
/* Update CIB asynchronously. Even if it fails, the resource state should be
* discovered during the next election. Worst case, the node is wrongly
* fenced for running a resource it isn't.
*/
crm_log_xml_trace(update, __func__);
controld_update_cib(XML_CIB_TAG_STATUS, update, call_opt, cib_rsc_callback);
free_xml(update);
}
/*!
* \internal
* \brief Erase an LRM history entry from the CIB, given the operation data
*
* \param[in] op Operation whose history should be deleted
*/
void
controld_delete_action_history(const lrmd_event_data_t *op)
{
xmlNode *xml_top = NULL;
CRM_CHECK(op != NULL, return);
xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);
if (op->interval_ms > 0) {
char *op_id = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
/* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
crm_xml_add(xml_top, XML_ATTR_ID, op_id);
free(op_id);
}
crm_debug("Erasing resource operation history for " PCMK__OP_FMT " (call=%d)",
op->rsc_id, op->op_type, op->interval_ms, op->call_id);
controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn,
XML_CIB_TAG_STATUS, xml_top,
cib_none);
crm_log_xml_trace(xml_top, "op:cancel");
free_xml(xml_top);
}
/* Define xpath to find LRM resource history entry by node and resource */
#define XPATH_HISTORY \
"/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
"/" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']" \
"/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
"/" XML_LRM_TAG_RESOURCE "[@" XML_ATTR_ID "='%s']" \
"/" XML_LRM_TAG_RSC_OP
/* ... and also by operation key */
#define XPATH_HISTORY_ID XPATH_HISTORY \
"[@" XML_ATTR_ID "='%s']"
/* ... and also by operation key and operation call ID */
#define XPATH_HISTORY_CALL XPATH_HISTORY \
"[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_CALLID "='%d']"
/* ... and also by operation key and original operation key */
#define XPATH_HISTORY_ORIG XPATH_HISTORY \
"[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_TASK_KEY "='%s']"
/*!
* \internal
* \brief Delete a last_failure resource history entry from the CIB
*
* \param[in] rsc_id Name of resource to clear history for
* \param[in] node Name of node to clear history for
* \param[in] action If specified, delete only if this was failed action
* \param[in] interval_ms If \p action is specified, it has this interval
*/
void
controld_cib_delete_last_failure(const char *rsc_id, const char *node,
const char *action, guint interval_ms)
{
char *xpath = NULL;
char *last_failure_key = NULL;
CRM_CHECK((rsc_id != NULL) && (node != NULL), return);
// Generate XPath to match desired entry
last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0);
if (action == NULL) {
xpath = crm_strdup_printf(XPATH_HISTORY_ID, node, rsc_id,
last_failure_key);
} else {
char *action_key = pcmk__op_key(rsc_id, action, interval_ms);
xpath = crm_strdup_printf(XPATH_HISTORY_ORIG, node, rsc_id,
last_failure_key, action_key);
free(action_key);
}
free(last_failure_key);
controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath,
NULL, cib_xpath);
free(xpath);
}
/*!
* \internal
* \brief Delete resource history entry from the CIB, given operation key
*
* \param[in] rsc_id Name of resource to clear history for
* \param[in] node Name of node to clear history for
* \param[in] key Operation key of operation to clear history for
* \param[in] call_id If specified, delete entry only if it has this call ID
*/
void
controld_delete_action_history_by_key(const char *rsc_id, const char *node,
const char *key, int call_id)
{
char *xpath = NULL;
CRM_CHECK((rsc_id != NULL) && (node != NULL) && (key != NULL), return);
if (call_id > 0) {
xpath = crm_strdup_printf(XPATH_HISTORY_CALL, node, rsc_id, key,
call_id);
} else {
xpath = crm_strdup_printf(XPATH_HISTORY_ID, node, rsc_id, key);
}
controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath,
NULL, cib_xpath);
free(xpath);
}
diff --git a/daemons/controld/controld_throttle.c b/daemons/controld/controld_throttle.c
index 63f469ee06..a4775e56cb 100644
--- a/daemons/controld/controld_throttle.c
+++ b/daemons/controld/controld_throttle.c
@@ -1,574 +1,574 @@
/*
* Copyright 2013-2021 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <ctype.h>
#include <dirent.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/cluster.h>
#include <pacemaker-controld.h>
/* These values don't need to be bits, but these particular values must be kept
* for backward compatibility during rolling upgrades.
*/
enum throttle_state_e {
throttle_none = 0x0000,
throttle_low = 0x0001,
throttle_med = 0x0010,
throttle_high = 0x0100,
throttle_extreme = 0x1000,
};
struct throttle_record_s {
int max;
enum throttle_state_e mode;
char *node;
};
static int throttle_job_max = 0;
static float throttle_load_target = 0.0;
#define THROTTLE_FACTOR_LOW 1.2
#define THROTTLE_FACTOR_MEDIUM 1.6
#define THROTTLE_FACTOR_HIGH 2.0
static GHashTable *throttle_records = NULL;
static mainloop_timer_t *throttle_timer = NULL;
static const char *
load2str(enum throttle_state_e mode)
{
switch (mode) {
case throttle_extreme: return "extreme";
case throttle_high: return "high";
case throttle_med: return "medium";
case throttle_low: return "low";
case throttle_none: return "negligible";
default: return "undetermined";
}
}
#if HAVE_LINUX_PROCFS
/*!
* \internal
* \brief Return name of /proc file containing the CIB daemon's load statistics
*
* \return Newly allocated memory with file name on success, NULL otherwise
*
* \note It is the caller's responsibility to free the return value.
* This will return NULL if the daemon is being run via valgrind.
* This should be called only on Linux systems.
*/
static char *
find_cib_loadfile(void)
{
pid_t pid = pcmk__procfs_pid_of("pacemaker-based");
return pid? crm_strdup_printf("/proc/%lld/stat", (long long) pid) : NULL;
}
static bool
throttle_cib_load(float *load)
{
/*
/proc/[pid]/stat
Status information about the process. This is used by ps(1). It is defined in /usr/src/linux/fs/proc/array.c.
The fields, in order, with their proper scanf(3) format specifiers, are:
pid %d (1) The process ID.
comm %s (2) The filename of the executable, in parentheses. This is visible whether or not the executable is swapped out.
state %c (3) One character from the string "RSDZTW" where R is running, S is sleeping in an interruptible wait, D is waiting in uninterruptible disk sleep, Z is zombie, T is traced or stopped (on a signal), and W is paging.
ppid %d (4) The PID of the parent.
pgrp %d (5) The process group ID of the process.
session %d (6) The session ID of the process.
tty_nr %d (7) The controlling terminal of the process. (The minor device number is contained in the combination of bits 31 to 20 and 7 to 0; the major device number is in bits 15 to 8.)
tpgid %d (8) The ID of the foreground process group of the controlling terminal of the process.
flags %u (%lu before Linux 2.6.22)
(9) The kernel flags word of the process. For bit meanings, see the PF_* defines in the Linux kernel source file include/linux/sched.h. Details depend on the kernel version.
minflt %lu (10) The number of minor faults the process has made which have not required loading a memory page from disk.
cminflt %lu (11) The number of minor faults that the process's waited-for children have made.
majflt %lu (12) The number of major faults the process has made which have required loading a memory page from disk.
cmajflt %lu (13) The number of major faults that the process's waited-for children have made.
utime %lu (14) Amount of time that this process has been scheduled in user mode, measured in clock ticks (divide by sysconf(_SC_CLK_TCK)). This includes guest time, guest_time (time spent running a virtual CPU, see below), so that applications that are not aware of the guest time field do not lose that time from their calculations.
stime %lu (15) Amount of time that this process has been scheduled in kernel mode, measured in clock ticks (divide by sysconf(_SC_CLK_TCK)).
*/
static char *loadfile = NULL;
static time_t last_call = 0;
static long ticks_per_s = 0;
static unsigned long last_utime, last_stime;
char buffer[64*1024];
FILE *stream = NULL;
time_t now = time(NULL);
if(load == NULL) {
return FALSE;
} else {
*load = 0.0;
}
if(loadfile == NULL) {
last_call = 0;
last_utime = 0;
last_stime = 0;
loadfile = find_cib_loadfile();
if (loadfile == NULL) {
crm_warn("Couldn't find CIB load file");
return FALSE;
}
ticks_per_s = sysconf(_SC_CLK_TCK);
crm_trace("Found %s", loadfile);
}
stream = fopen(loadfile, "r");
if(stream == NULL) {
int rc = errno;
crm_warn("Couldn't read %s: %s (%d)", loadfile, pcmk_rc_str(rc), rc);
free(loadfile); loadfile = NULL;
return FALSE;
}
if(fgets(buffer, sizeof(buffer), stream)) {
char *comm = calloc(1, 256);
char state = 0;
int rc = 0, pid = 0, ppid = 0, pgrp = 0, session = 0, tty_nr = 0, tpgid = 0;
unsigned long flags = 0, minflt = 0, cminflt = 0, majflt = 0, cmajflt = 0, utime = 0, stime = 0;
rc = sscanf(buffer, "%d %[^ ] %c %d %d %d %d %d %lu %lu %lu %lu %lu %lu %lu",
&pid, comm, &state,
&ppid, &pgrp, &session, &tty_nr, &tpgid,
&flags, &minflt, &cminflt, &majflt, &cmajflt, &utime, &stime);
free(comm);
if(rc != 15) {
crm_err("Only %d of 15 fields found in %s", rc, loadfile);
fclose(stream);
return FALSE;
} else if(last_call > 0
&& last_call < now
&& last_utime <= utime
&& last_stime <= stime) {
time_t elapsed = now - last_call;
unsigned long delta_utime = utime - last_utime;
unsigned long delta_stime = stime - last_stime;
*load = (delta_utime + delta_stime); /* Cast to a float before division */
*load /= ticks_per_s;
*load /= elapsed;
crm_debug("cib load: %f (%lu ticks in %lds)", *load, delta_utime + delta_stime, (long)elapsed);
} else {
crm_debug("Init %lu + %lu ticks at %ld (%lu tps)", utime, stime, (long)now, ticks_per_s);
}
last_call = now;
last_utime = utime;
last_stime = stime;
fclose(stream);
return TRUE;
}
fclose(stream);
return FALSE;
}
static bool
throttle_load_avg(float *load)
{
char buffer[256];
FILE *stream = NULL;
const char *loadfile = "/proc/loadavg";
if(load == NULL) {
return FALSE;
}
stream = fopen(loadfile, "r");
if(stream == NULL) {
int rc = errno;
crm_warn("Couldn't read %s: %s (%d)", loadfile, pcmk_rc_str(rc), rc);
return FALSE;
}
if(fgets(buffer, sizeof(buffer), stream)) {
char *nl = strstr(buffer, "\n");
/* Grab the 1-minute average, ignore the rest */
*load = strtof(buffer, NULL);
if(nl) { nl[0] = 0; }
fclose(stream);
return TRUE;
}
fclose(stream);
return FALSE;
}
/*!
* \internal
* \brief Check a load value against throttling thresholds
*
* \param[in] load Load value to check
* \param[in] desc Description of metric (for logging)
* \param[in] thresholds Low/medium/high/extreme thresholds
*
* \return Throttle mode corresponding to load value
*/
static enum throttle_state_e
throttle_check_thresholds(float load, const char *desc,
const float thresholds[4])
{
if (load > thresholds[3]) {
crm_notice("Extreme %s detected: %f", desc, load);
return throttle_extreme;
} else if (load > thresholds[2]) {
crm_notice("High %s detected: %f", desc, load);
return throttle_high;
} else if (load > thresholds[1]) {
crm_info("Moderate %s detected: %f", desc, load);
return throttle_med;
} else if (load > thresholds[0]) {
crm_debug("Noticeable %s detected: %f", desc, load);
return throttle_low;
}
crm_trace("Negligible %s detected: %f", desc, load);
return throttle_none;
}
static enum throttle_state_e
throttle_handle_load(float load, const char *desc, int cores)
{
float normalize;
float thresholds[4];
if (cores == 1) {
/* On a single core machine, a load of 1.0 is already too high */
normalize = 0.6;
} else {
/* Normalize the load to be per-core */
normalize = cores;
}
thresholds[0] = throttle_load_target * normalize * THROTTLE_FACTOR_LOW;
thresholds[1] = throttle_load_target * normalize * THROTTLE_FACTOR_MEDIUM;
thresholds[2] = throttle_load_target * normalize * THROTTLE_FACTOR_HIGH;
thresholds[3] = load + 1.0; /* never extreme */
return throttle_check_thresholds(load, desc, thresholds);
}
#endif // HAVE_LINUX_PROCFS
static enum throttle_state_e
throttle_mode(void)
{
enum throttle_state_e mode = throttle_none;
#if HAVE_LINUX_PROCFS
unsigned int cores;
float load;
float thresholds[4];
cores = pcmk__procfs_num_cores();
if(throttle_cib_load(&load)) {
float cib_max_cpu = 0.95;
/* The CIB is a single-threaded task and thus cannot consume
* more than 100% of a CPU (and 1/cores of the overall system
* load).
*
* On a many-cored system, the CIB might therefore be maxed out
* (causing operations to fail or appear to fail) even though
* the overall system load is still reasonable.
*
* Therefore, the 'normal' thresholds can not apply here, and we
* need a special case.
*/
if(cores == 1) {
cib_max_cpu = 0.4;
}
if(throttle_load_target > 0.0 && throttle_load_target < cib_max_cpu) {
cib_max_cpu = throttle_load_target;
}
thresholds[0] = cib_max_cpu * 0.8;
thresholds[1] = cib_max_cpu * 0.9;
thresholds[2] = cib_max_cpu;
/* Can only happen on machines with a low number of cores */
thresholds[3] = cib_max_cpu * 1.5;
mode = throttle_check_thresholds(load, "CIB load", thresholds);
}
if(throttle_load_target <= 0) {
/* If we ever make this a valid value, the cluster will at least behave as expected */
return mode;
}
if(throttle_load_avg(&load)) {
enum throttle_state_e cpu_load;
cpu_load = throttle_handle_load(load, "CPU load", cores);
if (cpu_load > mode) {
mode = cpu_load;
}
crm_debug("Current load is %f across %u core(s)", load, cores);
}
#endif // HAVE_LINUX_PROCFS
return mode;
}
static void
throttle_send_command(enum throttle_state_e mode)
{
xmlNode *xml = NULL;
static enum throttle_state_e last = -1;
if(mode != last) {
crm_info("New throttle mode: %s load (was %s)",
load2str(mode), load2str(last));
last = mode;
xml = create_request(CRM_OP_THROTTLE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
crm_xml_add_int(xml, F_CRM_THROTTLE_MODE, mode);
crm_xml_add_int(xml, F_CRM_THROTTLE_MAX, throttle_job_max);
send_cluster_message(NULL, crm_msg_crmd, xml, TRUE);
free_xml(xml);
}
}
static gboolean
throttle_timer_cb(gpointer data)
{
throttle_send_command(throttle_mode());
return TRUE;
}
static void
throttle_record_free(gpointer p)
{
struct throttle_record_s *r = p;
free(r->node);
free(r);
}
static void
throttle_set_load_target(float target)
{
throttle_load_target = target;
}
/*!
* \internal
* \brief Update the maximum number of simultaneous jobs
*
* \param[in] preference Cluster-wide node-action-limit from the CIB
*/
static void
throttle_update_job_max(const char *preference)
{
long long max = 0LL;
- const char *env_limit = getenv("PCMK_node_action_limit");
+ const char *env_limit = pcmk__env_option(PCMK__ENV_NODE_ACTION_LIMIT);
if (env_limit != NULL) {
preference = env_limit; // Per-node override
}
if (preference != NULL) {
pcmk__scan_ll(preference, &max, 0LL);
}
if (max > 0) {
throttle_job_max = (int) max;
} else {
// Default is based on the number of cores detected
throttle_job_max = 2 * pcmk__procfs_num_cores();
}
}
void
throttle_init(void)
{
if(throttle_records == NULL) {
throttle_records = pcmk__strkey_table(NULL, throttle_record_free);
throttle_timer = mainloop_timer_add("throttle", 30 * 1000, TRUE, throttle_timer_cb, NULL);
}
throttle_update_job_max(NULL);
mainloop_timer_start(throttle_timer);
}
/*!
* \internal
* \brief Configure throttle options based on the CIB
*
* \param[in,out] options Name/value pairs for configured options
*/
void
controld_configure_throttle(GHashTable *options)
{
const char *value = g_hash_table_lookup(options, "load-threshold");
if (value != NULL) {
throttle_set_load_target(strtof(value, NULL) / 100.0);
}
value = g_hash_table_lookup(options, "node-action-limit");
throttle_update_job_max(value);
}
void
throttle_fini(void)
{
if (throttle_timer != NULL) {
mainloop_timer_del(throttle_timer);
throttle_timer = NULL;
}
if (throttle_records != NULL) {
g_hash_table_destroy(throttle_records);
throttle_records = NULL;
}
}
int
throttle_get_total_job_limit(int l)
{
/* Cluster-wide limit */
GHashTableIter iter;
int limit = l;
int peers = crm_active_peers();
struct throttle_record_s *r = NULL;
g_hash_table_iter_init(&iter, throttle_records);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &r)) {
switch(r->mode) {
case throttle_extreme:
if(limit == 0 || limit > peers/4) {
limit = QB_MAX(1, peers/4);
}
break;
case throttle_high:
if(limit == 0 || limit > peers/2) {
limit = QB_MAX(1, peers/2);
}
break;
default:
break;
}
}
if(limit == l) {
/* crm_trace("No change to batch-limit=%d", limit); */
} else if(l == 0) {
crm_trace("Using batch-limit=%d", limit);
} else {
crm_trace("Using batch-limit=%d instead of %d", limit, l);
}
return limit;
}
int
throttle_get_job_limit(const char *node)
{
int jobs = 1;
struct throttle_record_s *r = NULL;
r = g_hash_table_lookup(throttle_records, node);
if(r == NULL) {
r = calloc(1, sizeof(struct throttle_record_s));
r->node = strdup(node);
r->mode = throttle_low;
r->max = throttle_job_max;
crm_trace("Defaulting to local values for unknown node %s", node);
g_hash_table_insert(throttle_records, r->node, r);
}
switch(r->mode) {
case throttle_extreme:
case throttle_high:
jobs = 1; /* At least one job must always be allowed */
break;
case throttle_med:
jobs = QB_MAX(1, r->max / 4);
break;
case throttle_low:
jobs = QB_MAX(1, r->max / 2);
break;
case throttle_none:
jobs = QB_MAX(1, r->max);
break;
default:
crm_err("Unknown throttle mode %.4x on %s", r->mode, node);
break;
}
return jobs;
}
void
throttle_update(xmlNode *xml)
{
int max = 0;
int mode = 0;
struct throttle_record_s *r = NULL;
const char *from = crm_element_value(xml, F_CRM_HOST_FROM);
crm_element_value_int(xml, F_CRM_THROTTLE_MODE, &mode);
crm_element_value_int(xml, F_CRM_THROTTLE_MAX, &max);
r = g_hash_table_lookup(throttle_records, from);
if(r == NULL) {
r = calloc(1, sizeof(struct throttle_record_s));
r->node = strdup(from);
g_hash_table_insert(throttle_records, r->node, r);
}
r->max = max;
r->mode = (enum throttle_state_e) mode;
crm_debug("Node %s has %s load and supports at most %d jobs; new job limit %d",
from, load2str((enum throttle_state_e) mode), max,
throttle_get_job_limit(from));
}
diff --git a/daemons/execd/pacemaker-execd.c b/daemons/execd/pacemaker-execd.c
index 83a8cd714e..e7e30eb009 100644
--- a/daemons/execd/pacemaker-execd.c
+++ b/daemons/execd/pacemaker-execd.c
@@ -1,582 +1,584 @@
/*
* Copyright 2012-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <glib.h>
#include <signal.h>
#include <sys/types.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/services.h>
#include <crm/common/cmdline_internal.h>
#include <crm/common/ipc.h>
#include <crm/common/ipc_internal.h>
#include <crm/common/mainloop.h>
#include <crm/common/output_internal.h>
#include <crm/common/remote_internal.h>
#include <crm/lrmd_internal.h>
#include "pacemaker-execd.h"
#ifdef PCMK__COMPILE_REMOTE
# define EXECD_TYPE "remote"
# define EXECD_NAME "pacemaker-remoted"
# define SUMMARY "resource agent executor daemon for Pacemaker Remote nodes"
#else
# define EXECD_TYPE "local"
# define EXECD_NAME "pacemaker-execd"
# define SUMMARY "resource agent executor daemon for Pacemaker cluster nodes"
#endif
static GMainLoop *mainloop = NULL;
static qb_ipcs_service_t *ipcs = NULL;
static stonith_t *stonith_api = NULL;
int lrmd_call_id = 0;
time_t start_time;
static struct {
gchar **log_files;
#ifdef PCMK__COMPILE_REMOTE
gchar *port;
#endif // PCMK__COMPILE_REMOTE
} options;
#ifdef PCMK__COMPILE_REMOTE
/* whether shutdown request has been sent */
static gboolean shutting_down = FALSE;
/* timer for waiting for acknowledgment of shutdown request */
static guint shutdown_ack_timer = 0;
static gboolean lrmd_exit(gpointer data);
#endif
static void
stonith_connection_destroy_cb(stonith_t * st, stonith_event_t * e)
{
stonith_api->state = stonith_disconnected;
stonith_connection_failed();
}
stonith_t *
get_stonith_connection(void)
{
if (stonith_api && stonith_api->state == stonith_disconnected) {
stonith_api_delete(stonith_api);
stonith_api = NULL;
}
if (stonith_api == NULL) {
int rc = pcmk_ok;
stonith_api = stonith_api_new();
if (stonith_api == NULL) {
crm_err("Could not connect to fencer: API memory allocation failed");
return NULL;
}
rc = stonith_api_connect_retry(stonith_api, crm_system_name, 10);
if (rc != pcmk_ok) {
crm_err("Could not connect to fencer in 10 attempts: %s "
CRM_XS " rc=%d", pcmk_strerror(rc), rc);
stonith_api_delete(stonith_api);
stonith_api = NULL;
} else {
stonith_api->cmds->register_notification(stonith_api,
T_STONITH_NOTIFY_DISCONNECT,
stonith_connection_destroy_cb);
}
}
return stonith_api;
}
static int32_t
lrmd_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
{
crm_trace("Connection %p", c);
if (pcmk__new_client(c, uid, gid) == NULL) {
return -EIO;
}
return 0;
}
static void
lrmd_ipc_created(qb_ipcs_connection_t * c)
{
pcmk__client_t *new_client = pcmk__find_client(c);
crm_trace("Connection %p", c);
CRM_ASSERT(new_client != NULL);
/* Now that the connection is offically established, alert
* the other clients a new connection exists. */
notify_of_new_client(new_client);
}
static int32_t
lrmd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size)
{
uint32_t id = 0;
uint32_t flags = 0;
pcmk__client_t *client = pcmk__find_client(c);
xmlNode *request = pcmk__client_data2xml(client, data, &id, &flags);
CRM_CHECK(client != NULL, crm_err("Invalid client");
return FALSE);
CRM_CHECK(client->id != NULL, crm_err("Invalid client: %p", client);
return FALSE);
CRM_CHECK(flags & crm_ipc_client_response, crm_err("Invalid client request: %p", client);
return FALSE);
if (!request) {
return 0;
}
if (!client->name) {
const char *value = crm_element_value(request, F_LRMD_CLIENTNAME);
if (value == NULL) {
client->name = pcmk__itoa(pcmk__client_pid(c));
} else {
client->name = strdup(value);
}
}
lrmd_call_id++;
if (lrmd_call_id < 1) {
lrmd_call_id = 1;
}
crm_xml_add(request, F_LRMD_CLIENTID, client->id);
crm_xml_add(request, F_LRMD_CLIENTNAME, client->name);
crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id);
process_lrmd_message(client, id, request);
free_xml(request);
return 0;
}
/*!
* \internal
* \brief Free a client connection, and exit if appropriate
*
* \param[in,out] client Client connection to free
*/
void
lrmd_client_destroy(pcmk__client_t *client)
{
pcmk__free_client(client);
#ifdef PCMK__COMPILE_REMOTE
/* If we were waiting to shut down, we can now safely do so
* if there are no more proxied IPC providers
*/
if (shutting_down && (ipc_proxy_get_provider() == NULL)) {
lrmd_exit(NULL);
}
#endif
}
static int32_t
lrmd_ipc_closed(qb_ipcs_connection_t * c)
{
pcmk__client_t *client = pcmk__find_client(c);
if (client == NULL) {
return 0;
}
crm_trace("Connection %p", c);
client_disconnect_cleanup(client->id);
#ifdef PCMK__COMPILE_REMOTE
ipc_proxy_remove_provider(client);
#endif
lrmd_client_destroy(client);
return 0;
}
static void
lrmd_ipc_destroy(qb_ipcs_connection_t * c)
{
lrmd_ipc_closed(c);
crm_trace("Connection %p", c);
}
static struct qb_ipcs_service_handlers lrmd_ipc_callbacks = {
.connection_accept = lrmd_ipc_accept,
.connection_created = lrmd_ipc_created,
.msg_process = lrmd_ipc_dispatch,
.connection_closed = lrmd_ipc_closed,
.connection_destroyed = lrmd_ipc_destroy
};
// \return Standard Pacemaker return code
int
lrmd_server_send_reply(pcmk__client_t *client, uint32_t id, xmlNode *reply)
{
crm_trace("Sending reply (%d) to client (%s)", id, client->id);
switch (PCMK__CLIENT_TYPE(client)) {
case pcmk__client_ipc:
return pcmk__ipc_send_xml(client, id, reply, FALSE);
#ifdef PCMK__COMPILE_REMOTE
case pcmk__client_tls:
return lrmd__remote_send_xml(client->remote, reply, id, "reply");
#endif
default:
crm_err("Could not send reply: unknown type for client %s "
CRM_XS " flags=%#llx",
pcmk__client_name(client), client->flags);
}
return ENOTCONN;
}
// \return Standard Pacemaker return code
int
lrmd_server_send_notify(pcmk__client_t *client, xmlNode *msg)
{
crm_trace("Sending notification to client (%s)", client->id);
switch (PCMK__CLIENT_TYPE(client)) {
case pcmk__client_ipc:
if (client->ipcs == NULL) {
crm_trace("Could not notify local client: disconnected");
return ENOTCONN;
}
return pcmk__ipc_send_xml(client, 0, msg, crm_ipc_server_event);
#ifdef PCMK__COMPILE_REMOTE
case pcmk__client_tls:
if (client->remote == NULL) {
crm_trace("Could not notify remote client: disconnected");
return ENOTCONN;
} else {
return lrmd__remote_send_xml(client->remote, msg, 0, "notify");
}
#endif
default:
crm_err("Could not notify client %s with unknown transport "
CRM_XS " flags=%#llx",
pcmk__client_name(client), client->flags);
}
return ENOTCONN;
}
/*!
* \internal
* \brief Clean up and exit immediately
*
* \param[in] data Ignored
*
* \return Doesn't return
* \note This can be used as a timer callback.
*/
static gboolean
lrmd_exit(gpointer data)
{
crm_info("Terminating with %d clients", pcmk__ipc_client_count());
if (stonith_api) {
stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT);
stonith_api->cmds->disconnect(stonith_api);
stonith_api_delete(stonith_api);
}
if (ipcs) {
mainloop_del_ipc_server(ipcs);
}
#ifdef PCMK__COMPILE_REMOTE
execd_stop_tls_server();
ipc_proxy_cleanup();
#endif
pcmk__client_cleanup();
g_hash_table_destroy(rsc_list);
if (mainloop) {
lrmd_drain_alerts(mainloop);
}
crm_exit(CRM_EX_OK);
return FALSE;
}
/*!
* \internal
* \brief Request cluster shutdown if appropriate, otherwise exit immediately
*
* \param[in] nsig Signal that caused invocation (ignored)
*/
static void
lrmd_shutdown(int nsig)
{
#ifdef PCMK__COMPILE_REMOTE
pcmk__client_t *ipc_proxy = ipc_proxy_get_provider();
/* If there are active proxied IPC providers, then we may be running
* resources, so notify the cluster that we wish to shut down.
*/
if (ipc_proxy) {
if (shutting_down) {
crm_notice("Waiting for cluster to stop resources before exiting");
return;
}
crm_info("Sending shutdown request to cluster");
if (ipc_proxy_shutdown_req(ipc_proxy) < 0) {
crm_crit("Shutdown request failed, exiting immediately");
} else {
/* We requested a shutdown. Now, we need to wait for an
* acknowledgement from the proxy host (which ensures the proxy host
* supports shutdown requests), then wait for all proxy hosts to
* disconnect (which ensures that all resources have been stopped).
*/
shutting_down = TRUE;
/* Stop accepting new proxy connections */
execd_stop_tls_server();
/* Older controller versions will never acknowledge our request, so
* set a fairly short timeout to exit quickly in that case. If we
* get the ack, we'll defuse this timer.
*/
shutdown_ack_timer = g_timeout_add_seconds(20, lrmd_exit, NULL);
/* Currently, we let the OS kill us if the clients don't disconnect
* in a reasonable time. We could instead set a long timer here
* (shorter than what the OS is likely to use) and exit immediately
* if it pops.
*/
return;
}
}
#endif
lrmd_exit(NULL);
}
/*!
* \internal
* \brief Defuse short exit timer if shutting down
*/
void
handle_shutdown_ack(void)
{
#ifdef PCMK__COMPILE_REMOTE
if (shutting_down) {
crm_info("Received shutdown ack");
if (shutdown_ack_timer > 0) {
g_source_remove(shutdown_ack_timer);
shutdown_ack_timer = 0;
}
return;
}
#endif
crm_debug("Ignoring unexpected shutdown ack");
}
/*!
* \internal
* \brief Make short exit timer fire immediately
*/
void
handle_shutdown_nack(void)
{
#ifdef PCMK__COMPILE_REMOTE
if (shutting_down) {
crm_info("Received shutdown nack");
if (shutdown_ack_timer > 0) {
g_source_remove(shutdown_ack_timer);
shutdown_ack_timer = g_timeout_add(0, lrmd_exit, NULL);
}
return;
}
#endif
crm_debug("Ignoring unexpected shutdown nack");
}
static GOptionEntry entries[] = {
{ "logfile", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME_ARRAY,
&options.log_files, "Send logs to the additional named logfile", NULL },
#ifdef PCMK__COMPILE_REMOTE
{ "port", 'p', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.port,
"Port to listen on (defaults to " G_STRINGIFY(DEFAULT_REMOTE_PORT) ")", NULL },
#endif // PCMK__COMPILE_REMOTE
{ NULL }
};
static pcmk__supported_format_t formats[] = {
PCMK__SUPPORTED_FORMAT_NONE,
PCMK__SUPPORTED_FORMAT_TEXT,
PCMK__SUPPORTED_FORMAT_XML,
{ NULL, NULL, NULL }
};
static GOptionContext *
build_arg_context(pcmk__common_args_t *args, GOptionGroup **group)
{
GOptionContext *context = NULL;
context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
pcmk__add_main_args(context, entries);
return context;
}
int
main(int argc, char **argv, char **envp)
{
int rc = pcmk_rc_ok;
crm_exit_t exit_code = CRM_EX_OK;
const char *option = NULL;
pcmk__output_t *out = NULL;
GError *error = NULL;
GOptionGroup *output_group = NULL;
pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
#ifdef PCMK__COMPILE_REMOTE
gchar **processed_args = pcmk__cmdline_preproc(argv, "lp");
#else
gchar **processed_args = pcmk__cmdline_preproc(argv, "l");
#endif // PCMK__COMPILE_REMOTE
GOptionContext *context = build_arg_context(args, &output_group);
#ifdef PCMK__COMPILE_REMOTE
// If necessary, create PID 1 now before any file descriptors are opened
remoted_spawn_pidone(argc, argv, envp);
#endif
crm_log_preinit(EXECD_NAME, argc, argv);
pcmk__register_formats(output_group, formats);
if (!g_option_context_parse_strv(context, &processed_args, &error)) {
exit_code = CRM_EX_USAGE;
goto done;
}
rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
if (rc != pcmk_rc_ok) {
exit_code = CRM_EX_ERROR;
g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
"Error creating output format %s: %s",
args->output_ty, pcmk_rc_str(rc));
goto done;
}
if (args->version) {
out->version(out, false);
goto done;
}
// Open additional log files
if (options.log_files != NULL) {
for (gchar **fname = options.log_files; *fname != NULL; fname++) {
rc = pcmk__add_logfile(*fname);
if (rc != pcmk_rc_ok) {
out->err(out, "Logging to %s is disabled: %s",
*fname, pcmk_rc_str(rc));
}
}
}
pcmk__cli_init_logging(EXECD_NAME, args->verbosity);
crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
+ // ocf_log() (in resource-agents) uses the capitalized env options below
option = pcmk__env_option(PCMK__ENV_LOGFACILITY);
if (!pcmk__str_eq(option, PCMK__VALUE_NONE,
pcmk__str_casei|pcmk__str_null_matches)
&& !pcmk__str_eq(option, "/dev/null", pcmk__str_none)) {
- setenv("HA_LOGFACILITY", option, 1); /* Used by the ocf_log/ha_log OCF macro */
+
+ pcmk__set_env_option("LOGFACILITY", option, true);
}
option = pcmk__env_option(PCMK__ENV_LOGFILE);
if (!pcmk__str_eq(option, PCMK__VALUE_NONE,
pcmk__str_casei|pcmk__str_null_matches)) {
- setenv("HA_LOGFILE", option, 1); /* Used by the ocf_log/ha_log OCF macro */
+ pcmk__set_env_option("LOGFILE", option, true);
if (pcmk__env_option_enabled(crm_system_name, PCMK__ENV_DEBUG)) {
- setenv("HA_DEBUGLOG", option, 1); /* Used by the ocf_log/ha_debug OCF macro */
+ pcmk__set_env_option("DEBUGLOG", option, true);
}
}
#ifdef PCMK__COMPILE_REMOTE
if (options.port != NULL) {
- setenv("PCMK_remote_port", options.port, 1);
+ pcmk__set_env_option(PCMK__ENV_REMOTE_PORT, options.port, false);
}
#endif // PCMK__COMPILE_REMOTE
start_time = time(NULL);
crm_notice("Starting Pacemaker " EXECD_TYPE " executor");
/* The presence of this variable allegedly controls whether child
* processes like httpd will try and use Systemd's sd_notify
* API
*/
unsetenv("NOTIFY_SOCKET");
{
// Temporary directory for resource agent use (leave owned by root)
int rc = pcmk__build_path(CRM_RSCTMP_DIR, 0755);
if (rc != pcmk_rc_ok) {
crm_warn("Could not create resource agent temporary directory "
CRM_RSCTMP_DIR ": %s", pcmk_rc_str(rc));
}
}
rsc_list = pcmk__strkey_table(NULL, free_rsc);
ipcs = mainloop_add_ipc_server(CRM_SYSTEM_LRMD, QB_IPC_SHM, &lrmd_ipc_callbacks);
if (ipcs == NULL) {
crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
exit_code = CRM_EX_FATAL;
goto done;
}
#ifdef PCMK__COMPILE_REMOTE
if (lrmd_init_remote_tls_server() < 0) {
crm_err("Failed to create TLS listener: shutting down and staying down");
exit_code = CRM_EX_FATAL;
goto done;
}
ipc_proxy_init();
#endif
mainloop_add_signal(SIGTERM, lrmd_shutdown);
mainloop = g_main_loop_new(NULL, FALSE);
crm_notice("Pacemaker " EXECD_TYPE " executor successfully started and accepting connections");
crm_notice("OCF resource agent search path is %s", OCF_RA_PATH);
g_main_loop_run(mainloop);
/* should never get here */
lrmd_exit(NULL);
done:
g_strfreev(options.log_files);
#ifdef PCMK__COMPILE_REMOTE
g_free(options.port);
#endif // PCMK__COMPILE_REMOTE
g_strfreev(processed_args);
pcmk__free_arg_context(context);
pcmk__output_and_clear_error(&error, out);
if (out != NULL) {
out->finish(out, exit_code, true, NULL);
pcmk__output_free(out);
}
pcmk__unregister_formats();
crm_exit(exit_code);
}
diff --git a/daemons/execd/remoted_pidone.c b/daemons/execd/remoted_pidone.c
index 47a891ba03..96fee90c07 100644
--- a/daemons/execd/remoted_pidone.c
+++ b/daemons/execd/remoted_pidone.c
@@ -1,298 +1,301 @@
/*
- * Copyright 2017-2020 the Pacemaker project contributors
+ * Copyright 2017-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <stdio.h>
#include <ctype.h>
#include <stdlib.h>
#include <signal.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <crm/crm.h>
#include "pacemaker-execd.h"
static pid_t main_pid = 0;
static void
sigdone(void)
{
exit(CRM_EX_OK);
}
static void
sigreap(void)
{
pid_t pid = 0;
int status;
do {
/*
* Opinions seem to differ as to what to put here:
* -1, any child process
* 0, any child process whose process group ID is equal to that of the calling process
*/
pid = waitpid(-1, &status, WNOHANG);
if (pid == main_pid) {
/* Exit when pacemaker-remote exits and use the same return code */
if (WIFEXITED(status)) {
exit(WEXITSTATUS(status));
}
exit(CRM_EX_ERROR);
}
} while (pid > 0);
}
static struct {
int sig;
void (*handler)(void);
} sigmap[] = {
{ SIGCHLD, sigreap },
{ SIGINT, sigdone },
};
/*!
* \internal
* \brief Check a line of text for a valid environment variable name
*
* \param[in] line Text to check
* \param[out] first First character of valid name if found, NULL otherwise
* \param[out] last Last character of valid name if found, NULL otherwise
*
* \return TRUE if valid name found, FALSE otherwise
* \note It's reasonable to impose limitations on environment variable names
* beyond what C or setenv() does: We only allow names that contain only
* [a-zA-Z0-9_] characters and do not start with a digit.
*/
static bool
find_env_var_name(char *line, char **first, char **last)
{
// Skip leading whitespace
*first = line;
while (isspace(**first)) {
++*first;
}
if (isalpha(**first) || (**first == '_')) { // Valid first character
*last = *first;
while (isalnum(*(*last + 1)) || (*(*last + 1) == '_')) {
++*last;
}
return TRUE;
}
*first = *last = NULL;
return FALSE;
}
static void
load_env_vars(const char *filename)
{
/* We haven't forked or initialized logging yet, so don't leave any file
* descriptors open, and don't log -- silently ignore errors.
*/
FILE *fp = fopen(filename, "r");
if (fp != NULL) {
char line[LINE_MAX] = { '\0', };
while (fgets(line, LINE_MAX, fp) != NULL) {
char *name = NULL;
char *end = NULL;
char *value = NULL;
char *quote = NULL;
// Look for valid name immediately followed by equals sign
if (find_env_var_name(line, &name, &end) && (*++end == '=')) {
// Null-terminate name, and advance beyond equals sign
*end++ = '\0';
// Check whether value is quoted
if ((*end == '\'') || (*end == '"')) {
quote = end++;
}
value = end;
if (quote) {
/* Value is remaining characters up to next non-backslashed
* matching quote character.
*/
while (((*end != *quote) || (*(end - 1) == '\\'))
&& (*end != '\0')) {
end++;
}
if (*end == *quote) {
// Null-terminate value, and advance beyond close quote
*end++ = '\0';
} else {
// Matching closing quote wasn't found
value = NULL;
}
} else {
/* Value is remaining characters up to next non-backslashed
* whitespace.
*/
while ((!isspace(*end) || (*(end - 1) == '\\'))
&& (*end != '\0')) {
++end;
}
if (end == (line + LINE_MAX - 1)) {
// Line was too long
value = NULL;
}
// Do NOT null-terminate value (yet)
}
/* We have a valid name and value, and end is now the character
* after the closing quote or the first whitespace after the
* unquoted value. Make sure the rest of the line is just
* whitespace or a comment.
*/
if (value) {
char *value_end = end;
while (isspace(*end) && (*end != '\n')) {
++end;
}
if ((*end == '\n') || (*end == '#')) {
if (quote == NULL) {
// Now we can null-terminate an unquoted value
*value_end = '\0';
}
// Don't overwrite (bundle options take precedence)
setenv(name, value, 0);
} else {
value = NULL;
}
}
}
if ((value == NULL) && (strchr(line, '\n') == NULL)) {
// Eat remainder of line beyond LINE_MAX
if (fscanf(fp, "%*[^\n]\n") == EOF) {
value = NULL; // Don't care, make compiler happy
}
}
}
fclose(fp);
}
}
void
remoted_spawn_pidone(int argc, char **argv, char **envp)
{
sigset_t set;
/* This environment variable exists for two purposes:
* - For testing, setting it to "full" enables full PID 1 behavior even
* when PID is not 1
* - Setting to "vars" enables just the loading of environment variables
* from /etc/pacemaker/pcmk-init.env, which could be useful for testing or
* containers with a custom PID 1 script that launches pacemaker-remoted.
*/
- const char *pid1 = (getpid() == 1)? "full" : getenv("PCMK_remote_pid1");
+ const char *pid1 = "full";
- if (pid1 == NULL) {
- return;
+ if (getpid() != 1) {
+ pid1 = pcmk__env_option(PCMK__ENV_REMOTE_PID1);
+ if (pid1 == NULL) {
+ return;
+ }
}
/* When a container is launched, it may be given specific environment
* variables, which for Pacemaker bundles are given in the bundle
* configuration. However, that does not allow for host-specific values.
* To allow for that, look for a special file containing a shell-like syntax
* of name/value pairs, and export those into the environment.
*/
load_env_vars("/etc/pacemaker/pcmk-init.env");
if (strcmp(pid1, "full")) {
return;
}
/* Containers can be expected to have /var/log, but they may not have
* /var/log/pacemaker, so use a different default if no value has been
* explicitly configured in the container's environment.
*/
if (pcmk__env_option(PCMK__ENV_LOGFILE) == NULL) {
- pcmk__set_env_option(PCMK__ENV_LOGFILE, "/var/log/pcmk-init.log");
+ pcmk__set_env_option(PCMK__ENV_LOGFILE, "/var/log/pcmk-init.log", true);
}
sigfillset(&set);
sigprocmask(SIG_BLOCK, &set, 0);
main_pid = fork();
switch (main_pid) {
case 0:
sigprocmask(SIG_UNBLOCK, &set, NULL);
setsid();
setpgid(0, 0);
// Child remains as pacemaker-remoted
return;
case -1:
crm_err("fork failed: %s", pcmk_rc_str(errno));
}
/* Parent becomes the reaper of zombie processes */
/* Safe to initialize logging now if needed */
# ifdef HAVE_PROGNAME
/* Differentiate ourselves in the 'ps' output */
{
char *p;
int i, maxlen;
char *LastArgv = NULL;
const char *name = "pcmk-init";
for (i = 0; i < argc; i++) {
if (!i || (LastArgv + 1 == argv[i]))
LastArgv = argv[i] + strlen(argv[i]);
}
for (i = 0; envp[i] != NULL; i++) {
if ((LastArgv + 1) == envp[i]) {
LastArgv = envp[i] + strlen(envp[i]);
}
}
maxlen = (LastArgv - argv[0]) - 2;
i = strlen(name);
/* We can overwrite individual argv[] arguments */
snprintf(argv[0], maxlen, "%s", name);
/* Now zero out everything else */
p = &argv[0][i];
while (p < LastArgv) {
*p++ = '\0';
}
argv[1] = NULL;
}
# endif // HAVE_PROGNAME
while (1) {
int sig;
size_t i;
sigwait(&set, &sig);
for (i = 0; i < PCMK__NELEM(sigmap); i++) {
if (sigmap[i].sig == sig) {
sigmap[i].handler();
break;
}
}
}
}
diff --git a/daemons/execd/remoted_tls.c b/daemons/execd/remoted_tls.c
index 79281eb0e8..23a2dcf456 100644
--- a/daemons/execd/remoted_tls.c
+++ b/daemons/execd/remoted_tls.c
@@ -1,436 +1,436 @@
/*
* Copyright 2012-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <glib.h>
#include <unistd.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/mainloop.h>
#include <crm/common/remote_internal.h>
#include <crm/lrmd_internal.h>
#include <netdb.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#include "pacemaker-execd.h"
#ifdef HAVE_GNUTLS_GNUTLS_H
# include <gnutls/gnutls.h>
# define LRMD_REMOTE_AUTH_TIMEOUT 10000
gnutls_psk_server_credentials_t psk_cred_s;
gnutls_dh_params_t dh_params;
static int ssock = -1;
extern int lrmd_call_id;
static void
debug_log(int level, const char *str)
{
fputs(str, stderr);
}
/*!
* \internal
* \brief Read (more) TLS handshake data from client
*
* \param[in,out] client IPC client doing handshake
*
* \return 0 on success or more data needed, -1 on error
*/
static int
remoted__read_handshake_data(pcmk__client_t *client)
{
int rc = pcmk__read_handshake_data(client);
if (rc == EAGAIN) {
/* No more data is available at the moment. Just return for now;
* we'll get invoked again once the client sends more.
*/
return 0;
} else if (rc != pcmk_rc_ok) {
return -1;
}
if (client->remote->auth_timeout) {
g_source_remove(client->remote->auth_timeout);
}
client->remote->auth_timeout = 0;
pcmk__set_client_flags(client, pcmk__client_tls_handshake_complete);
crm_notice("Remote client connection accepted");
/* Only a client with access to the TLS key can connect, so we can treat
* it as privileged.
*/
pcmk__set_client_flags(client, pcmk__client_privileged);
// Alert other clients of the new connection
notify_of_new_client(client);
return 0;
}
static int
lrmd_remote_client_msg(gpointer data)
{
int id = 0;
int rc;
xmlNode *request = NULL;
pcmk__client_t *client = data;
if (!pcmk_is_set(client->flags,
pcmk__client_tls_handshake_complete)) {
return remoted__read_handshake_data(client);
}
switch (pcmk__remote_ready(client->remote, 0)) {
case pcmk_rc_ok:
break;
case ETIME: // No message available to read
return 0;
default: // Error
crm_info("Remote client disconnected while polling it");
return -1;
}
rc = pcmk__read_remote_message(client->remote, -1);
request = pcmk__remote_message_xml(client->remote);
while (request) {
crm_element_value_int(request, F_LRMD_REMOTE_MSG_ID, &id);
crm_trace("Processing remote client request %d", id);
if (!client->name) {
const char *value = crm_element_value(request, F_LRMD_CLIENTNAME);
if (value) {
client->name = strdup(value);
}
}
lrmd_call_id++;
if (lrmd_call_id < 1) {
lrmd_call_id = 1;
}
crm_xml_add(request, F_LRMD_CLIENTID, client->id);
crm_xml_add(request, F_LRMD_CLIENTNAME, client->name);
crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id);
process_lrmd_message(client, id, request);
free_xml(request);
/* process all the messages in the current buffer */
request = pcmk__remote_message_xml(client->remote);
}
if (rc == ENOTCONN) {
crm_info("Remote client disconnected while reading from it");
return -1;
}
return 0;
}
static void
lrmd_remote_client_destroy(gpointer user_data)
{
pcmk__client_t *client = user_data;
if (client == NULL) {
return;
}
crm_notice("Cleaning up after remote client %s disconnected",
pcmk__client_name(client));
ipc_proxy_remove_provider(client);
/* if this is the last remote connection, stop recurring
* operations */
if (pcmk__ipc_client_count() == 1) {
client_disconnect_cleanup(NULL);
}
if (client->remote->tls_session) {
void *sock_ptr;
int csock;
sock_ptr = gnutls_transport_get_ptr(*client->remote->tls_session);
csock = GPOINTER_TO_INT(sock_ptr);
gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_RDWR);
gnutls_deinit(*client->remote->tls_session);
gnutls_free(client->remote->tls_session);
close(csock);
}
lrmd_client_destroy(client);
return;
}
static gboolean
lrmd_auth_timeout_cb(gpointer data)
{
pcmk__client_t *client = data;
client->remote->auth_timeout = 0;
if (pcmk_is_set(client->flags,
pcmk__client_tls_handshake_complete)) {
return FALSE;
}
mainloop_del_fd(client->remote->source);
client->remote->source = NULL;
crm_err("Remote client authentication timed out");
return FALSE;
}
// Dispatch callback for remote server socket
static int
lrmd_remote_listen(gpointer data)
{
int csock = -1;
gnutls_session_t *session = NULL;
pcmk__client_t *new_client = NULL;
// For client socket
static struct mainloop_fd_callbacks lrmd_remote_fd_cb = {
.dispatch = lrmd_remote_client_msg,
.destroy = lrmd_remote_client_destroy,
};
CRM_CHECK(ssock >= 0, return TRUE);
if (pcmk__accept_remote_connection(ssock, &csock) != pcmk_rc_ok) {
return TRUE;
}
session = pcmk__new_tls_session(csock, GNUTLS_SERVER, GNUTLS_CRD_PSK,
psk_cred_s);
if (session == NULL) {
close(csock);
return TRUE;
}
new_client = pcmk__new_unauth_client(NULL);
new_client->remote = calloc(1, sizeof(pcmk__remote_t));
pcmk__set_client_flags(new_client, pcmk__client_tls);
new_client->remote->tls_session = session;
// Require the client to authenticate within this time
new_client->remote->auth_timeout = g_timeout_add(LRMD_REMOTE_AUTH_TIMEOUT,
lrmd_auth_timeout_cb,
new_client);
crm_info("Remote client pending authentication "
CRM_XS " %p id: %s", new_client, new_client->id);
new_client->remote->source =
mainloop_add_fd("pacemaker-remote-client", G_PRIORITY_DEFAULT, csock,
new_client, &lrmd_remote_fd_cb);
return TRUE;
}
static void
tls_server_dropped(gpointer user_data)
{
crm_notice("TLS server session ended");
return;
}
// \return 0 on success, -1 on error (gnutls_psk_server_credentials_function)
static int
lrmd_tls_server_key_cb(gnutls_session_t session, const char *username, gnutls_datum_t * key)
{
return (lrmd__init_remote_key(key) == pcmk_rc_ok)? 0 : -1;
}
static int
bind_and_listen(struct addrinfo *addr)
{
int optval;
int fd;
int rc;
char buffer[INET6_ADDRSTRLEN] = { 0, };
pcmk__sockaddr2str(addr->ai_addr, buffer);
crm_trace("Attempting to bind to address %s", buffer);
fd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
if (fd < 0) {
rc = errno;
crm_err("Listener socket creation failed: %", pcmk_rc_str(rc));
return -rc;
}
/* reuse address */
optval = 1;
rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
if (rc < 0) {
rc = errno;
crm_err("Local address reuse not allowed on %s: %s", buffer, pcmk_rc_str(rc));
close(fd);
return -rc;
}
if (addr->ai_family == AF_INET6) {
optval = 0;
rc = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &optval, sizeof(optval));
if (rc < 0) {
rc = errno;
crm_err("Couldn't disable IPV6-only on %s: %s", buffer, pcmk_rc_str(rc));
close(fd);
return -rc;
}
}
if (bind(fd, addr->ai_addr, addr->ai_addrlen) != 0) {
rc = errno;
crm_err("Cannot bind to %s: %s", buffer, pcmk_rc_str(rc));
close(fd);
return -rc;
}
if (listen(fd, 10) == -1) {
rc = errno;
crm_err("Cannot listen on %s: %s", buffer, pcmk_rc_str(rc));
close(fd);
return -rc;
}
return fd;
}
static int
get_address_info(const char *bind_name, int port, struct addrinfo **res)
{
int rc;
char port_str[6]; // at most "65535"
struct addrinfo hints;
memset(&hints, 0, sizeof(struct addrinfo));
hints.ai_flags = AI_PASSIVE;
hints.ai_family = AF_UNSPEC; // IPv6 or IPv4
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
snprintf(port_str, sizeof(port_str), "%d", port);
rc = getaddrinfo(bind_name, port_str, &hints, res);
rc = pcmk__gaierror2rc(rc);
if (rc != pcmk_rc_ok) {
crm_err("Unable to get IP address(es) for %s: %s",
(bind_name? bind_name : "local node"), pcmk_rc_str(rc));
return rc;
}
return pcmk_rc_ok;
}
int
lrmd_init_remote_tls_server(void)
{
int filter;
int port = crm_default_remote_port();
struct addrinfo *res = NULL, *iter;
gnutls_datum_t psk_key = { NULL, 0 };
- const char *bind_name = getenv("PCMK_remote_address");
+ const char *bind_name = pcmk__env_option(PCMK__ENV_REMOTE_ADDRESS);
static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
.dispatch = lrmd_remote_listen,
.destroy = tls_server_dropped,
};
CRM_CHECK(ssock == -1, return ssock);
crm_debug("Starting TLS listener on %s port %d",
(bind_name? bind_name : "all addresses on"), port);
crm_gnutls_global_init();
gnutls_global_set_log_function(debug_log);
if (pcmk__init_tls_dh(&dh_params) != pcmk_rc_ok) {
return -1;
}
gnutls_psk_allocate_server_credentials(&psk_cred_s);
gnutls_psk_set_server_credentials_function(psk_cred_s, lrmd_tls_server_key_cb);
gnutls_psk_set_server_dh_params(psk_cred_s, dh_params);
/* The key callback won't get called until the first client connection
* attempt. Do it once here, so we can warn the user at start-up if we can't
* read the key. We don't error out, though, because it's fine if the key is
* going to be added later.
*/
if (lrmd__init_remote_key(&psk_key) != pcmk_rc_ok) {
crm_warn("A cluster connection will not be possible until the key is available");
}
gnutls_free(psk_key.data);
if (get_address_info(bind_name, port, &res) != pcmk_rc_ok) {
return -1;
}
/* Currently we listen on only one address from the resulting list (the
* first IPv6 address we can bind to if possible, otherwise the first IPv4
* address we can bind to). When bind_name is NULL, this should be the
* respective wildcard address.
*
* @TODO If there is demand for specifying more than one address, allow
* bind_name to be a space-separated list, call getaddrinfo() for each,
* and create a socket for each result (set IPV6_V6ONLY on IPv6 sockets
* since IPv4 listeners will have their own sockets).
*/
iter = res;
filter = AF_INET6;
while (iter) {
if (iter->ai_family == filter) {
ssock = bind_and_listen(iter);
}
if (ssock >= 0) {
break;
}
iter = iter->ai_next;
if (iter == NULL && filter == AF_INET6) {
iter = res;
filter = AF_INET;
}
}
if (ssock >= 0) {
mainloop_add_fd("pacemaker-remote-server", G_PRIORITY_DEFAULT, ssock,
NULL, &remote_listen_fd_callbacks);
crm_debug("Started TLS listener on %s port %d",
(bind_name? bind_name : "all addresses on"), port);
}
freeaddrinfo(res);
return ssock;
}
void
execd_stop_tls_server(void)
{
if (psk_cred_s) {
gnutls_psk_free_server_credentials(psk_cred_s);
psk_cred_s = 0;
}
if (ssock >= 0) {
close(ssock);
ssock = -1;
}
}
#endif
diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c
index 6a14fa4871..365b743d21 100644
--- a/daemons/pacemakerd/pacemakerd.c
+++ b/daemons/pacemakerd/pacemakerd.c
@@ -1,487 +1,485 @@
/*
* Copyright 2010-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include "pacemakerd.h"
#include <pwd.h>
#include <errno.h>
#include <unistd.h>
#include <stdio.h>
#include <stdbool.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <crm/crm.h> /* indirectly: CRM_EX_* */
#include <crm/msg_xml.h>
#include <crm/common/mainloop.h>
#include <crm/common/cmdline_internal.h>
#include <crm/common/ipc_pacemakerd.h>
#include <crm/common/output_internal.h>
#include <crm/cluster/internal.h>
#include <crm/cluster.h>
#define SUMMARY "pacemakerd - primary Pacemaker daemon that launches and monitors all subsidiary Pacemaker daemons"
struct {
gboolean features;
gboolean foreground;
gboolean shutdown;
gboolean standby;
} options;
static pcmk__output_t *out = NULL;
static pcmk__supported_format_t formats[] = {
PCMK__SUPPORTED_FORMAT_NONE,
PCMK__SUPPORTED_FORMAT_TEXT,
PCMK__SUPPORTED_FORMAT_XML,
{ NULL, NULL, NULL }
};
PCMK__OUTPUT_ARGS("features")
static int
pacemakerd_features(pcmk__output_t *out, va_list args) {
out->info(out, "Pacemaker %s (Build: %s)\n Supporting v%s: %s", PACEMAKER_VERSION,
BUILD_VERSION, CRM_FEATURE_SET, CRM_FEATURES);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("features")
static int
pacemakerd_features_xml(pcmk__output_t *out, va_list args) {
gchar **feature_list = g_strsplit(CRM_FEATURES, " ", 0);
pcmk__output_xml_create_parent(out, "pacemakerd",
"version", PACEMAKER_VERSION,
"build", BUILD_VERSION,
"feature_set", CRM_FEATURE_SET,
NULL);
out->begin_list(out, NULL, NULL, "features");
for (char **s = feature_list; *s != NULL; s++) {
pcmk__output_create_xml_text_node(out, "feature", *s);
}
out->end_list(out);
g_strfreev(feature_list);
return pcmk_rc_ok;
}
static pcmk__message_entry_t fmt_functions[] = {
{ "features", "default", pacemakerd_features },
{ "features", "xml", pacemakerd_features_xml },
{ NULL, NULL, NULL }
};
static gboolean
pid_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
return TRUE;
}
static gboolean
standby_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
options.standby = TRUE;
- pcmk__set_env_option("node_start_state", "standby");
+ pcmk__set_env_option(PCMK__ENV_NODE_START_STATE, "standby", false);
return TRUE;
}
static GOptionEntry entries[] = {
{ "features", 'F', 0, G_OPTION_ARG_NONE, &options.features,
"Display full version and list of features Pacemaker was built with",
NULL },
{ "foreground", 'f', 0, G_OPTION_ARG_NONE, &options.foreground,
"(Ignored) Pacemaker always runs in the foreground",
NULL },
{ "pid-file", 'p', 0, G_OPTION_ARG_CALLBACK, pid_cb,
"(Ignored) Daemon pid file location",
"FILE" },
{ "shutdown", 'S', 0, G_OPTION_ARG_NONE, &options.shutdown,
"Instruct Pacemaker to shutdown on this machine",
NULL },
{ "standby", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, standby_cb,
"Start node in standby state",
NULL },
{ NULL }
};
static void
pcmk_ignore(int nsig)
{
crm_info("Ignoring signal %s (%d)", strsignal(nsig), nsig);
}
static void
pcmk_sigquit(int nsig)
{
pcmk__panic(__func__);
}
static void
-mcp_chown(const char *path, uid_t uid, gid_t gid)
+pacemakerd_chown(const char *path, uid_t uid, gid_t gid)
{
int rc = chown(path, uid, gid);
if (rc < 0) {
crm_warn("Cannot change the ownership of %s to user %s and gid %d: %s",
path, CRM_DAEMON_USER, gid, pcmk_rc_str(errno));
}
}
static void
create_pcmk_dirs(void)
{
uid_t pcmk_uid = 0;
gid_t pcmk_gid = 0;
const char *dirs[] = {
CRM_PACEMAKER_DIR, // core/blackbox/scheduler/CIB files
CRM_CORE_DIR, // core files
CRM_BLACKBOX_DIR, // blackbox dumps
PE_STATE_DIR, // scheduler inputs
CRM_CONFIG_DIR, // the Cluster Information Base (CIB)
// Don't build CRM_RSCTMP_DIR, pacemaker-execd will do it
NULL
};
if (pcmk_daemon_user(&pcmk_uid, &pcmk_gid) < 0) {
crm_err("Cluster user %s does not exist, aborting Pacemaker startup",
CRM_DAEMON_USER);
crm_exit(CRM_EX_NOUSER);
}
// Used by some resource agents
if ((mkdir(CRM_STATE_DIR, 0750) < 0) && (errno != EEXIST)) {
crm_warn("Could not create directory " CRM_STATE_DIR ": %s",
pcmk_rc_str(errno));
} else {
- mcp_chown(CRM_STATE_DIR, pcmk_uid, pcmk_gid);
+ pacemakerd_chown(CRM_STATE_DIR, pcmk_uid, pcmk_gid);
}
for (int i = 0; dirs[i] != NULL; ++i) {
int rc = pcmk__build_path(dirs[i], 0750);
if (rc != pcmk_rc_ok) {
crm_warn("Could not create directory %s: %s",
dirs[i], pcmk_rc_str(rc));
} else {
- mcp_chown(dirs[i], pcmk_uid, pcmk_gid);
+ pacemakerd_chown(dirs[i], pcmk_uid, pcmk_gid);
}
}
}
static void
remove_core_file_limit(void)
{
struct rlimit cores;
// Get current limits
if (getrlimit(RLIMIT_CORE, &cores) < 0) {
crm_notice("Unable to check system core file limits "
"(consider ensuring the size is unlimited): %s",
strerror(errno));
return;
}
// Check whether core dumps are disabled
if (cores.rlim_max == 0) {
if (geteuid() != 0) { // Yes, and there's nothing we can do about it
crm_notice("Core dumps are disabled (consider enabling them)");
return;
}
cores.rlim_max = RLIM_INFINITY; // Yes, but we're root, so enable them
}
// Raise soft limit to hard limit (if not already done)
if (cores.rlim_cur != cores.rlim_max) {
cores.rlim_cur = cores.rlim_max;
if (setrlimit(RLIMIT_CORE, &cores) < 0) {
crm_notice("Unable to raise system limit on core file size "
"(consider doing so manually): %s",
strerror(errno));
return;
}
}
if (cores.rlim_cur == RLIM_INFINITY) {
crm_trace("Core file size is unlimited");
} else {
crm_trace("Core file size is limited to %llu bytes",
(unsigned long long) cores.rlim_cur);
}
}
static void
pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
enum pcmk_ipc_event event_type, crm_exit_t status,
void *event_data, void *user_data)
{
pcmk_pacemakerd_api_reply_t *reply = event_data;
switch (event_type) {
case pcmk_ipc_event_reply:
break;
default:
return;
}
if (status != CRM_EX_OK) {
out->err(out, "Bad reply from pacemakerd: %s", crm_exit_str(status));
return;
}
if (reply->reply_type != pcmk_pacemakerd_reply_shutdown) {
out->err(out, "Unknown reply type %d from pacemakerd",
reply->reply_type);
}
}
static GOptionContext *
build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
GOptionContext *context = NULL;
context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
pcmk__add_main_args(context, entries);
return context;
}
int
main(int argc, char **argv)
{
int rc = pcmk_rc_ok;
crm_exit_t exit_code = CRM_EX_OK;
GError *error = NULL;
GOptionGroup *output_group = NULL;
pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
gchar **processed_args = pcmk__cmdline_preproc(argv, "p");
GOptionContext *context = build_arg_context(args, &output_group);
bool old_instance_connected = false;
pcmk_ipc_api_t *old_instance = NULL;
qb_ipcs_service_t *ipcs = NULL;
subdaemon_check_progress = time(NULL);
setenv("LC_ALL", "C", 1); // Ensure logs are in a common language
crm_log_preinit(NULL, argc, argv);
mainloop_add_signal(SIGHUP, pcmk_ignore);
mainloop_add_signal(SIGQUIT, pcmk_sigquit);
pcmk__register_formats(output_group, formats);
if (!g_option_context_parse_strv(context, &processed_args, &error)) {
exit_code = CRM_EX_USAGE;
goto done;
}
rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
if ((rc != pcmk_rc_ok) || (out == NULL)) {
exit_code = CRM_EX_ERROR;
g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s",
args->output_ty, pcmk_rc_str(rc));
goto done;
}
pcmk__force_args(context, &error, "%s --xml-simple-list", g_get_prgname());
pcmk__register_messages(out, fmt_functions);
if (options.features) {
out->message(out, "features");
exit_code = CRM_EX_OK;
goto done;
}
if (args->version) {
out->version(out, false);
goto done;
}
- pcmk__set_env_option("mcp", "true");
+ // @COMPAT Drop at 3.0.0; likely last used in 1.1.24
+ pcmk__set_env_option(PCMK__ENV_MCP, "true", true);
if (options.shutdown) {
pcmk__cli_init_logging("pacemakerd", args->verbosity);
} else {
crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
}
crm_debug("Checking for existing Pacemaker instance");
rc = pcmk_new_ipc_api(&old_instance, pcmk_ipc_pacemakerd);
if (old_instance == NULL) {
out->err(out, "Could not check for existing pacemakerd: %s", pcmk_rc_str(rc));
exit_code = pcmk_rc2exitc(rc);
goto done;
}
pcmk_register_ipc_callback(old_instance, pacemakerd_event_cb, NULL);
rc = pcmk__connect_ipc(old_instance, pcmk_ipc_dispatch_sync, 2);
if (rc != pcmk_rc_ok) {
crm_debug("No existing %s instance found: %s",
pcmk_ipc_name(old_instance, true), pcmk_rc_str(rc));
}
old_instance_connected = pcmk_ipc_is_connected(old_instance);
if (options.shutdown) {
if (old_instance_connected) {
rc = pcmk_pacemakerd_api_shutdown(old_instance, crm_system_name);
pcmk_dispatch_ipc(old_instance);
exit_code = pcmk_rc2exitc(rc);
if (exit_code != CRM_EX_OK) {
pcmk_free_ipc_api(old_instance);
goto done;
}
/* We get the ACK immediately, and the response right after that,
* but it might take a while for pacemakerd to get around to
* shutting down. Wait for that to happen (with 30-minute timeout).
*/
for (int i = 0; i < 900; i++) {
if (!pcmk_ipc_is_connected(old_instance)) {
exit_code = CRM_EX_OK;
pcmk_free_ipc_api(old_instance);
goto done;
}
sleep(2);
}
exit_code = CRM_EX_TIMEOUT;
pcmk_free_ipc_api(old_instance);
goto done;
} else {
out->err(out, "Could not request shutdown "
"of existing Pacemaker instance: %s", pcmk_rc_str(rc));
pcmk_free_ipc_api(old_instance);
exit_code = CRM_EX_DISCONNECT;
goto done;
}
} else if (old_instance_connected) {
pcmk_free_ipc_api(old_instance);
crm_err("Aborting start-up because active Pacemaker instance found");
exit_code = CRM_EX_FATAL;
goto done;
}
pcmk_free_ipc_api(old_instance);
/* Don't allow any accidental output after this point. */
if (out != NULL) {
out->finish(out, exit_code, true, NULL);
pcmk__output_free(out);
out = NULL;
}
#ifdef SUPPORT_COROSYNC
- if (mcp_read_config() == FALSE) {
+ if (pacemakerd_read_config() == FALSE) {
crm_exit(CRM_EX_UNAVAILABLE);
}
#endif
// OCF shell functions and cluster-glue need facility under different name
{
const char *facility = pcmk__env_option(PCMK__ENV_LOGFACILITY);
if (!pcmk__str_eq(facility, PCMK__VALUE_NONE,
pcmk__str_casei|pcmk__str_null_matches)) {
- setenv("HA_LOGFACILITY", facility, 1);
+ pcmk__set_env_option("LOGFACILITY", facility, true);
}
}
crm_notice("Starting Pacemaker %s "CRM_XS" build=%s features:%s",
PACEMAKER_VERSION, BUILD_VERSION, CRM_FEATURES);
mainloop = g_main_loop_new(NULL, FALSE);
remove_core_file_limit();
create_pcmk_dirs();
- pcmk__serve_pacemakerd_ipc(&ipcs, &mcp_ipc_callbacks);
+ pcmk__serve_pacemakerd_ipc(&ipcs, &pacemakerd_ipc_callbacks);
#ifdef SUPPORT_COROSYNC
/* Allows us to block shutdown */
if (!cluster_connect_cfg()) {
exit_code = CRM_EX_PROTOCOL;
goto done;
}
#endif
if (pcmk__locate_sbd() > 0) {
- setenv("PCMK_watchdog", "true", 1);
running_with_sbd = TRUE;
- } else {
- setenv("PCMK_watchdog", "false", 1);
}
switch (find_and_track_existing_processes()) {
case pcmk_rc_ok:
break;
case pcmk_rc_ipc_unauthorized:
exit_code = CRM_EX_CANTCREAT;
goto done;
default:
exit_code = CRM_EX_FATAL;
goto done;
};
mainloop_add_signal(SIGTERM, pcmk_shutdown);
mainloop_add_signal(SIGINT, pcmk_shutdown);
if ((running_with_sbd) && pcmk__get_sbd_sync_resource_startup()) {
crm_notice("Waiting for startup-trigger from SBD.");
pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_WAITPING;
startup_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, init_children_processes, NULL);
} else {
if (running_with_sbd) {
crm_warn("Enabling SBD_SYNC_RESOURCE_STARTUP would (if supported "
"by your SBD version) improve reliability of "
"interworking between SBD & pacemaker.");
}
pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS;
init_children_processes(NULL);
}
crm_notice("Pacemaker daemon successfully started and accepting connections");
g_main_loop_run(mainloop);
if (ipcs) {
crm_trace("Closing IPC server");
mainloop_del_ipc_server(ipcs);
ipcs = NULL;
}
g_main_loop_unref(mainloop);
#ifdef SUPPORT_COROSYNC
cluster_disconnect_cfg();
#endif
done:
g_strfreev(processed_args);
pcmk__free_arg_context(context);
pcmk__output_and_clear_error(&error, out);
if (out != NULL) {
out->finish(out, exit_code, true, NULL);
pcmk__output_free(out);
}
pcmk__unregister_formats();
crm_exit(exit_code);
}
diff --git a/daemons/pacemakerd/pacemakerd.h b/daemons/pacemakerd/pacemakerd.h
index b2a6864f4e..ee6facfb36 100644
--- a/daemons/pacemakerd/pacemakerd.h
+++ b/daemons/pacemakerd/pacemakerd.h
@@ -1,35 +1,35 @@
/*
- * Copyright 2010-2022 the Pacemaker project contributors
+ * Copyright 2010-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <stdint.h>
#define MAX_RESPAWN 100
extern GMainLoop *mainloop;
-extern struct qb_ipcs_service_handlers mcp_ipc_callbacks;
+extern struct qb_ipcs_service_handlers pacemakerd_ipc_callbacks;
extern const char *pacemakerd_state;
extern gboolean running_with_sbd;
extern unsigned int shutdown_complete_state_reported_to;
extern gboolean shutdown_complete_state_reported_client_closed;
extern crm_trigger_t *shutdown_trigger;
extern crm_trigger_t *startup_trigger;
extern time_t subdaemon_check_progress;
-gboolean mcp_read_config(void);
+gboolean pacemakerd_read_config(void);
gboolean cluster_connect_cfg(void);
void cluster_disconnect_cfg(void);
int find_and_track_existing_processes(void);
gboolean init_children_processes(void *user_data);
void restart_cluster_subdaemons(void);
void pcmk_shutdown(int nsig);
void pcmkd_shutdown_corosync(void);
bool pcmkd_corosync_connected(void);
diff --git a/daemons/pacemakerd/pcmkd_corosync.c b/daemons/pacemakerd/pcmkd_corosync.c
index 2648756451..8a1a8671eb 100644
--- a/daemons/pacemakerd/pcmkd_corosync.c
+++ b/daemons/pacemakerd/pcmkd_corosync.c
@@ -1,371 +1,373 @@
/*
- * Copyright 2010-2022 the Pacemaker project contributors
+ * Copyright 2010-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include "pacemakerd.h"
#include <sys/utsname.h>
#include <sys/stat.h> /* for calls to stat() */
#include <libgen.h> /* For basename() and dirname() */
#include <sys/types.h>
#include <pwd.h> /* For getpwname() */
#include <corosync/hdb.h>
#include <corosync/cfg.h>
#include <corosync/cpg.h>
#include <corosync/cmap.h>
#include <crm/cluster/internal.h>
#include <crm/common/ipc.h> /* for crm_ipc_is_authentic_process */
#include <crm/common/mainloop.h>
#include <crm/common/ipc_internal.h> /* PCMK__SPECIAL_PID* */
static corosync_cfg_handle_t cfg_handle = 0;
static mainloop_timer_t *reconnect_timer = NULL;
/* =::=::=::= CFG - Shutdown stuff =::=::=::= */
static void
cfg_shutdown_callback(corosync_cfg_handle_t h, corosync_cfg_shutdown_flags_t flags)
{
crm_info("Corosync wants to shut down: %s",
(flags == COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE) ? "immediate" :
(flags == COROSYNC_CFG_SHUTDOWN_FLAG_REGARDLESS) ? "forced" : "optional");
/* Never allow corosync to shut down while we're running */
corosync_cfg_replyto_shutdown(h, COROSYNC_CFG_SHUTDOWN_FLAG_NO);
}
static corosync_cfg_callbacks_t cfg_callbacks = {
.corosync_cfg_shutdown_callback = cfg_shutdown_callback,
};
static int
pcmk_cfg_dispatch(gpointer user_data)
{
corosync_cfg_handle_t *handle = (corosync_cfg_handle_t *) user_data;
cs_error_t rc = corosync_cfg_dispatch(*handle, CS_DISPATCH_ALL);
if (rc != CS_OK) {
return -1;
}
return 0;
}
static void
close_cfg(void)
{
if (cfg_handle != 0) {
#ifdef HAVE_COROSYNC_CFG_TRACKSTART
/* Ideally, we would call corosync_cfg_trackstop(cfg_handle) here, but a
* bug in corosync 3.1.1 and 3.1.2 makes it hang forever. Thankfully,
* it's not necessary since we exit immediately after this.
*/
#endif
corosync_cfg_finalize(cfg_handle);
cfg_handle = 0;
}
}
static gboolean
cluster_reconnect_cb(gpointer data)
{
if (cluster_connect_cfg()) {
mainloop_timer_del(reconnect_timer);
reconnect_timer = NULL;
crm_notice("Cluster reconnect succeeded");
- mcp_read_config();
+ pacemakerd_read_config();
restart_cluster_subdaemons();
return G_SOURCE_REMOVE;
} else {
crm_info("Cluster reconnect failed "
"(connection will be reattempted once per second)");
}
/*
* In theory this will continue forever. In practice the CIB connection from
* attrd will timeout and shut down Pacemaker when it gets bored.
*/
return G_SOURCE_CONTINUE;
}
static void
cfg_connection_destroy(gpointer user_data)
{
crm_warn("Lost connection to cluster layer "
"(connection will be reattempted once per second)");
corosync_cfg_finalize(cfg_handle);
cfg_handle = 0;
reconnect_timer = mainloop_timer_add("corosync reconnect", 1000, TRUE, cluster_reconnect_cb, NULL);
mainloop_timer_start(reconnect_timer);
}
void
cluster_disconnect_cfg(void)
{
close_cfg();
if (reconnect_timer != NULL) {
/* The mainloop should be gone by this point, so this isn't necessary,
* but cleaning up memory should make valgrind happier.
*/
mainloop_timer_del(reconnect_timer);
reconnect_timer = NULL;
}
}
#define cs_repeat(counter, max, code) do { \
code; \
if(rc == CS_ERR_TRY_AGAIN || rc == CS_ERR_QUEUE_FULL) { \
counter++; \
crm_debug("Retrying Corosync operation after %ds", counter); \
sleep(counter); \
} else { \
break; \
} \
} while(counter < max)
gboolean
cluster_connect_cfg(void)
{
cs_error_t rc;
int fd = -1, retries = 0, rv;
uid_t found_uid = 0;
gid_t found_gid = 0;
pid_t found_pid = 0;
uint32_t nodeid;
static struct mainloop_fd_callbacks cfg_fd_callbacks = {
.dispatch = pcmk_cfg_dispatch,
.destroy = cfg_connection_destroy,
};
cs_repeat(retries, 30, rc = corosync_cfg_initialize(&cfg_handle, &cfg_callbacks));
if (rc != CS_OK) {
crm_crit("Could not connect to Corosync CFG: %s " CRM_XS " rc=%d",
cs_strerror(rc), rc);
return FALSE;
}
rc = corosync_cfg_fd_get(cfg_handle, &fd);
if (rc != CS_OK) {
crm_crit("Could not get Corosync CFG descriptor: %s " CRM_XS " rc=%d",
cs_strerror(rc), rc);
goto bail;
}
/* CFG provider run as root (in given user namespace, anyway)? */
if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
&found_uid, &found_gid))) {
crm_crit("Rejecting Corosync CFG provider because process %lld "
"is running as uid %lld gid %lld, not root",
(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
(long long) found_uid, (long long) found_gid);
goto bail;
} else if (rv < 0) {
crm_crit("Could not authenticate Corosync CFG provider: %s "
CRM_XS " rc=%d", strerror(-rv), -rv);
goto bail;
}
retries = 0;
cs_repeat(retries, 30, rc = corosync_cfg_local_get(cfg_handle, &nodeid));
if (rc != CS_OK) {
crm_crit("Could not get local node ID from Corosync: %s "
CRM_XS " rc=%d", cs_strerror(rc), rc);
goto bail;
}
crm_debug("Corosync reports local node ID is %lu", (unsigned long) nodeid);
#ifdef HAVE_COROSYNC_CFG_TRACKSTART
retries = 0;
cs_repeat(retries, 30, rc = corosync_cfg_trackstart(cfg_handle, 0));
if (rc != CS_OK) {
crm_crit("Could not enable Corosync CFG shutdown tracker: %s " CRM_XS " rc=%d",
cs_strerror(rc), rc);
goto bail;
}
#endif
mainloop_add_fd("corosync-cfg", G_PRIORITY_DEFAULT, fd, &cfg_handle, &cfg_fd_callbacks);
return TRUE;
bail:
corosync_cfg_finalize(cfg_handle);
return FALSE;
}
void
pcmkd_shutdown_corosync(void)
{
cs_error_t rc;
if (cfg_handle == 0) {
crm_warn("Unable to shut down Corosync: No connection");
return;
}
crm_info("Asking Corosync to shut down");
rc = corosync_cfg_try_shutdown(cfg_handle,
COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE);
if (rc == CS_OK) {
close_cfg();
} else {
crm_warn("Corosync shutdown failed: %s " CRM_XS " rc=%d",
cs_strerror(rc), rc);
}
}
bool
pcmkd_corosync_connected(void)
{
cpg_handle_t local_handle = 0;
cpg_model_v1_data_t cpg_model_info = {CPG_MODEL_V1, NULL, NULL, NULL, 0};
int fd = -1;
if (cpg_model_initialize(&local_handle, CPG_MODEL_V1, (cpg_model_data_t *) &cpg_model_info, NULL) != CS_OK) {
return false;
}
if (cpg_fd_get(local_handle, &fd) != CS_OK) {
return false;
}
cpg_finalize(local_handle);
return true;
}
/* =::=::=::= Configuration =::=::=::= */
static int
get_config_opt(uint64_t unused, cmap_handle_t object_handle, const char *key, char **value,
const char *fallback)
{
int rc = 0, retries = 0;
cs_repeat(retries, 5, rc = cmap_get_string(object_handle, key, value));
if (rc != CS_OK) {
crm_trace("Search for %s failed %d, defaulting to %s", key, rc, fallback);
pcmk__str_update(value, fallback);
}
crm_trace("%s: %s", key, *value);
return rc;
}
gboolean
-mcp_read_config(void)
+pacemakerd_read_config(void)
{
cs_error_t rc = CS_OK;
int retries = 0;
cmap_handle_t local_handle;
uint64_t config = 0;
int fd = -1;
uid_t found_uid = 0;
gid_t found_gid = 0;
pid_t found_pid = 0;
int rv;
enum cluster_type_e stack;
// There can be only one possibility
do {
rc = pcmk__init_cmap(&local_handle);
if (rc != CS_OK) {
retries++;
crm_info("Could not connect to Corosync CMAP: %s (retrying in %ds) "
CRM_XS " rc=%d", cs_strerror(rc), retries, rc);
sleep(retries);
} else {
break;
}
} while (retries < 5);
if (rc != CS_OK) {
crm_crit("Could not connect to Corosync CMAP: %s "
CRM_XS " rc=%d", cs_strerror(rc), rc);
return FALSE;
}
rc = cmap_fd_get(local_handle, &fd);
if (rc != CS_OK) {
crm_crit("Could not get Corosync CMAP descriptor: %s " CRM_XS " rc=%d",
cs_strerror(rc), rc);
cmap_finalize(local_handle);
return FALSE;
}
/* CMAP provider run as root (in given user namespace, anyway)? */
if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
&found_uid, &found_gid))) {
crm_crit("Rejecting Corosync CMAP provider because process %lld "
"is running as uid %lld gid %lld, not root",
(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
(long long) found_uid, (long long) found_gid);
cmap_finalize(local_handle);
return FALSE;
} else if (rv < 0) {
crm_crit("Could not authenticate Corosync CMAP provider: %s "
CRM_XS " rc=%d", strerror(-rv), -rv);
cmap_finalize(local_handle);
return FALSE;
}
stack = get_cluster_type();
if (stack != pcmk_cluster_corosync) {
crm_crit("Expected Corosync cluster layer but detected %s "
CRM_XS " stack=%d", name_for_cluster_type(stack), stack);
return FALSE;
}
crm_info("Reading configuration for %s stack",
name_for_cluster_type(stack));
- pcmk__set_env_option(PCMK__ENV_CLUSTER_TYPE, "corosync");
- pcmk__set_env_option(PCMK__ENV_QUORUM_TYPE, "corosync");
+ pcmk__set_env_option(PCMK__ENV_CLUSTER_TYPE, "corosync", true);
+
+ // @COMPAT Drop at 3.0.0; added unused in 1.1.9
+ pcmk__set_env_option(PCMK__ENV_QUORUM_TYPE, "corosync", true);
// If debug logging is not configured, check whether corosync has it
if (pcmk__env_option(PCMK__ENV_DEBUG) == NULL) {
char *debug_enabled = NULL;
get_config_opt(config, local_handle, "logging.debug", &debug_enabled, "off");
if (crm_is_true(debug_enabled)) {
- pcmk__set_env_option(PCMK__ENV_DEBUG, "1");
+ pcmk__set_env_option(PCMK__ENV_DEBUG, "1", true);
if (get_crm_log_level() < LOG_DEBUG) {
set_crm_log_level(LOG_DEBUG);
}
} else {
- pcmk__set_env_option(PCMK__ENV_DEBUG, "0");
+ pcmk__set_env_option(PCMK__ENV_DEBUG, "0", true);
}
free(debug_enabled);
}
if(local_handle){
gid_t gid = 0;
if (pcmk_daemon_user(NULL, &gid) < 0) {
crm_warn("Could not authorize group with Corosync " CRM_XS
" No group found for user %s", CRM_DAEMON_USER);
} else {
char key[PATH_MAX];
snprintf(key, PATH_MAX, "uidgid.gid.%u", gid);
rc = cmap_set_uint8(local_handle, key, 1);
if (rc != CS_OK) {
crm_warn("Could not authorize group with Corosync: %s " CRM_XS
" group=%u rc=%d", pcmk__cs_err_str(rc), gid, rc);
}
}
}
cmap_finalize(local_handle);
return TRUE;
}
diff --git a/daemons/pacemakerd/pcmkd_messages.c b/daemons/pacemakerd/pcmkd_messages.c
index 7ed9899d67..4e6f8229ea 100644
--- a/daemons/pacemakerd/pcmkd_messages.c
+++ b/daemons/pacemakerd/pcmkd_messages.c
@@ -1,278 +1,278 @@
/*
* Copyright 2010-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include "pacemakerd.h"
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <errno.h>
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <time.h>
#include <sys/types.h>
static GHashTable *pcmkd_handlers = NULL;
static xmlNode *
handle_node_cache_request(pcmk__request_t *request)
{
crm_trace("Ignoring request from client %s to purge node "
"because peer cache is not used",
pcmk__client_name(request->ipc_client));
pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
"ack", NULL, CRM_EX_OK);
return NULL;
}
static xmlNode *
handle_ping_request(pcmk__request_t *request)
{
xmlNode *msg = request->xml;
const char *value = NULL;
xmlNode *ping = NULL;
xmlNode *reply = NULL;
const char *from = crm_element_value(msg, F_CRM_SYS_FROM);
/* Pinged for status */
crm_trace("Pinged from " F_CRM_SYS_FROM "='%s' " F_CRM_ORIGIN "='%s'",
pcmk__s(from, ""),
pcmk__s(crm_element_value(msg, F_CRM_ORIGIN), ""));
pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
"ack", NULL, CRM_EX_INDETERMINATE);
ping = create_xml_node(NULL, XML_CRM_TAG_PING);
value = crm_element_value(msg, F_CRM_SYS_TO);
crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value);
crm_xml_add(ping, XML_PING_ATTR_PACEMAKERDSTATE, pacemakerd_state);
crm_xml_add_ll(ping, XML_ATTR_TSTAMP,
(long long) subdaemon_check_progress);
crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok");
reply = create_reply(msg, ping);
free_xml(ping);
if (reply == NULL) {
pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
"Failed building ping reply for client %s",
pcmk__client_name(request->ipc_client));
} else {
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
}
/* just proceed state on sbd pinging us */
if (from && strstr(from, "sbd")) {
if (pcmk__str_eq(pacemakerd_state, XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE, pcmk__str_none)) {
if (pcmk__get_sbd_sync_resource_startup()) {
crm_notice("Shutdown-complete-state passed to SBD.");
}
shutdown_complete_state_reported_to = request->ipc_client->pid;
} else if (pcmk__str_eq(pacemakerd_state, XML_PING_ATTR_PACEMAKERDSTATE_WAITPING, pcmk__str_none)) {
crm_notice("Received startup-trigger from SBD.");
pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS;
mainloop_set_trigger(startup_trigger);
}
}
return reply;
}
static xmlNode *
handle_shutdown_request(pcmk__request_t *request)
{
xmlNode *msg = request->xml;
xmlNode *shutdown = NULL;
xmlNode *reply = NULL;
/* Only allow privileged users (i.e. root or hacluster) to shut down
* Pacemaker from the command line (or direct IPC), so that other users
* are forced to go through the CIB and have ACLs applied.
*/
bool allowed = pcmk_is_set(request->ipc_client->flags, pcmk__client_privileged);
pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
"ack", NULL, CRM_EX_INDETERMINATE);
shutdown = create_xml_node(NULL, XML_CIB_ATTR_SHUTDOWN);
if (allowed) {
crm_notice("Shutting down in response to IPC request %s from %s",
crm_element_value(msg, F_CRM_REFERENCE),
crm_element_value(msg, F_CRM_ORIGIN));
crm_xml_add_int(shutdown, XML_LRM_ATTR_OPSTATUS, CRM_EX_OK);
} else {
crm_warn("Ignoring shutdown request from unprivileged client %s",
pcmk__client_name(request->ipc_client));
crm_xml_add_int(shutdown, XML_LRM_ATTR_OPSTATUS, CRM_EX_INSUFFICIENT_PRIV);
}
reply = create_reply(msg, shutdown);
free_xml(shutdown);
if (reply == NULL) {
pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
"Failed building shutdown reply for client %s",
pcmk__client_name(request->ipc_client));
} else {
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
}
if (allowed) {
pcmk_shutdown(15);
}
return reply;
}
static xmlNode *
handle_unknown_request(pcmk__request_t *request)
{
pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
"ack", NULL, CRM_EX_INVALID_PARAM);
pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
"Unknown IPC request type '%s' (bug?)",
pcmk__client_name(request->ipc_client));
return NULL;
}
static void
pcmkd_register_handlers(void)
{
pcmk__server_command_t handlers[] = {
{ CRM_OP_RM_NODE_CACHE, handle_node_cache_request },
{ CRM_OP_PING, handle_ping_request },
{ CRM_OP_QUIT, handle_shutdown_request },
{ NULL, handle_unknown_request },
};
pcmkd_handlers = pcmk__register_handlers(handlers);
}
static int32_t
pcmk_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
{
crm_trace("Connection %p", c);
if (pcmk__new_client(c, uid, gid) == NULL) {
return -EIO;
}
return 0;
}
/* Error code means? */
static int32_t
pcmk_ipc_closed(qb_ipcs_connection_t * c)
{
pcmk__client_t *client = pcmk__find_client(c);
if (client == NULL) {
return 0;
}
crm_trace("Connection %p", c);
if (shutdown_complete_state_reported_to == client->pid) {
shutdown_complete_state_reported_client_closed = TRUE;
if (shutdown_trigger) {
mainloop_set_trigger(shutdown_trigger);
}
}
pcmk__free_client(client);
return 0;
}
static void
pcmk_ipc_destroy(qb_ipcs_connection_t * c)
{
crm_trace("Connection %p", c);
pcmk_ipc_closed(c);
}
/* Exit code means? */
static int32_t
pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
{
uint32_t id = 0;
uint32_t flags = 0;
xmlNode *msg = NULL;
pcmk__client_t *c = pcmk__find_client(qbc);
CRM_CHECK(c != NULL, return 0);
if (pcmkd_handlers == NULL) {
pcmkd_register_handlers();
}
msg = pcmk__client_data2xml(c, data, &id, &flags);
if (msg == NULL) {
pcmk__ipc_send_ack(c, id, flags, "ack", NULL, CRM_EX_PROTOCOL);
return 0;
} else {
char *log_msg = NULL;
const char *reason = NULL;
xmlNode *reply = NULL;
pcmk__request_t request = {
.ipc_client = c,
.ipc_id = id,
.ipc_flags = flags,
.peer = NULL,
.xml = msg,
.call_options = 0,
.result = PCMK__UNKNOWN_RESULT,
};
request.op = crm_element_value_copy(request.xml, F_CRM_TASK);
CRM_CHECK(request.op != NULL, return 0);
reply = pcmk__process_request(&request, pcmkd_handlers);
if (reply != NULL) {
pcmk__ipc_send_xml(c, id, reply, crm_ipc_server_event);
free_xml(reply);
}
reason = request.result.exit_reason;
log_msg = crm_strdup_printf("Processed %s request from %s %s: %s%s%s%s",
request.op, pcmk__request_origin_type(&request),
pcmk__request_origin(&request),
pcmk_exec_status_str(request.result.execution_status),
(reason == NULL)? "" : " (",
(reason == NULL)? "" : reason,
(reason == NULL)? "" : ")");
if (!pcmk__result_ok(&request.result)) {
crm_warn("%s", log_msg);
} else {
crm_debug("%s", log_msg);
}
free(log_msg);
pcmk__reset_request(&request);
}
free_xml(msg);
return 0;
}
-struct qb_ipcs_service_handlers mcp_ipc_callbacks = {
+struct qb_ipcs_service_handlers pacemakerd_ipc_callbacks = {
.connection_accept = pcmk_ipc_accept,
.connection_created = NULL,
.msg_process = pcmk_ipc_dispatch,
.connection_closed = pcmk_ipc_closed,
.connection_destroyed = pcmk_ipc_destroy
};
diff --git a/daemons/pacemakerd/pcmkd_subdaemons.c b/daemons/pacemakerd/pcmkd_subdaemons.c
index 3b08eccb37..e06b8eb9c8 100644
--- a/daemons/pacemakerd/pcmkd_subdaemons.c
+++ b/daemons/pacemakerd/pcmkd_subdaemons.c
@@ -1,888 +1,890 @@
/*
- * Copyright 2010-2022 the Pacemaker project contributors
+ * Copyright 2010-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include "pacemakerd.h"
#include <errno.h>
#include <grp.h>
#include <signal.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h>
#include <crm/cluster.h>
#include <crm/msg_xml.h>
typedef struct pcmk_child_s {
pid_t pid;
int respawn_count;
bool respawn;
const char *name;
const char *uid;
const char *command;
const char *endpoint; /* IPC server name */
bool needs_cluster;
int check_count;
/* Anything below here will be dynamically initialized */
bool needs_retry;
bool active_before_startup;
} pcmk_child_t;
#define PCMK_PROCESS_CHECK_INTERVAL 1
#define PCMK_PROCESS_CHECK_RETRIES 5
#define SHUTDOWN_ESCALATION_PERIOD 180000 /* 3m */
/* Index into the array below */
#define PCMK_CHILD_CONTROLD 5
static pcmk_child_t pcmk_children[] = {
{
0, 0, true, "pacemaker-based", CRM_DAEMON_USER,
CRM_DAEMON_DIR "/pacemaker-based", PCMK__SERVER_BASED_RO,
true
},
{
0, 0, true, "pacemaker-fenced", NULL,
CRM_DAEMON_DIR "/pacemaker-fenced", "stonith-ng",
true
},
{
0, 0, true, "pacemaker-execd", NULL,
CRM_DAEMON_DIR "/pacemaker-execd", CRM_SYSTEM_LRMD,
false
},
{
0, 0, true, "pacemaker-attrd", CRM_DAEMON_USER,
CRM_DAEMON_DIR "/pacemaker-attrd", T_ATTRD,
true
},
{
0, 0, true, "pacemaker-schedulerd", CRM_DAEMON_USER,
CRM_DAEMON_DIR "/pacemaker-schedulerd", CRM_SYSTEM_PENGINE,
false
},
{
0, 0, true, "pacemaker-controld", CRM_DAEMON_USER,
CRM_DAEMON_DIR "/pacemaker-controld", CRM_SYSTEM_CRMD,
true
},
};
static char *opts_default[] = { NULL, NULL };
static char *opts_vgrind[] = { NULL, NULL, NULL, NULL, NULL };
crm_trigger_t *shutdown_trigger = NULL;
crm_trigger_t *startup_trigger = NULL;
time_t subdaemon_check_progress = 0;
// Whether we need root group access to talk to cluster layer
static bool need_root_group = true;
/* When contacted via pacemakerd-api by a client having sbd in
* the name we assume it is sbd-daemon which wants to know
* if pacemakerd shutdown gracefully.
* Thus when everything is shutdown properly pacemakerd
* waits till it has reported the graceful completion of
* shutdown to sbd and just when sbd-client closes the
* connection we can assume that the report has arrived
* properly so that pacemakerd can finally exit.
* Following two variables are used to track that handshake.
*/
unsigned int shutdown_complete_state_reported_to = 0;
gboolean shutdown_complete_state_reported_client_closed = FALSE;
/* state we report when asked via pacemakerd-api status-ping */
const char *pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_INIT;
gboolean running_with_sbd = FALSE; /* local copy */
GMainLoop *mainloop = NULL;
static gboolean fatal_error = FALSE;
static int child_liveness(pcmk_child_t *child);
static gboolean escalate_shutdown(gpointer data);
static int start_child(pcmk_child_t * child);
static void pcmk_child_exit(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode);
static void pcmk_process_exit(pcmk_child_t * child);
static gboolean pcmk_shutdown_worker(gpointer user_data);
static gboolean stop_child(pcmk_child_t * child, int signal);
static bool
pcmkd_cluster_connected(void)
{
#if SUPPORT_COROSYNC
return pcmkd_corosync_connected();
#else
return true;
#endif
}
static gboolean
check_next_subdaemon(gpointer user_data)
{
static int next_child = 0;
int rc = child_liveness(&pcmk_children[next_child]);
crm_trace("Checked %s[%lld]: %s (%d)",
pcmk_children[next_child].name,
(long long) PCMK__SPECIAL_PID_AS_0(pcmk_children[next_child].pid),
pcmk_rc_str(rc), rc);
switch (rc) {
case pcmk_rc_ok:
pcmk_children[next_child].check_count = 0;
subdaemon_check_progress = time(NULL);
break;
case pcmk_rc_ipc_pid_only: // This case: it was previously OK
pcmk_children[next_child].check_count++;
if (pcmk_children[next_child].check_count >= PCMK_PROCESS_CHECK_RETRIES) {
crm_err("%s[%lld] is unresponsive to ipc after %d tries but "
"we found the pid so have it killed that we can restart",
pcmk_children[next_child].name,
(long long) PCMK__SPECIAL_PID_AS_0(
pcmk_children[next_child].pid),
pcmk_children[next_child].check_count);
stop_child(&pcmk_children[next_child], SIGKILL);
if (pcmk_children[next_child].respawn) {
/* as long as the respawn-limit isn't reached
give it another round of check retries
*/
pcmk_children[next_child].check_count = 0;
}
} else {
crm_notice("%s[%lld] is unresponsive to ipc after %d tries",
pcmk_children[next_child].name,
(long long) PCMK__SPECIAL_PID_AS_0(
pcmk_children[next_child].pid),
pcmk_children[next_child].check_count);
if (pcmk_children[next_child].respawn) {
/* as long as the respawn-limit isn't reached
and we haven't run out of connect retries
we account this as progress we are willing
to tell to sbd
*/
subdaemon_check_progress = time(NULL);
}
}
/* go to the next child and see if
we can make progress there
*/
break;
case pcmk_rc_ipc_unresponsive:
if (!pcmk_children[next_child].respawn) {
/* if a subdaemon is down and we don't want it
to be restarted this is a success during
shutdown. if it isn't restarted anymore
due to MAX_RESPAWN it is
rather no success.
*/
if (pcmk_children[next_child].respawn_count <= MAX_RESPAWN) {
subdaemon_check_progress = time(NULL);
}
}
if (!pcmk_children[next_child].active_before_startup) {
crm_trace("found %s[%lld] missing - signal-handler "
"will take care of it",
pcmk_children[next_child].name,
(long long) PCMK__SPECIAL_PID_AS_0(
pcmk_children[next_child].pid));
break;
}
if (pcmk_children[next_child].respawn) {
crm_err("%s[%lld] terminated",
pcmk_children[next_child].name,
(long long) PCMK__SPECIAL_PID_AS_0(
pcmk_children[next_child].pid));
} else {
/* orderly shutdown */
crm_notice("%s[%lld] terminated",
pcmk_children[next_child].name,
(long long) PCMK__SPECIAL_PID_AS_0(
pcmk_children[next_child].pid));
}
pcmk_process_exit(&(pcmk_children[next_child]));
break;
default:
crm_exit(CRM_EX_FATAL);
break; /* static analysis/noreturn */
}
next_child++;
if (next_child >= PCMK__NELEM(pcmk_children)) {
next_child = 0;
}
return G_SOURCE_CONTINUE;
}
static gboolean
escalate_shutdown(gpointer data)
{
pcmk_child_t *child = data;
if (child->pid == PCMK__SPECIAL_PID) {
pcmk_process_exit(child);
} else if (child->pid != 0) {
/* Use SIGSEGV instead of SIGKILL to create a core so we can see what it was up to */
crm_err("Child %s not terminating in a timely manner, forcing", child->name);
stop_child(child, SIGSEGV);
}
return FALSE;
}
static void
pcmk_child_exit(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode)
{
pcmk_child_t *child = mainloop_child_userdata(p);
const char *name = mainloop_child_name(p);
if (signo) {
do_crm_log(((signo == SIGKILL)? LOG_WARNING : LOG_ERR),
"%s[%d] terminated with signal %d (%s)%s",
name, pid, signo, strsignal(signo),
(core? " and dumped core" : ""));
} else {
switch(exitcode) {
case CRM_EX_OK:
crm_info("%s[%d] exited with status %d (%s)",
name, pid, exitcode, crm_exit_str(exitcode));
break;
case CRM_EX_FATAL:
crm_warn("Shutting cluster down because %s[%d] had fatal failure",
name, pid);
child->respawn = false;
fatal_error = TRUE;
pcmk_shutdown(SIGTERM);
break;
case CRM_EX_PANIC:
crm_emerg("%s[%d] instructed the machine to reset", name, pid);
child->respawn = false;
fatal_error = TRUE;
pcmk__panic(__func__);
pcmk_shutdown(SIGTERM);
break;
default:
crm_err("%s[%d] exited with status %d (%s)",
name, pid, exitcode, crm_exit_str(exitcode));
break;
}
}
pcmk_process_exit(child);
}
static void
pcmk_process_exit(pcmk_child_t * child)
{
child->pid = 0;
child->active_before_startup = false;
child->check_count = 0;
child->respawn_count += 1;
if (child->respawn_count > MAX_RESPAWN) {
crm_err("Child respawn count exceeded by %s", child->name);
child->respawn = false;
}
if (shutdown_trigger) {
/* resume step-wise shutdown (returned TRUE yields no parallelizing) */
mainloop_set_trigger(shutdown_trigger);
} else if (!child->respawn) {
/* nothing to do */
- } else if (crm_is_true(getenv("PCMK_fail_fast"))) {
+ } else if (crm_is_true(pcmk__env_option(PCMK__ENV_FAIL_FAST))) {
crm_err("Rebooting system because of %s", child->name);
pcmk__panic(__func__);
} else if (child_liveness(child) == pcmk_rc_ok) {
crm_warn("One-off suppressing strict respawning of a child process %s,"
" appears alright per %s IPC end-point",
child->name, child->endpoint);
} else if (child->needs_cluster && !pcmkd_cluster_connected()) {
crm_notice("Not respawning %s subdaemon until cluster returns",
child->name);
child->needs_retry = true;
} else {
crm_notice("Respawning %s subdaemon after unexpected exit",
child->name);
start_child(child);
}
}
static gboolean
pcmk_shutdown_worker(gpointer user_data)
{
static int phase = PCMK__NELEM(pcmk_children) - 1;
static time_t next_log = 0;
if (phase == PCMK__NELEM(pcmk_children) - 1) {
crm_notice("Shutting down Pacemaker");
pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN;
}
for (; phase >= 0; phase--) {
pcmk_child_t *child = &(pcmk_children[phase]);
if (child->pid != 0) {
time_t now = time(NULL);
if (child->respawn) {
if (child->pid == PCMK__SPECIAL_PID) {
crm_warn("The process behind %s IPC cannot be"
" terminated, so either wait the graceful"
" period of %ld s for its native termination"
" if it vitally depends on some other daemons"
" going down in a controlled way already,"
" or locate and kill the correct %s process"
- " on your own; set PCMK_fail_fast=1 to avoid"
- " this altogether next time around",
+ " on your own; set PCMK_" PCMK__ENV_FAIL_FAST "=1"
+ " to avoid this altogether next time around",
child->name, (long) SHUTDOWN_ESCALATION_PERIOD,
child->command);
}
next_log = now + 30;
child->respawn = false;
stop_child(child, SIGTERM);
if (phase < PCMK_CHILD_CONTROLD) {
g_timeout_add(SHUTDOWN_ESCALATION_PERIOD,
escalate_shutdown, child);
}
} else if (now >= next_log) {
next_log = now + 30;
crm_notice("Still waiting for %s to terminate "
CRM_XS " pid=%lld",
child->name, (long long) child->pid);
}
return TRUE;
}
/* cleanup */
crm_debug("%s confirmed stopped", child->name);
child->pid = 0;
}
crm_notice("Shutdown complete");
pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE;
if (!fatal_error && running_with_sbd &&
pcmk__get_sbd_sync_resource_startup() &&
!shutdown_complete_state_reported_client_closed) {
crm_notice("Waiting for SBD to pick up shutdown-complete-state.");
return TRUE;
}
{
const char *delay = pcmk__env_option(PCMK__ENV_SHUTDOWN_DELAY);
if(delay) {
sync();
pcmk__sleep_ms(crm_get_msec(delay));
}
}
g_main_loop_quit(mainloop);
if (fatal_error) {
crm_notice("Shutting down and staying down after fatal error");
#ifdef SUPPORT_COROSYNC
pcmkd_shutdown_corosync();
#endif
crm_exit(CRM_EX_FATAL);
}
return TRUE;
}
/* TODO once libqb is taught to juggle with IPC end-points carried over as
bare file descriptor (https://github.com/ClusterLabs/libqb/issues/325)
it shall hand over these descriptors here if/once they are successfully
pre-opened in (presumably) child_liveness(), to avoid any remaining
room for races */
// \return Standard Pacemaker return code
static int
start_child(pcmk_child_t * child)
{
uid_t uid = 0;
gid_t gid = 0;
gboolean use_valgrind = FALSE;
gboolean use_callgrind = FALSE;
- const char *env_valgrind = getenv("PCMK_valgrind_enabled");
- const char *env_callgrind = getenv("PCMK_callgrind_enabled");
+ const char *env_valgrind = pcmk__env_option(PCMK__ENV_VALGRIND_ENABLED);
+ const char *env_callgrind = pcmk__env_option(PCMK__ENV_CALLGRIND_ENABLED);
child->active_before_startup = false;
child->check_count = 0;
if (child->command == NULL) {
crm_info("Nothing to do for child \"%s\"", child->name);
return pcmk_rc_ok;
}
if (env_callgrind != NULL && crm_is_true(env_callgrind)) {
use_callgrind = TRUE;
use_valgrind = TRUE;
} else if (env_callgrind != NULL && strstr(env_callgrind, child->name)) {
use_callgrind = TRUE;
use_valgrind = TRUE;
} else if (env_valgrind != NULL && crm_is_true(env_valgrind)) {
use_valgrind = TRUE;
} else if (env_valgrind != NULL && strstr(env_valgrind, child->name)) {
use_valgrind = TRUE;
}
if (use_valgrind && strlen(VALGRIND_BIN) == 0) {
crm_warn("Cannot enable valgrind for %s:"
" The location of the valgrind binary is unknown", child->name);
use_valgrind = FALSE;
}
if (child->uid) {
if (crm_user_lookup(child->uid, &uid, &gid) < 0) {
crm_err("Invalid user (%s) for %s: not found", child->uid, child->name);
return EACCES;
}
crm_info("Using uid=%u and group=%u for process %s", uid, gid, child->name);
}
child->pid = fork();
CRM_ASSERT(child->pid != -1);
if (child->pid > 0) {
/* parent */
mainloop_child_add(child->pid, 0, child->name, child, pcmk_child_exit);
crm_info("Forked child %lld for process %s%s",
(long long) child->pid, child->name,
use_valgrind ? " (valgrind enabled: " VALGRIND_BIN ")" : "");
return pcmk_rc_ok;
} else {
/* Start a new session */
(void)setsid();
/* Setup the two alternate arg arrays */
opts_vgrind[0] = strdup(VALGRIND_BIN);
if (use_callgrind) {
opts_vgrind[1] = strdup("--tool=callgrind");
opts_vgrind[2] = strdup("--callgrind-out-file=" CRM_STATE_DIR "/callgrind.out.%p");
opts_vgrind[3] = strdup(child->command);
opts_vgrind[4] = NULL;
} else {
opts_vgrind[1] = strdup(child->command);
opts_vgrind[2] = NULL;
opts_vgrind[3] = NULL;
opts_vgrind[4] = NULL;
}
opts_default[0] = strdup(child->command);
if(gid) {
// Drop root group access if not needed
if (!need_root_group && (setgid(gid) < 0)) {
crm_warn("Could not set group to %d: %s", gid, strerror(errno));
}
/* Initialize supplementary groups to only those always granted to
* the user, plus haclient (so we can access IPC).
*/
if (initgroups(child->uid, gid) < 0) {
crm_err("Cannot initialize groups for %s: %s (%d)",
child->uid, pcmk_rc_str(errno), errno);
}
}
if (uid && setuid(uid) < 0) {
crm_warn("Could not set user to %s (id %d): %s",
child->uid, uid, strerror(errno));
}
pcmk__close_fds_in_child(true);
pcmk__open_devnull(O_RDONLY); // stdin (fd 0)
pcmk__open_devnull(O_WRONLY); // stdout (fd 1)
pcmk__open_devnull(O_WRONLY); // stderr (fd 2)
if (use_valgrind) {
(void)execvp(VALGRIND_BIN, opts_vgrind);
} else {
(void)execvp(child->command, opts_default);
}
crm_crit("Could not execute %s: %s", child->command, strerror(errno));
crm_exit(CRM_EX_FATAL);
}
return pcmk_rc_ok; /* never reached */
}
/*!
* \internal
* \brief Check the liveness of the child based on IPC name and PID if tracked
*
* \param[in,out] child Child tracked data
*
* \return Standard Pacemaker return code
*
* \note Return codes of particular interest include pcmk_rc_ipc_unresponsive
* indicating that no trace of IPC liveness was detected,
* pcmk_rc_ipc_unauthorized indicating that the IPC endpoint is blocked by
* an unauthorized process, and pcmk_rc_ipc_pid_only indicating that
* the child is up by PID but not IPC end-point (possibly starting).
* \note This function doesn't modify any of \p child members but \c pid,
* and is not actively toying with processes as such but invoking
* \c stop_child in one particular case (there's for some reason
* a different authentic holder of the IPC end-point).
*/
static int
child_liveness(pcmk_child_t *child)
{
uid_t cl_uid = 0;
gid_t cl_gid = 0;
const uid_t root_uid = 0;
const gid_t root_gid = 0;
const uid_t *ref_uid;
const gid_t *ref_gid;
int rc = pcmk_rc_ipc_unresponsive;
pid_t ipc_pid = 0;
if (child->endpoint == NULL
&& (child->pid <= 0 || child->pid == PCMK__SPECIAL_PID)) {
crm_err("Cannot track child %s for missing both API end-point and PID",
child->name);
rc = EINVAL; // Misuse of function when child is not trackable
} else if (child->endpoint != NULL) {
int legacy_rc = pcmk_ok;
if (child->uid == NULL) {
ref_uid = &root_uid;
ref_gid = &root_gid;
} else {
ref_uid = &cl_uid;
ref_gid = &cl_gid;
legacy_rc = pcmk_daemon_user(&cl_uid, &cl_gid);
}
if (legacy_rc < 0) {
rc = pcmk_legacy2rc(legacy_rc);
crm_err("Could not find user and group IDs for user %s: %s "
CRM_XS " rc=%d", CRM_DAEMON_USER, pcmk_rc_str(rc), rc);
} else {
rc = pcmk__ipc_is_authentic_process_active(child->endpoint,
*ref_uid, *ref_gid,
&ipc_pid);
if ((rc == pcmk_rc_ok) || (rc == pcmk_rc_ipc_unresponsive)) {
if (child->pid <= 0) {
/* If rc is pcmk_rc_ok, ipc_pid is nonzero and this
* initializes a new child. If rc is
* pcmk_rc_ipc_unresponsive, ipc_pid is zero, and we will
* investigate further.
*/
child->pid = ipc_pid;
} else if ((ipc_pid != 0) && (child->pid != ipc_pid)) {
/* An unexpected (but authorized) process is responding to
* IPC. Investigate further.
*/
rc = pcmk_rc_ipc_unresponsive;
}
}
}
}
if (rc == pcmk_rc_ipc_unresponsive) {
/* If we get here, a child without IPC is being tracked, no IPC liveness
* has been detected, or IPC liveness has been detected with an
* unexpected (but authorized) process. This is safe on FreeBSD since
* the only change possible from a proper child's PID into "special" PID
* of 1 behind more loosely related process.
*/
int ret = pcmk__pid_active(child->pid, child->name);
if (ipc_pid && ((ret != pcmk_rc_ok)
|| ipc_pid == PCMK__SPECIAL_PID
|| (pcmk__pid_active(ipc_pid,
child->name) == pcmk_rc_ok))) {
/* An unexpected (but authorized) process was detected at the IPC
* endpoint, and either it is active, or the child we're tracking is
* not.
*/
if (ret == pcmk_rc_ok) {
/* The child we're tracking is active. Kill it, and adopt the
* detected process. This assumes that our children don't fork
* (thus getting a different PID owning the IPC), but rather the
* tracking got out of sync because of some means external to
* Pacemaker, and adopting the detected process is better than
* killing it and possibly having to spawn a new child.
*/
/* not possessing IPC, afterall (what about corosync CPG?) */
stop_child(child, SIGKILL);
}
rc = pcmk_rc_ok;
child->pid = ipc_pid;
} else if (ret == pcmk_rc_ok) {
// Our tracked child's PID was found active, but not its IPC
rc = pcmk_rc_ipc_pid_only;
} else if ((child->pid == 0) && (ret == EINVAL)) {
// FreeBSD can return EINVAL
rc = pcmk_rc_ipc_unresponsive;
} else {
switch (ret) {
case EACCES:
rc = pcmk_rc_ipc_unauthorized;
break;
case ESRCH:
rc = pcmk_rc_ipc_unresponsive;
break;
default:
rc = ret;
break;
}
}
}
return rc;
}
/*!
* \internal
* \brief Initial one-off check of the pre-existing "child" processes
*
* With "child" process, we mean the subdaemon that defines an API end-point
* (all of them do as of the comment) -- the possible complement is skipped
* as it is deemed it has no such shared resources to cause conflicts about,
* hence it can presumably be started anew without hesitation.
* If that won't hold true in the future, the concept of a shared resource
* will have to be generalized beyond the API end-point.
*
* For boundary cases that the "child" is still starting (IPC end-point is yet
* to be witnessed), or more rarely (practically FreeBSD only), when there's
* a pre-existing "untrackable" authentic process, we give the situation some
* time to possibly unfold in the right direction, meaning that said socket
* will appear or the unattainable process will disappear per the observable
* IPC, respectively.
*
* \return Standard Pacemaker return code
*
* \note Since this gets run at the very start, \c respawn_count fields
* for particular children get temporarily overloaded with "rounds
* of waiting" tracking, restored once we are about to finish with
* success (i.e. returning value >=0) and will remain unrestored
* otherwise. One way to suppress liveness detection logic for
* particular child is to set the said value to a negative number.
*/
#define WAIT_TRIES 4 /* together with interleaved sleeps, worst case ~ 1s */
int
find_and_track_existing_processes(void)
{
bool wait_in_progress;
int rc;
size_t i, rounds;
for (rounds = 1; rounds <= WAIT_TRIES; rounds++) {
wait_in_progress = false;
for (i = 0; i < PCMK__NELEM(pcmk_children); i++) {
if ((pcmk_children[i].endpoint == NULL)
|| (pcmk_children[i].respawn_count < 0)) {
continue;
}
rc = child_liveness(&pcmk_children[i]);
if (rc == pcmk_rc_ipc_unresponsive) {
/* As a speculation, don't give up if there are more rounds to
* come for other reasons, but don't artificially wait just
* because of this, since we would preferably start ASAP.
*/
continue;
}
+ // @TODO Functionize more of this to reduce nesting
pcmk_children[i].respawn_count = rounds;
switch (rc) {
case pcmk_rc_ok:
if (pcmk_children[i].pid == PCMK__SPECIAL_PID) {
- if (crm_is_true(getenv("PCMK_fail_fast"))) {
+ if (crm_is_true(pcmk__env_option(PCMK__ENV_FAIL_FAST))) {
crm_crit("Cannot reliably track pre-existing"
" authentic process behind %s IPC on this"
- " platform and PCMK_fail_fast requested",
+ " platform and PCMK_" PCMK__ENV_FAIL_FAST
+ " requested",
pcmk_children[i].endpoint);
return EOPNOTSUPP;
} else if (pcmk_children[i].respawn_count == WAIT_TRIES) {
crm_notice("Assuming pre-existing authentic, though"
" on this platform untrackable, process"
" behind %s IPC is stable (was in %d"
" previous samples) so rather than"
- " bailing out (PCMK_fail_fast not"
- " requested), we just switch to a less"
- " optimal IPC liveness monitoring"
+ " bailing out (PCMK_" PCMK__ENV_FAIL_FAST
+ " not requested), we just switch to a"
+ " less optimal IPC liveness monitoring"
" (not very suitable for heavy load)",
pcmk_children[i].name, WAIT_TRIES - 1);
crm_warn("The process behind %s IPC cannot be"
" terminated, so the overall shutdown"
" will get delayed implicitly (%ld s),"
" which serves as a graceful period for"
" its native termination if it vitally"
" depends on some other daemons going"
" down in a controlled way already",
pcmk_children[i].name,
(long) SHUTDOWN_ESCALATION_PERIOD);
} else {
wait_in_progress = true;
crm_warn("Cannot reliably track pre-existing"
" authentic process behind %s IPC on this"
" platform, can still disappear in %d"
" attempt(s)", pcmk_children[i].endpoint,
WAIT_TRIES - pcmk_children[i].respawn_count);
continue;
}
}
crm_notice("Tracking existing %s process (pid=%lld)",
pcmk_children[i].name,
(long long) PCMK__SPECIAL_PID_AS_0(
pcmk_children[i].pid));
pcmk_children[i].respawn_count = -1; /* 0~keep watching */
pcmk_children[i].active_before_startup = true;
break;
case pcmk_rc_ipc_pid_only:
if (pcmk_children[i].respawn_count == WAIT_TRIES) {
crm_crit("%s IPC end-point for existing authentic"
" process %lld did not (re)appear",
pcmk_children[i].endpoint,
(long long) PCMK__SPECIAL_PID_AS_0(
pcmk_children[i].pid));
return rc;
}
wait_in_progress = true;
crm_warn("Cannot find %s IPC end-point for existing"
" authentic process %lld, can still (re)appear"
" in %d attempts (?)",
pcmk_children[i].endpoint,
(long long) PCMK__SPECIAL_PID_AS_0(
pcmk_children[i].pid),
WAIT_TRIES - pcmk_children[i].respawn_count);
continue;
default:
crm_crit("Checked liveness of %s: %s " CRM_XS " rc=%d",
pcmk_children[i].name, pcmk_rc_str(rc), rc);
return rc;
}
}
if (!wait_in_progress) {
break;
}
pcmk__sleep_ms(250); // Wait a bit for changes to possibly happen
}
for (i = 0; i < PCMK__NELEM(pcmk_children); i++) {
pcmk_children[i].respawn_count = 0; /* restore pristine state */
}
g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL, check_next_subdaemon,
NULL);
return pcmk_rc_ok;
}
gboolean
init_children_processes(void *user_data)
{
if (is_corosync_cluster()) {
/* Corosync clusters can drop root group access, because we set
* uidgid.gid.${gid}=1 via CMAP, which allows these processes to connect
* to corosync.
*/
need_root_group = false;
}
/* start any children that have not been detected */
for (int i = 0; i < PCMK__NELEM(pcmk_children); i++) {
if (pcmk_children[i].pid != 0) {
/* we are already tracking it */
continue;
}
start_child(&(pcmk_children[i]));
}
/* From this point on, any daemons being started will be due to
* respawning rather than node start.
*
* This may be useful for the daemons to know
*/
- setenv("PCMK_respawned", "true", 1);
+ pcmk__set_env_option(PCMK__ENV_RESPAWNED, "true", false);
pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_RUNNING;
return TRUE;
}
void
pcmk_shutdown(int nsig)
{
if (shutdown_trigger == NULL) {
shutdown_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, pcmk_shutdown_worker, NULL);
}
mainloop_set_trigger(shutdown_trigger);
}
void
restart_cluster_subdaemons(void)
{
for (int i = 0; i < PCMK__NELEM(pcmk_children); i++) {
if (!pcmk_children[i].needs_retry || pcmk_children[i].pid != 0) {
continue;
}
crm_notice("Respawning cluster-based subdaemon: %s", pcmk_children[i].name);
if (start_child(&pcmk_children[i])) {
pcmk_children[i].needs_retry = false;
}
}
}
static gboolean
stop_child(pcmk_child_t * child, int signal)
{
if (signal == 0) {
signal = SIGTERM;
}
/* why to skip PID of 1?
- FreeBSD ~ how untrackable process behind IPC is masqueraded as
- elsewhere: how "init" task is designated; in particular, in systemd
arrangement of socket-based activation, this is pretty real */
if (child->command == NULL || child->pid == PCMK__SPECIAL_PID) {
crm_debug("Nothing to do for child \"%s\" (process %lld)",
child->name, (long long) PCMK__SPECIAL_PID_AS_0(child->pid));
return TRUE;
}
if (child->pid <= 0) {
crm_trace("Client %s not running", child->name);
return TRUE;
}
errno = 0;
if (kill(child->pid, signal) == 0) {
crm_notice("Stopping %s "CRM_XS" sent signal %d to process %lld",
child->name, signal, (long long) child->pid);
} else {
crm_err("Could not stop %s (process %lld) with signal %d: %s",
child->name, (long long) child->pid, signal, strerror(errno));
}
return TRUE;
}
diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h
index 4157b58d73..f80f8fcded 100644
--- a/include/crm/common/options_internal.h
+++ b/include/crm/common/options_internal.h
@@ -1,118 +1,146 @@
/*
- * Copyright 2006-2022 the Pacemaker project contributors
+ * Copyright 2006-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef PCMK__OPTIONS_INTERNAL__H
# define PCMK__OPTIONS_INTERNAL__H
# ifndef PCMK__CONFIG_H
# define PCMK__CONFIG_H
# include <config.h> // _Noreturn
# endif
# include <glib.h> // GHashTable
# include <stdbool.h> // bool
_Noreturn void pcmk__cli_help(char cmd);
/*
* Environment variable option handling
*/
const char *pcmk__env_option(const char *option);
-void pcmk__set_env_option(const char *option, const char *value);
+void pcmk__set_env_option(const char *option, const char *value, bool compat);
bool pcmk__env_option_enabled(const char *daemon, const char *option);
/*
* Cluster option handling
*/
typedef struct pcmk__cluster_option_s {
const char *name;
const char *alt_name;
const char *type;
const char *values;
const char *default_value;
bool (*is_valid)(const char *);
const char *description_short;
const char *description_long;
} pcmk__cluster_option_t;
const char *pcmk__cluster_option(GHashTable *options,
const pcmk__cluster_option_t *option_list,
int len, const char *name);
gchar *pcmk__format_option_metadata(const char *name, const char *desc_short,
const char *desc_long,
pcmk__cluster_option_t *option_list,
int len);
void pcmk__validate_cluster_options(GHashTable *options,
pcmk__cluster_option_t *option_list,
int len);
bool pcmk__valid_interval_spec(const char *value);
bool pcmk__valid_boolean(const char *value);
bool pcmk__valid_number(const char *value);
bool pcmk__valid_positive_number(const char *value);
bool pcmk__valid_quorum(const char *value);
bool pcmk__valid_script(const char *value);
bool pcmk__valid_percentage(const char *value);
// from watchdog.c
long pcmk__get_sbd_timeout(void);
bool pcmk__get_sbd_sync_resource_startup(void);
long pcmk__auto_watchdog_timeout(void);
bool pcmk__valid_sbd_timeout(const char *value);
// Constants for environment variable names
+#define PCMK__ENV_AUTHKEY_LOCATION "authkey_location"
#define PCMK__ENV_BLACKBOX "blackbox"
+#define PCMK__ENV_CALLGRIND_ENABLED "callgrind_enabled"
+#define PCMK__ENV_CIB_TIMEOUT "cib_timeout"
#define PCMK__ENV_CLUSTER_TYPE "cluster_type"
#define PCMK__ENV_DEBUG "debug"
+#define PCMK__ENV_DH_MAX_BITS "dh_max_bits"
+#define PCMK__ENV_DH_MIN_BITS "dh_min_bits"
+#define PCMK__ENV_FAIL_FAST "fail_fast"
+#define PCMK__ENV_IPC_BUFFER "ipc_buffer"
+#define PCMK__ENV_IPC_TYPE "ipc_type"
#define PCMK__ENV_LOGFACILITY "logfacility"
#define PCMK__ENV_LOGFILE "logfile"
+#define PCMK__ENV_LOGFILE_MODE "logfile_mode"
#define PCMK__ENV_LOGPRIORITY "logpriority"
-#define PCMK__ENV_MCP "mcp"
+#define PCMK__ENV_NODE_ACTION_LIMIT "node_action_limit"
#define PCMK__ENV_NODE_START_STATE "node_start_state"
+#define PCMK__ENV_PANIC_ACTION "panic_action"
#define PCMK__ENV_PHYSICAL_HOST "physical_host"
-#define PCMK__ENV_QUORUM_TYPE "quorum_type"
+#define PCMK__ENV_REMOTE_ADDRESS "remote_address"
+#define PCMK__ENV_REMOTE_PID1 "remote_pid1"
+#define PCMK__ENV_REMOTE_PORT "remote_port"
+#define PCMK__ENV_RESPAWNED "respawned"
+#define PCMK__ENV_SCHEMA_DIRECTORY "schema_directory"
+#define PCMK__ENV_SERVICE "service"
#define PCMK__ENV_SHUTDOWN_DELAY "shutdown_delay"
#define PCMK__ENV_STDERR "stderr"
+#define PCMK__ENV_TLS_PRIORITIES "tls_priorities"
+#define PCMK__ENV_TRACE_BLACKBOX "trace_blackbox"
+#define PCMK__ENV_TRACE_FILES "trace_files"
+#define PCMK__ENV_TRACE_FORMATS "trace_formats"
+#define PCMK__ENV_TRACE_FUNCTIONS "trace_functions"
+#define PCMK__ENV_TRACE_TAGS "trace_tags"
+#define PCMK__ENV_VALGRIND_ENABLED "valgrind_enabled"
+
+// @COMPAT Drop at 3.0.0; likely last used in 1.1.24
+#define PCMK__ENV_MCP "mcp"
+
+// @COMPAT Drop at 3.0.0; added unused in 1.1.9
+#define PCMK__ENV_QUORUM_TYPE "quorum_type"
// Constants for cluster option names
#define PCMK__OPT_NODE_HEALTH_BASE "node-health-base"
#define PCMK__OPT_NODE_HEALTH_GREEN "node-health-green"
#define PCMK__OPT_NODE_HEALTH_RED "node-health-red"
#define PCMK__OPT_NODE_HEALTH_STRATEGY "node-health-strategy"
#define PCMK__OPT_NODE_HEALTH_YELLOW "node-health-yellow"
// Constants for meta-attribute names
#define PCMK__META_ALLOW_UNHEALTHY_NODES "allow-unhealthy-nodes"
// Constants for enumerated values for various options
#define PCMK__VALUE_CLUSTER "cluster"
#define PCMK__VALUE_CUSTOM "custom"
#define PCMK__VALUE_FENCING "fencing"
#define PCMK__VALUE_GREEN "green"
#define PCMK__VALUE_LOCAL "local"
#define PCMK__VALUE_MIGRATE_ON_RED "migrate-on-red"
#define PCMK__VALUE_NONE "none"
#define PCMK__VALUE_NOTHING "nothing"
#define PCMK__VALUE_ONLY_GREEN "only-green"
#define PCMK__VALUE_PROGRESSIVE "progressive"
#define PCMK__VALUE_QUORUM "quorum"
#define PCMK__VALUE_RED "red"
#define PCMK__VALUE_UNFENCING "unfencing"
#define PCMK__VALUE_YELLOW "yellow"
#endif // PCMK__OPTIONS_INTERNAL__H
diff --git a/lib/common/ipc_common.c b/lib/common/ipc_common.c
index d0c06365cd..a48b0e976a 100644
--- a/lib/common/ipc_common.c
+++ b/lib/common/ipc_common.c
@@ -1,110 +1,110 @@
/*
* Copyright 2004-2021 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <stdio.h>
#include <stdint.h> // uint64_t
#include <sys/types.h>
#include <crm/msg_xml.h>
#include "crmcommon_private.h"
#define MIN_MSG_SIZE 12336 // sizeof(struct qb_ipc_connection_response)
#define MAX_MSG_SIZE 128*1024 // 128k default
/*!
* \internal
* \brief Choose an IPC buffer size in bytes
*
* \param[in] max Use this value if environment/default is lower
*
* \return Maximum of max and value of PCMK_ipc_buffer (default 128KB)
*/
unsigned int
pcmk__ipc_buffer_size(unsigned int max)
{
static unsigned int global_max = 0;
if (global_max == 0) {
long long global_ll;
- if ((pcmk__scan_ll(getenv("PCMK_ipc_buffer"), &global_ll,
+ if ((pcmk__scan_ll(pcmk__env_option(PCMK__ENV_IPC_BUFFER), &global_ll,
0LL) != pcmk_rc_ok)
|| (global_ll <= 0)) {
global_max = MAX_MSG_SIZE; // Default for unset or invalid
} else if (global_ll < MIN_MSG_SIZE) {
global_max = MIN_MSG_SIZE;
} else if (global_ll > UINT_MAX) {
global_max = UINT_MAX;
} else {
global_max = (unsigned int) global_ll;
}
}
return QB_MAX(max, global_max);
}
/*!
* \brief Return pacemaker's default IPC buffer size
*
* \return IPC buffer size in bytes
*/
unsigned int
crm_ipc_default_buffer_size(void)
{
static unsigned int default_size = 0;
if (default_size == 0) {
default_size = pcmk__ipc_buffer_size(0);
}
return default_size;
}
/*!
* \internal
* \brief Check whether an IPC header is valid
*
* \param[in] header IPC header to check
*
* \return true if IPC header has a supported version, false otherwise
*/
bool
pcmk__valid_ipc_header(const pcmk__ipc_header_t *header)
{
if (header == NULL) {
crm_err("IPC message without header");
return false;
} else if (header->version > PCMK__IPC_VERSION) {
crm_err("Filtering incompatible v%d IPC message (only versions <= %d supported)",
header->version, PCMK__IPC_VERSION);
return false;
}
return true;
}
const char *
pcmk__client_type_str(uint64_t client_type)
{
switch (client_type) {
case pcmk__client_ipc:
return "IPC";
case pcmk__client_tcp:
return "TCP";
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
return "TLS";
#endif
default:
return "unknown";
}
}
diff --git a/lib/common/logging.c b/lib/common/logging.c
index fe9eac7398..955304084f 100644
--- a/lib/common/logging.c
+++ b/lib/common/logging.c
@@ -1,1281 +1,1282 @@
/*
* Copyright 2004-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <sys/utsname.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <limits.h>
#include <ctype.h>
#include <pwd.h>
#include <grp.h>
#include <time.h>
#include <libgen.h>
#include <signal.h>
#include <bzlib.h>
#include <qb/qbdefs.h>
#include <crm/crm.h>
#include <crm/common/mainloop.h>
// Use high-resolution (millisecond) timestamps if libqb supports them
#ifdef QB_FEATURE_LOG_HIRES_TIMESTAMPS
#define TIMESTAMP_FORMAT_SPEC "%%T"
typedef struct timespec *log_time_t;
#else
#define TIMESTAMP_FORMAT_SPEC "%%t"
typedef time_t log_time_t;
#endif
unsigned int crm_log_level = LOG_INFO;
unsigned int crm_trace_nonlog = 0;
bool pcmk__is_daemon = false;
char *pcmk__our_nodename = NULL;
static unsigned int crm_log_priority = LOG_NOTICE;
static GLogFunc glib_log_default = NULL;
static pcmk__output_t *logger_out = NULL;
static gboolean crm_tracing_enabled(void);
static void
crm_glib_handler(const gchar * log_domain, GLogLevelFlags flags, const gchar * message,
gpointer user_data)
{
int log_level = LOG_WARNING;
GLogLevelFlags msg_level = (flags & G_LOG_LEVEL_MASK);
static struct qb_log_callsite *glib_cs = NULL;
if (glib_cs == NULL) {
glib_cs = qb_log_callsite_get(__func__, __FILE__, "glib-handler",
LOG_DEBUG, __LINE__, crm_trace_nonlog);
}
switch (msg_level) {
case G_LOG_LEVEL_CRITICAL:
log_level = LOG_CRIT;
if (!crm_is_callsite_active(glib_cs, LOG_DEBUG, crm_trace_nonlog)) {
/* log and record how we got here */
crm_abort(__FILE__, __func__, __LINE__, message, TRUE, TRUE);
}
break;
case G_LOG_LEVEL_ERROR:
log_level = LOG_ERR;
break;
case G_LOG_LEVEL_MESSAGE:
log_level = LOG_NOTICE;
break;
case G_LOG_LEVEL_INFO:
log_level = LOG_INFO;
break;
case G_LOG_LEVEL_DEBUG:
log_level = LOG_DEBUG;
break;
case G_LOG_LEVEL_WARNING:
case G_LOG_FLAG_RECURSION:
case G_LOG_FLAG_FATAL:
case G_LOG_LEVEL_MASK:
log_level = LOG_WARNING;
break;
}
do_crm_log(log_level, "%s: %s", log_domain, message);
}
#ifndef NAME_MAX
# define NAME_MAX 256
#endif
/*!
* \internal
* \brief Write out a blackbox (enabling blackboxes if needed)
*
* \param[in] nsig Signal number that was received
*
* \note This is a true signal handler, and so must be async-safe.
*/
static void
crm_trigger_blackbox(int nsig)
{
if(nsig == SIGTRAP) {
/* Turn it on if it wasn't already */
crm_enable_blackbox(nsig);
}
crm_write_blackbox(nsig, NULL);
}
void
crm_log_deinit(void)
{
if (glib_log_default != NULL) {
g_log_set_default_handler(glib_log_default, NULL);
}
}
#define FMT_MAX 256
/*!
* \internal
* \brief Set the log format string based on the passed-in method
*
* \param[in] method The detail level of the log output
* \param[in] daemon The daemon ID included in error messages
* \param[in] use_pid Cached result of getpid() call, for efficiency
* \param[in] use_nodename Cached result of uname() call, for efficiency
*
*/
/* XXX __attribute__((nonnull)) for use_nodename parameter */
static void
set_format_string(int method, const char *daemon, pid_t use_pid,
const char *use_nodename)
{
if (method == QB_LOG_SYSLOG) {
// The system log gets a simplified, user-friendly format
crm_extended_logging(method, QB_FALSE);
qb_log_format_set(method, "%g %p: %b");
} else {
// Everything else gets more detail, for advanced troubleshooting
int offset = 0;
char fmt[FMT_MAX];
if (method > QB_LOG_STDERR) {
// If logging to file, prefix with timestamp, node name, daemon ID
offset += snprintf(fmt + offset, FMT_MAX - offset,
TIMESTAMP_FORMAT_SPEC " %s %-20s[%lu] ",
use_nodename, daemon, (unsigned long) use_pid);
}
// Add function name (in parentheses)
offset += snprintf(fmt + offset, FMT_MAX - offset, "(%%n");
if (crm_tracing_enabled()) {
// When tracing, add file and line number
offset += snprintf(fmt + offset, FMT_MAX - offset, "@%%f:%%l");
}
offset += snprintf(fmt + offset, FMT_MAX - offset, ")");
// Add tag (if any), severity, and actual message
offset += snprintf(fmt + offset, FMT_MAX - offset, " %%g\t%%p: %%b");
CRM_LOG_ASSERT(offset > 0);
qb_log_format_set(method, fmt);
}
}
#define DEFAULT_LOG_FILE CRM_LOG_DIR "/pacemaker.log"
static bool
logfile_disabled(const char *filename)
{
return pcmk__str_eq(filename, PCMK__VALUE_NONE, pcmk__str_casei)
|| pcmk__str_eq(filename, "/dev/null", pcmk__str_none);
}
/*!
* \internal
* \brief Fix log file ownership if group is wrong or doesn't have access
*
* \param[in] filename Log file name (for logging only)
* \param[in] logfd Log file descriptor
*
* \return Standard Pacemaker return code
*/
static int
chown_logfile(const char *filename, int logfd)
{
uid_t pcmk_uid = 0;
gid_t pcmk_gid = 0;
struct stat st;
int rc;
// Get the log file's current ownership and permissions
if (fstat(logfd, &st) < 0) {
return errno;
}
// Any other errors don't prevent file from being used as log
rc = pcmk_daemon_user(&pcmk_uid, &pcmk_gid);
if (rc != pcmk_ok) {
rc = pcmk_legacy2rc(rc);
crm_warn("Not changing '%s' ownership because user information "
"unavailable: %s", filename, pcmk_rc_str(rc));
return pcmk_rc_ok;
}
if ((st.st_gid == pcmk_gid)
&& ((st.st_mode & S_IRWXG) == (S_IRGRP|S_IWGRP))) {
return pcmk_rc_ok;
}
if (fchown(logfd, pcmk_uid, pcmk_gid) < 0) {
crm_warn("Couldn't change '%s' ownership to user %s gid %d: %s",
filename, CRM_DAEMON_USER, pcmk_gid, strerror(errno));
}
return pcmk_rc_ok;
}
// Reset log file permissions (using environment variable if set)
static void
chmod_logfile(const char *filename, int logfd)
{
- const char *modestr = getenv("PCMK_logfile_mode");
+ const char *modestr = pcmk__env_option(PCMK__ENV_LOGFILE_MODE);
mode_t filemode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP;
if (modestr != NULL) {
long filemode_l = strtol(modestr, NULL, 8);
if ((filemode_l != LONG_MIN) && (filemode_l != LONG_MAX)) {
filemode = (mode_t) filemode_l;
}
}
if ((filemode != 0) && (fchmod(logfd, filemode) < 0)) {
crm_warn("Couldn't change '%s' mode to %04o: %s",
filename, filemode, strerror(errno));
}
}
// If we're root, correct a log file's permissions if needed
static int
set_logfile_permissions(const char *filename, FILE *logfile)
{
if (geteuid() == 0) {
int logfd = fileno(logfile);
int rc = chown_logfile(filename, logfd);
if (rc != pcmk_rc_ok) {
return rc;
}
chmod_logfile(filename, logfd);
}
return pcmk_rc_ok;
}
// Enable libqb logging to a new log file
static void
enable_logfile(int fd)
{
qb_log_ctl(fd, QB_LOG_CONF_ENABLED, QB_TRUE);
#if 0
qb_log_ctl(fd, QB_LOG_CONF_FILE_SYNC, 1); // Turn on synchronous writes
#endif
#ifdef HAVE_qb_log_conf_QB_LOG_CONF_MAX_LINE_LEN
// Longer than default, for logging long XML lines
qb_log_ctl(fd, QB_LOG_CONF_MAX_LINE_LEN, 800);
#endif
crm_update_callsites();
}
static inline void
disable_logfile(int fd)
{
qb_log_ctl(fd, QB_LOG_CONF_ENABLED, QB_FALSE);
}
static void
setenv_logfile(const char *filename)
{
// Some resource agents will log only if environment variable is set
if (pcmk__env_option(PCMK__ENV_LOGFILE) == NULL) {
- pcmk__set_env_option(PCMK__ENV_LOGFILE, filename);
+ pcmk__set_env_option(PCMK__ENV_LOGFILE, filename, true);
}
}
/*!
* \brief Add a file to be used as a Pacemaker detail log
*
* \param[in] filename Name of log file to use
*
* \return Standard Pacemaker return code
*/
int
pcmk__add_logfile(const char *filename)
{
/* No log messages from this function will be logged to the new log!
* If another target such as syslog has already been added, the messages
* should show up there.
*/
int fd = 0;
int rc = pcmk_rc_ok;
FILE *logfile = NULL;
bool is_default = false;
static int default_fd = -1;
static bool have_logfile = false;
// Use default if caller didn't specify (and we don't already have one)
if (filename == NULL) {
if (have_logfile) {
return pcmk_rc_ok;
}
filename = DEFAULT_LOG_FILE;
}
// If the user doesn't want logging, we're done
if (logfile_disabled(filename)) {
return pcmk_rc_ok;
}
// If the caller wants the default and we already have it, we're done
is_default = pcmk__str_eq(filename, DEFAULT_LOG_FILE, pcmk__str_none);
if (is_default && (default_fd >= 0)) {
return pcmk_rc_ok;
}
// Check whether we have write access to the file
logfile = fopen(filename, "a");
if (logfile == NULL) {
rc = errno;
crm_warn("Logging to '%s' is disabled: %s " CRM_XS " uid=%u gid=%u",
filename, strerror(rc), geteuid(), getegid());
return rc;
}
rc = set_logfile_permissions(filename, logfile);
if (rc != pcmk_rc_ok) {
crm_warn("Logging to '%s' is disabled: %s " CRM_XS " permissions",
filename, strerror(rc));
fclose(logfile);
return rc;
}
// Close and reopen as libqb logging target
fclose(logfile);
fd = qb_log_file_open(filename);
if (fd < 0) {
crm_warn("Logging to '%s' is disabled: %s " CRM_XS " qb_log_file_open",
filename, strerror(-fd));
return -fd; // == +errno
}
if (is_default) {
default_fd = fd;
setenv_logfile(filename);
} else if (default_fd >= 0) {
crm_notice("Switching logging to %s", filename);
disable_logfile(default_fd);
}
crm_notice("Additional logging available in %s", filename);
enable_logfile(fd);
have_logfile = true;
return pcmk_rc_ok;
}
/*!
* \brief Add multiple additional log files
*
* \param[in] log_files Array of log files to add
* \param[in] out Output object to use for error reporting
*
* \return Standard Pacemaker return code
*/
void
pcmk__add_logfiles(gchar **log_files, pcmk__output_t *out)
{
if (log_files == NULL) {
return;
}
for (gchar **fname = log_files; *fname != NULL; fname++) {
int rc = pcmk__add_logfile(*fname);
if (rc != pcmk_rc_ok) {
out->err(out, "Logging to %s is disabled: %s",
*fname, pcmk_rc_str(rc));
}
}
}
static int blackbox_trigger = 0;
static volatile char *blackbox_file_prefix = NULL;
static void
blackbox_logger(int32_t t, struct qb_log_callsite *cs, log_time_t timestamp,
const char *msg)
{
if(cs && cs->priority < LOG_ERR) {
crm_write_blackbox(SIGTRAP, cs); /* Bypass the over-dumping logic */
} else {
crm_write_blackbox(0, cs);
}
}
static void
crm_control_blackbox(int nsig, bool enable)
{
int lpc = 0;
if (blackbox_file_prefix == NULL) {
pid_t pid = getpid();
blackbox_file_prefix = crm_strdup_printf("%s/%s-%lu",
CRM_BLACKBOX_DIR,
crm_system_name,
(unsigned long) pid);
}
if (enable && qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_STATE_GET, 0) != QB_LOG_STATE_ENABLED) {
qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_SIZE, 5 * 1024 * 1024); /* Any size change drops existing entries */
qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_TRUE); /* Setting the size seems to disable it */
/* Enable synchronous logging */
for (lpc = QB_LOG_BLACKBOX; lpc < QB_LOG_TARGET_MAX; lpc++) {
qb_log_ctl(lpc, QB_LOG_CONF_FILE_SYNC, QB_TRUE);
}
crm_notice("Initiated blackbox recorder: %s", blackbox_file_prefix);
/* Save to disk on abnormal termination */
crm_signal_handler(SIGSEGV, crm_trigger_blackbox);
crm_signal_handler(SIGABRT, crm_trigger_blackbox);
crm_signal_handler(SIGILL, crm_trigger_blackbox);
crm_signal_handler(SIGBUS, crm_trigger_blackbox);
crm_signal_handler(SIGFPE, crm_trigger_blackbox);
crm_update_callsites();
blackbox_trigger = qb_log_custom_open(blackbox_logger, NULL, NULL, NULL);
qb_log_ctl(blackbox_trigger, QB_LOG_CONF_ENABLED, QB_TRUE);
crm_trace("Trigger: %d is %d %d", blackbox_trigger,
qb_log_ctl(blackbox_trigger, QB_LOG_CONF_STATE_GET, 0), QB_LOG_STATE_ENABLED);
crm_update_callsites();
} else if (!enable && qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_STATE_GET, 0) == QB_LOG_STATE_ENABLED) {
qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_FALSE);
/* Disable synchronous logging again when the blackbox is disabled */
for (lpc = QB_LOG_BLACKBOX; lpc < QB_LOG_TARGET_MAX; lpc++) {
qb_log_ctl(lpc, QB_LOG_CONF_FILE_SYNC, QB_FALSE);
}
}
}
void
crm_enable_blackbox(int nsig)
{
crm_control_blackbox(nsig, TRUE);
}
void
crm_disable_blackbox(int nsig)
{
crm_control_blackbox(nsig, FALSE);
}
/*!
* \internal
* \brief Write out a blackbox, if blackboxes are enabled
*
* \param[in] nsig Signal that was received
* \param[in] cs libqb callsite
*
* \note This may be called via a true signal handler and so must be async-safe.
* @TODO actually make this async-safe
*/
void
crm_write_blackbox(int nsig, const struct qb_log_callsite *cs)
{
static volatile int counter = 1;
static volatile time_t last = 0;
char buffer[NAME_MAX];
time_t now = time(NULL);
if (blackbox_file_prefix == NULL) {
return;
}
switch (nsig) {
case 0:
case SIGTRAP:
/* The graceful case - such as assertion failure or user request */
if (nsig == 0 && now == last) {
/* Prevent over-dumping */
return;
}
snprintf(buffer, NAME_MAX, "%s.%d", blackbox_file_prefix, counter++);
if (nsig == SIGTRAP) {
crm_notice("Blackbox dump requested, please see %s for contents", buffer);
} else if (cs) {
syslog(LOG_NOTICE,
"Problem detected at %s:%d (%s), please see %s for additional details",
cs->function, cs->lineno, cs->filename, buffer);
} else {
crm_notice("Problem detected, please see %s for additional details", buffer);
}
last = now;
qb_log_blackbox_write_to_file(buffer);
/* Flush the existing contents
* A size change would also work
*/
qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_FALSE);
qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_TRUE);
break;
default:
/* Do as little as possible, just try to get what we have out
* We logged the filename when the blackbox was enabled
*/
crm_signal_handler(nsig, SIG_DFL);
qb_log_blackbox_write_to_file((const char *)blackbox_file_prefix);
qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_FALSE);
raise(nsig);
break;
}
}
static const char *
crm_quark_to_string(uint32_t tag)
{
const char *text = g_quark_to_string(tag);
if (text) {
return text;
}
return "";
}
static void
crm_log_filter_source(int source, const char *trace_files, const char *trace_fns,
const char *trace_fmts, const char *trace_tags, const char *trace_blackbox,
struct qb_log_callsite *cs)
{
if (qb_log_ctl(source, QB_LOG_CONF_STATE_GET, 0) != QB_LOG_STATE_ENABLED) {
return;
} else if (cs->tags != crm_trace_nonlog && source == QB_LOG_BLACKBOX) {
/* Blackbox gets everything if enabled */
qb_bit_set(cs->targets, source);
} else if (source == blackbox_trigger && blackbox_trigger > 0) {
/* Should this log message result in the blackbox being dumped */
if (cs->priority <= LOG_ERR) {
qb_bit_set(cs->targets, source);
} else if (trace_blackbox) {
char *key = crm_strdup_printf("%s:%d", cs->function, cs->lineno);
if (strstr(trace_blackbox, key) != NULL) {
qb_bit_set(cs->targets, source);
}
free(key);
}
} else if (source == QB_LOG_SYSLOG) { /* No tracing to syslog */
if (cs->priority <= crm_log_priority && cs->priority <= crm_log_level) {
qb_bit_set(cs->targets, source);
}
/* Log file tracing options... */
} else if (cs->priority <= crm_log_level) {
qb_bit_set(cs->targets, source);
} else if (trace_files && strstr(trace_files, cs->filename) != NULL) {
qb_bit_set(cs->targets, source);
} else if (trace_fns && strstr(trace_fns, cs->function) != NULL) {
qb_bit_set(cs->targets, source);
} else if (trace_fmts && strstr(trace_fmts, cs->format) != NULL) {
qb_bit_set(cs->targets, source);
} else if (trace_tags
&& cs->tags != 0
&& cs->tags != crm_trace_nonlog && g_quark_to_string(cs->tags) != NULL) {
qb_bit_set(cs->targets, source);
}
}
#ifndef HAVE_STRCHRNUL
/* strchrnul() is a GNU extension. If not present, use our own definition.
* The GNU version returns char*, but we only need it to be const char*.
*/
static const char *
strchrnul(const char *s, int c)
{
while ((*s != c) && (*s != '\0')) {
++s;
}
return s;
}
#endif
static void
crm_log_filter(struct qb_log_callsite *cs)
{
int lpc = 0;
static int need_init = 1;
static const char *trace_fns = NULL;
static const char *trace_tags = NULL;
static const char *trace_fmts = NULL;
static const char *trace_files = NULL;
static const char *trace_blackbox = NULL;
if (need_init) {
need_init = 0;
- trace_fns = getenv("PCMK_trace_functions");
- trace_fmts = getenv("PCMK_trace_formats");
- trace_tags = getenv("PCMK_trace_tags");
- trace_files = getenv("PCMK_trace_files");
- trace_blackbox = getenv("PCMK_trace_blackbox");
+ trace_fns = pcmk__env_option(PCMK__ENV_TRACE_FUNCTIONS);
+ trace_fmts = pcmk__env_option(PCMK__ENV_TRACE_FORMATS);
+ trace_tags = pcmk__env_option(PCMK__ENV_TRACE_TAGS);
+ trace_files = pcmk__env_option(PCMK__ENV_TRACE_FILES);
+ trace_blackbox = pcmk__env_option(PCMK__ENV_TRACE_BLACKBOX);
if (trace_tags != NULL) {
uint32_t tag;
char token[500];
const char *offset = NULL;
const char *next = trace_tags;
do {
offset = next;
next = strchrnul(offset, ',');
snprintf(token, sizeof(token), "%.*s", (int)(next - offset), offset);
tag = g_quark_from_string(token);
crm_info("Created GQuark %u from token '%s' in '%s'", tag, token, trace_tags);
if (next[0] != 0) {
next++;
}
} while (next != NULL && next[0] != 0);
}
}
cs->targets = 0; /* Reset then find targets to enable */
for (lpc = QB_LOG_SYSLOG; lpc < QB_LOG_TARGET_MAX; lpc++) {
crm_log_filter_source(lpc, trace_files, trace_fns, trace_fmts, trace_tags, trace_blackbox,
cs);
}
}
gboolean
crm_is_callsite_active(struct qb_log_callsite *cs, uint8_t level, uint32_t tags)
{
gboolean refilter = FALSE;
if (cs == NULL) {
return FALSE;
}
if (cs->priority != level) {
cs->priority = level;
refilter = TRUE;
}
if (cs->tags != tags) {
cs->tags = tags;
refilter = TRUE;
}
if (refilter) {
crm_log_filter(cs);
}
if (cs->targets == 0) {
return FALSE;
}
return TRUE;
}
void
crm_update_callsites(void)
{
static gboolean log = TRUE;
if (log) {
log = FALSE;
crm_debug
("Enabling callsites based on priority=%d, files=%s, functions=%s, formats=%s, tags=%s",
- crm_log_level, getenv("PCMK_trace_files"), getenv("PCMK_trace_functions"),
- getenv("PCMK_trace_formats"), getenv("PCMK_trace_tags"));
+ crm_log_level, pcmk__env_option(PCMK__ENV_TRACE_FILES),
+ pcmk__env_option(PCMK__ENV_TRACE_FUNCTIONS),
+ pcmk__env_option(PCMK__ENV_TRACE_FORMATS),
+ pcmk__env_option(PCMK__ENV_TRACE_TAGS));
}
qb_log_filter_fn_set(crm_log_filter);
}
static gboolean
crm_tracing_enabled(void)
{
- if (crm_log_level == LOG_TRACE) {
- return TRUE;
- } else if (getenv("PCMK_trace_files") || getenv("PCMK_trace_functions")
- || getenv("PCMK_trace_formats") || getenv("PCMK_trace_tags")) {
- return TRUE;
- }
- return FALSE;
+ return (crm_log_level == LOG_TRACE)
+ || (pcmk__env_option(PCMK__ENV_TRACE_FILES) != NULL)
+ || (pcmk__env_option(PCMK__ENV_TRACE_FUNCTIONS) != NULL)
+ || (pcmk__env_option(PCMK__ENV_TRACE_FORMATS) != NULL)
+ || (pcmk__env_option(PCMK__ENV_TRACE_TAGS) != NULL);
}
static int
crm_priority2int(const char *name)
{
struct syslog_names {
const char *name;
int priority;
};
static struct syslog_names p_names[] = {
{"emerg", LOG_EMERG},
{"alert", LOG_ALERT},
{"crit", LOG_CRIT},
{"error", LOG_ERR},
{"warning", LOG_WARNING},
{"notice", LOG_NOTICE},
{"info", LOG_INFO},
{"debug", LOG_DEBUG},
{NULL, -1}
};
int lpc;
for (lpc = 0; name != NULL && p_names[lpc].name != NULL; lpc++) {
if (pcmk__str_eq(p_names[lpc].name, name, pcmk__str_none)) {
return p_names[lpc].priority;
}
}
return crm_log_priority;
}
/*!
* \internal
* \brief Set the identifier for the current process
*
* If the identifier crm_system_name is not already set, then it is set as follows:
* - it is passed to the function via the "entity" parameter, or
* - it is derived from the executable name
*
* The identifier can be used in logs, IPC, and more.
*
* This method also sets the PCMK_service environment variable.
*
* \param[in] entity If not NULL, will be assigned to the identifier
* \param[in] argc The number of command line parameters
* \param[in] argv The command line parameter values
*/
static void
set_identity(const char *entity, int argc, char *const *argv)
{
if (crm_system_name != NULL) {
return; // Already set, don't overwrite
}
if (entity != NULL) {
crm_system_name = strdup(entity);
} else if ((argc > 0) && (argv != NULL)) {
char *mutable = strdup(argv[0]);
char *modified = basename(mutable);
if (strstr(modified, "lt-") == modified) {
modified += 3;
}
crm_system_name = strdup(modified);
free(mutable);
} else {
crm_system_name = strdup("Unknown");
}
CRM_ASSERT(crm_system_name != NULL);
- setenv("PCMK_service", crm_system_name, 1);
+ // Used by fencing.py.py (in fence-agents)
+ pcmk__set_env_option(PCMK__ENV_SERVICE, crm_system_name, false);
}
void
crm_log_preinit(const char *entity, int argc, char *const *argv)
{
/* Configure libqb logging with nothing turned on */
struct utsname res;
int lpc = 0;
int32_t qb_facility = 0;
pid_t pid = getpid();
const char *nodename = "localhost";
static bool have_logging = false;
if (have_logging) {
return;
}
have_logging = true;
crm_xml_init(); /* Sets buffer allocation strategy */
if (crm_trace_nonlog == 0) {
crm_trace_nonlog = g_quark_from_static_string("Pacemaker non-logging tracepoint");
}
umask(S_IWGRP | S_IWOTH | S_IROTH);
/* Redirect messages from glib functions to our handler */
glib_log_default = g_log_set_default_handler(crm_glib_handler, NULL);
/* and for good measure... - this enum is a bit field (!) */
g_log_set_always_fatal((GLogLevelFlags) 0); /*value out of range */
/* Set crm_system_name, which is used as the logging name. It may also
* be used for other purposes such as an IPC client name.
*/
set_identity(entity, argc, argv);
qb_facility = qb_log_facility2int("local0");
qb_log_init(crm_system_name, qb_facility, LOG_ERR);
crm_log_level = LOG_CRIT;
/* Nuke any syslog activity until it's asked for */
qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_ENABLED, QB_FALSE);
#ifdef HAVE_qb_log_conf_QB_LOG_CONF_MAX_LINE_LEN
// Shorter than default, generous for what we *should* send to syslog
qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_MAX_LINE_LEN, 256);
#endif
if (uname(memset(&res, 0, sizeof(res))) == 0 && *res.nodename != '\0') {
nodename = res.nodename;
}
/* Set format strings and disable threading
* Pacemaker and threads do not mix well (due to the amount of forking)
*/
qb_log_tags_stringify_fn_set(crm_quark_to_string);
for (lpc = QB_LOG_SYSLOG; lpc < QB_LOG_TARGET_MAX; lpc++) {
qb_log_ctl(lpc, QB_LOG_CONF_THREADED, QB_FALSE);
#ifdef HAVE_qb_log_conf_QB_LOG_CONF_ELLIPSIS
// End truncated lines with '...'
qb_log_ctl(lpc, QB_LOG_CONF_ELLIPSIS, QB_TRUE);
#endif
set_format_string(lpc, crm_system_name, pid, nodename);
}
#ifdef ENABLE_NLS
/* Enable translations (experimental). Currently we only have a few
* proof-of-concept translations for some option help. The goal would be to
* offer translations for option help and man pages rather than logs or
* documentation, to reduce the burden of maintaining them.
*/
// Load locale information for the local host from the environment
setlocale(LC_ALL, "");
// Tell gettext where to find Pacemaker message catalogs
CRM_ASSERT(bindtextdomain(PACKAGE, PCMK__LOCALE_DIR) != NULL);
// Tell gettext to use the Pacemaker message catalogs
CRM_ASSERT(textdomain(PACKAGE) != NULL);
// Tell gettext that the translated strings are stored in UTF-8
bind_textdomain_codeset(PACKAGE, "UTF-8");
#endif
}
gboolean
crm_log_init(const char *entity, uint8_t level, gboolean daemon, gboolean to_stderr,
int argc, char **argv, gboolean quiet)
{
const char *syslog_priority = NULL;
const char *facility = pcmk__env_option(PCMK__ENV_LOGFACILITY);
const char *f_copy = facility;
pcmk__is_daemon = daemon;
crm_log_preinit(entity, argc, argv);
if (level > LOG_TRACE) {
level = LOG_TRACE;
}
if(level > crm_log_level) {
crm_log_level = level;
}
/* Should we log to syslog */
if (facility == NULL) {
if (pcmk__is_daemon) {
facility = "daemon";
} else {
facility = PCMK__VALUE_NONE;
}
- pcmk__set_env_option(PCMK__ENV_LOGFACILITY, facility);
+ pcmk__set_env_option(PCMK__ENV_LOGFACILITY, facility, true);
}
if (pcmk__str_eq(facility, PCMK__VALUE_NONE, pcmk__str_casei)) {
quiet = TRUE;
} else {
qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_FACILITY, qb_log_facility2int(facility));
}
if (pcmk__env_option_enabled(crm_system_name, PCMK__ENV_DEBUG)) {
/* Override the default setting */
crm_log_level = LOG_DEBUG;
}
/* What lower threshold do we have for sending to syslog */
syslog_priority = pcmk__env_option(PCMK__ENV_LOGPRIORITY);
if (syslog_priority) {
crm_log_priority = crm_priority2int(syslog_priority);
}
qb_log_filter_ctl(QB_LOG_SYSLOG, QB_LOG_FILTER_ADD, QB_LOG_FILTER_FILE, "*",
crm_log_priority);
// Log to syslog unless requested to be quiet
if (!quiet) {
qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_ENABLED, QB_TRUE);
}
/* Should we log to stderr */
if (pcmk__env_option_enabled(crm_system_name, PCMK__ENV_STDERR)) {
/* Override the default setting */
to_stderr = TRUE;
}
crm_enable_stderr(to_stderr);
// Log to a file if we're a daemon or user asked for one
{
const char *logfile = pcmk__env_option(PCMK__ENV_LOGFILE);
if (!pcmk__str_eq(PCMK__VALUE_NONE, logfile, pcmk__str_casei)
&& (pcmk__is_daemon || (logfile != NULL))) {
// Daemons always get a log file, unless explicitly set to "none"
pcmk__add_logfile(logfile);
}
}
if (pcmk__is_daemon
&& pcmk__env_option_enabled(crm_system_name, PCMK__ENV_BLACKBOX)) {
crm_enable_blackbox(0);
}
/* Summary */
crm_trace("Quiet: %d, facility %s", quiet, f_copy);
pcmk__env_option(PCMK__ENV_LOGFILE);
pcmk__env_option(PCMK__ENV_LOGFACILITY);
crm_update_callsites();
/* Ok, now we can start logging... */
// Disable daemon request if user isn't root or Pacemaker daemon user
if (pcmk__is_daemon) {
const char *user = getenv("USER");
if (user != NULL && !pcmk__strcase_any_of(user, "root", CRM_DAEMON_USER, NULL)) {
crm_trace("Not switching to corefile directory for %s", user);
pcmk__is_daemon = false;
}
}
if (pcmk__is_daemon) {
int user = getuid();
struct passwd *pwent = getpwuid(user);
if (pwent == NULL) {
crm_perror(LOG_ERR, "Cannot get name for uid: %d", user);
} else if (!pcmk__strcase_any_of(pwent->pw_name, "root", CRM_DAEMON_USER, NULL)) {
crm_trace("Don't change active directory for regular user: %s", pwent->pw_name);
} else if (chdir(CRM_CORE_DIR) < 0) {
crm_perror(LOG_INFO, "Cannot change active directory to " CRM_CORE_DIR);
} else {
crm_info("Changed active directory to " CRM_CORE_DIR);
}
/* Original meanings from signal(7)
*
* Signal Value Action Comment
* SIGTRAP 5 Core Trace/breakpoint trap
* SIGUSR1 30,10,16 Term User-defined signal 1
* SIGUSR2 31,12,17 Term User-defined signal 2
*
* Our usage is as similar as possible
*/
mainloop_add_signal(SIGUSR1, crm_enable_blackbox);
mainloop_add_signal(SIGUSR2, crm_disable_blackbox);
mainloop_add_signal(SIGTRAP, crm_trigger_blackbox);
} else if (!quiet) {
crm_log_args(argc, argv);
}
return TRUE;
}
/* returns the old value */
unsigned int
set_crm_log_level(unsigned int level)
{
unsigned int old = crm_log_level;
if (level > LOG_TRACE) {
level = LOG_TRACE;
}
crm_log_level = level;
crm_update_callsites();
crm_trace("New log level: %d", level);
return old;
}
void
crm_enable_stderr(int enable)
{
if (enable && qb_log_ctl(QB_LOG_STDERR, QB_LOG_CONF_STATE_GET, 0) != QB_LOG_STATE_ENABLED) {
qb_log_ctl(QB_LOG_STDERR, QB_LOG_CONF_ENABLED, QB_TRUE);
crm_update_callsites();
} else if (enable == FALSE) {
qb_log_ctl(QB_LOG_STDERR, QB_LOG_CONF_ENABLED, QB_FALSE);
}
}
/*!
* \brief Make logging more verbose
*
* If logging to stderr is not already enabled when this function is called,
* enable it. Otherwise, increase the log level by 1.
*
* \param[in] argc Ignored
* \param[in] argv Ignored
*/
void
crm_bump_log_level(int argc, char **argv)
{
if (qb_log_ctl(QB_LOG_STDERR, QB_LOG_CONF_STATE_GET, 0)
!= QB_LOG_STATE_ENABLED) {
crm_enable_stderr(TRUE);
} else {
set_crm_log_level(crm_log_level + 1);
}
}
unsigned int
get_crm_log_level(void)
{
return crm_log_level;
}
/*!
* \brief Log the command line (once)
*
* \param[in] Number of values in \p argv
* \param[in] Command-line arguments (including command name)
*
* \note This function will only log once, even if called with different
* arguments.
*/
void
crm_log_args(int argc, char **argv)
{
static bool logged = false;
gchar *arg_string = NULL;
if ((argc == 0) || (argv == NULL) || logged) {
return;
}
logged = true;
arg_string = g_strjoinv(" ", argv);
crm_notice("Invoked: %s", arg_string);
g_free(arg_string);
}
void
crm_log_output_fn(const char *file, const char *function, int line, int level, const char *prefix,
const char *output)
{
const char *next = NULL;
const char *offset = NULL;
if (level == LOG_NEVER) {
return;
}
if (output == NULL) {
if (level != LOG_STDOUT) {
level = LOG_TRACE;
}
output = "-- empty --";
}
next = output;
do {
offset = next;
next = strchrnul(offset, '\n');
do_crm_log_alias(level, file, function, line, "%s [ %.*s ]", prefix,
(int)(next - offset), offset);
if (next[0] != 0) {
next++;
}
} while (next != NULL && next[0] != 0);
}
void
pcmk__cli_init_logging(const char *name, unsigned int verbosity)
{
crm_log_init(name, LOG_ERR, FALSE, FALSE, 0, NULL, TRUE);
for (int i = 0; i < verbosity; i++) {
/* These arguments are ignored, so pass placeholders. */
crm_bump_log_level(0, NULL);
}
}
/*!
* \brief Log XML line-by-line in a formatted fashion
*
* \param[in] file File name to use for log filtering
* \param[in] function Function name to use for log filtering
* \param[in] line Line number to use for log filtering
* \param[in] tags Logging tags to use for log filtering
* \param[in] level Priority at which to log the messages
* \param[in] text Prefix for each line
* \param[in] xml XML to log
*
* \note This does nothing when \p level is \p LOG_STDOUT.
* \note Do not call this function directly. It should be called only from the
* \p do_crm_log_xml() macro.
*/
void
pcmk_log_xml_as(const char *file, const char *function, uint32_t line,
uint32_t tags, uint8_t level, const char *text, const xmlNode *xml)
{
if (xml == NULL) {
do_crm_log(level, "%s%sNo data to dump as XML",
pcmk__s(text, ""), pcmk__str_empty(text)? "" : " ");
} else {
if (logger_out == NULL) {
CRM_CHECK(pcmk__log_output_new(&logger_out) == pcmk_rc_ok, return);
}
pcmk__output_set_log_level(logger_out, level);
pcmk__output_set_log_filter(logger_out, file, function, line, tags);
pcmk__xml_show(logger_out, text, xml, 1,
pcmk__xml_fmt_pretty
|pcmk__xml_fmt_open
|pcmk__xml_fmt_children
|pcmk__xml_fmt_close);
pcmk__output_set_log_filter(logger_out, NULL, NULL, 0U, 0U);
}
}
/*!
* \internal
* \brief Log XML changes line-by-line in a formatted fashion
*
* \param[in] file File name to use for log filtering
* \param[in] function Function name to use for log filtering
* \param[in] line Line number to use for log filtering
* \param[in] tags Logging tags to use for log filtering
* \param[in] level Priority at which to log the messages
* \param[in] xml XML whose changes to log
*
* \note This does nothing when \p level is \c LOG_STDOUT.
*/
void
pcmk__log_xml_changes_as(const char *file, const char *function, uint32_t line,
uint32_t tags, uint8_t level, const xmlNode *xml)
{
if (xml == NULL) {
do_crm_log(level, "No XML to dump");
return;
}
if (logger_out == NULL) {
CRM_CHECK(pcmk__log_output_new(&logger_out) == pcmk_rc_ok, return);
}
pcmk__output_set_log_level(logger_out, level);
pcmk__output_set_log_filter(logger_out, file, function, line, tags);
pcmk__xml_show_changes(logger_out, xml);
pcmk__output_set_log_filter(logger_out, NULL, NULL, 0U, 0U);
}
/*!
* \internal
* \brief Log an XML patchset line-by-line in a formatted fashion
*
* \param[in] file File name to use for log filtering
* \param[in] function Function name to use for log filtering
* \param[in] line Line number to use for log filtering
* \param[in] tags Logging tags to use for log filtering
* \param[in] level Priority at which to log the messages
* \param[in] patchset XML patchset to log
*
* \note This does nothing when \p level is \c LOG_STDOUT.
*/
void
pcmk__log_xml_patchset_as(const char *file, const char *function, uint32_t line,
uint32_t tags, uint8_t level, const xmlNode *patchset)
{
if (patchset == NULL) {
do_crm_log(level, "No patchset to dump");
return;
}
if (logger_out == NULL) {
CRM_CHECK(pcmk__log_output_new(&logger_out) == pcmk_rc_ok, return);
}
pcmk__output_set_log_level(logger_out, level);
pcmk__output_set_log_filter(logger_out, file, function, line, tags);
logger_out->message(logger_out, "xml-patchset", patchset);
pcmk__output_set_log_filter(logger_out, NULL, NULL, 0U, 0U);
}
/*!
* \internal
* \brief Free the logging library's internal log output object
*/
void
pcmk__free_common_logger(void)
{
if (logger_out != NULL) {
logger_out->finish(logger_out, CRM_EX_OK, true, NULL);
pcmk__output_free(logger_out);
logger_out = NULL;
}
}
// Deprecated functions kept only for backward API compatibility
// LCOV_EXCL_START
#include <crm/common/logging_compat.h>
gboolean
crm_log_cli_init(const char *entity)
{
pcmk__cli_init_logging(entity, 0);
return TRUE;
}
gboolean
crm_add_logfile(const char *filename)
{
return pcmk__add_logfile(filename) == pcmk_rc_ok;
}
void
pcmk_log_xml_impl(uint8_t level, const char *text, const xmlNode *xml)
{
pcmk_log_xml_as(__FILE__, __func__, __LINE__, 0, level, text, xml);
}
// LCOV_EXCL_STOP
// End deprecated API
diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c
index 6af794b08d..e06779b474 100644
--- a/lib/common/mainloop.c
+++ b/lib/common/mainloop.c
@@ -1,1484 +1,1484 @@
/*
* Copyright 2004-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#ifndef _GNU_SOURCE
# define _GNU_SOURCE
#endif
#include <stdlib.h>
#include <string.h>
#include <signal.h>
#include <errno.h>
#include <sys/wait.h>
#include <crm/crm.h>
#include <crm/common/xml.h>
#include <crm/common/mainloop.h>
#include <crm/common/ipc_internal.h>
#include <qb/qbarray.h>
struct mainloop_child_s {
pid_t pid;
char *desc;
unsigned timerid;
gboolean timeout;
void *privatedata;
enum mainloop_child_flags flags;
/* Called when a process dies */
void (*callback) (mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode);
};
struct trigger_s {
GSource source;
gboolean running;
gboolean trigger;
void *user_data;
guint id;
};
struct mainloop_timer_s {
guint id;
guint period_ms;
bool repeat;
char *name;
GSourceFunc cb;
void *userdata;
};
static gboolean
crm_trigger_prepare(GSource * source, gint * timeout)
{
crm_trigger_t *trig = (crm_trigger_t *) source;
/* cluster-glue's FD and IPC related sources make use of
* g_source_add_poll() but do not set a timeout in their prepare
* functions
*
* This means mainloop's poll() will block until an event for one
* of these sources occurs - any /other/ type of source, such as
* this one or g_idle_*, that doesn't use g_source_add_poll() is
* S-O-L and won't be processed until there is something fd-based
* happens.
*
* Luckily the timeout we can set here affects all sources and
* puts an upper limit on how long poll() can take.
*
* So unconditionally set a small-ish timeout, not too small that
* we're in constant motion, which will act as an upper bound on
* how long the signal handling might be delayed for.
*/
*timeout = 500; /* Timeout in ms */
return trig->trigger;
}
static gboolean
crm_trigger_check(GSource * source)
{
crm_trigger_t *trig = (crm_trigger_t *) source;
return trig->trigger;
}
/*!
* \internal
* \brief GSource dispatch function for crm_trigger_t
*
* \param[in] source crm_trigger_t being dispatched
* \param[in] callback Callback passed at source creation
* \param[in,out] userdata User data passed at source creation
*
* \return G_SOURCE_REMOVE to remove source, G_SOURCE_CONTINUE to keep it
*/
static gboolean
crm_trigger_dispatch(GSource *source, GSourceFunc callback, gpointer userdata)
{
gboolean rc = G_SOURCE_CONTINUE;
crm_trigger_t *trig = (crm_trigger_t *) source;
if (trig->running) {
/* Wait until the existing job is complete before starting the next one */
return G_SOURCE_CONTINUE;
}
trig->trigger = FALSE;
if (callback) {
int callback_rc = callback(trig->user_data);
if (callback_rc < 0) {
crm_trace("Trigger handler %p not yet complete", trig);
trig->running = TRUE;
} else if (callback_rc == 0) {
rc = G_SOURCE_REMOVE;
}
}
return rc;
}
static void
crm_trigger_finalize(GSource * source)
{
crm_trace("Trigger %p destroyed", source);
}
static GSourceFuncs crm_trigger_funcs = {
crm_trigger_prepare,
crm_trigger_check,
crm_trigger_dispatch,
crm_trigger_finalize,
};
static crm_trigger_t *
mainloop_setup_trigger(GSource * source, int priority, int (*dispatch) (gpointer user_data),
gpointer userdata)
{
crm_trigger_t *trigger = NULL;
trigger = (crm_trigger_t *) source;
trigger->id = 0;
trigger->trigger = FALSE;
trigger->user_data = userdata;
if (dispatch) {
g_source_set_callback(source, dispatch, trigger, NULL);
}
g_source_set_priority(source, priority);
g_source_set_can_recurse(source, FALSE);
trigger->id = g_source_attach(source, NULL);
return trigger;
}
void
mainloop_trigger_complete(crm_trigger_t * trig)
{
crm_trace("Trigger handler %p complete", trig);
trig->running = FALSE;
}
/*!
* \brief Create a trigger to be used as a mainloop source
*
* \param[in] priority Relative priority of source (lower number is higher priority)
* \param[in] dispatch Trigger dispatch function (should return 0 to remove the
* trigger from the mainloop, -1 if the trigger should be
* kept but the job is still running and not complete, and
* 1 if the trigger should be kept and the job is complete)
* \param[in] userdata Pointer to pass to \p dispatch
*
* \return Newly allocated mainloop source for trigger
*/
crm_trigger_t *
mainloop_add_trigger(int priority, int (*dispatch) (gpointer user_data),
gpointer userdata)
{
GSource *source = NULL;
CRM_ASSERT(sizeof(crm_trigger_t) > sizeof(GSource));
source = g_source_new(&crm_trigger_funcs, sizeof(crm_trigger_t));
CRM_ASSERT(source != NULL);
return mainloop_setup_trigger(source, priority, dispatch, userdata);
}
void
mainloop_set_trigger(crm_trigger_t * source)
{
if(source) {
source->trigger = TRUE;
}
}
gboolean
mainloop_destroy_trigger(crm_trigger_t * source)
{
GSource *gs = NULL;
if(source == NULL) {
return TRUE;
}
gs = (GSource *)source;
g_source_destroy(gs); /* Remove from mainloop, ref_count-- */
g_source_unref(gs); /* The caller no longer carries a reference to source
*
* At this point the source should be free'd,
* unless we're currently processing said
* source, in which case mainloop holds an
* additional reference and it will be free'd
* once our processing completes
*/
return TRUE;
}
// Define a custom glib source for signal handling
// Data structure for custom glib source
typedef struct signal_s {
crm_trigger_t trigger; // trigger that invoked source (must be first)
void (*handler) (int sig); // signal handler
int signal; // signal that was received
} crm_signal_t;
// Table to associate signal handlers with signal numbers
static crm_signal_t *crm_signals[NSIG];
/*!
* \internal
* \brief Dispatch an event from custom glib source for signals
*
* Given an signal event, clear the event trigger and call any registered
* signal handler.
*
* \param[in] source glib source that triggered this dispatch
* \param[in] callback (ignored)
* \param[in] userdata (ignored)
*/
static gboolean
crm_signal_dispatch(GSource *source, GSourceFunc callback, gpointer userdata)
{
crm_signal_t *sig = (crm_signal_t *) source;
if(sig->signal != SIGCHLD) {
crm_notice("Caught '%s' signal "CRM_XS" %d (%s handler)",
strsignal(sig->signal), sig->signal,
(sig->handler? "invoking" : "no"));
}
sig->trigger.trigger = FALSE;
if (sig->handler) {
sig->handler(sig->signal);
}
return TRUE;
}
/*!
* \internal
* \brief Handle a signal by setting a trigger for signal source
*
* \param[in] sig Signal number that was received
*
* \note This is the true signal handler for the mainloop signal source, and
* must be async-safe.
*/
static void
mainloop_signal_handler(int sig)
{
if (sig > 0 && sig < NSIG && crm_signals[sig] != NULL) {
mainloop_set_trigger((crm_trigger_t *) crm_signals[sig]);
}
}
// Functions implementing our custom glib source for signal handling
static GSourceFuncs crm_signal_funcs = {
crm_trigger_prepare,
crm_trigger_check,
crm_signal_dispatch,
crm_trigger_finalize,
};
/*!
* \internal
* \brief Set a true signal handler
*
* signal()-like interface to sigaction()
*
* \param[in] sig Signal number to register handler for
* \param[in] dispatch Signal handler
*
* \return The previous value of the signal handler, or SIG_ERR on error
* \note The dispatch function must be async-safe.
*/
sighandler_t
crm_signal_handler(int sig, sighandler_t dispatch)
{
sigset_t mask;
struct sigaction sa;
struct sigaction old;
if (sigemptyset(&mask) < 0) {
crm_err("Could not set handler for signal %d: %s",
sig, pcmk_rc_str(errno));
return SIG_ERR;
}
memset(&sa, 0, sizeof(struct sigaction));
sa.sa_handler = dispatch;
sa.sa_flags = SA_RESTART;
sa.sa_mask = mask;
if (sigaction(sig, &sa, &old) < 0) {
crm_err("Could not set handler for signal %d: %s",
sig, pcmk_rc_str(errno));
return SIG_ERR;
}
return old.sa_handler;
}
static void
mainloop_destroy_signal_entry(int sig)
{
crm_signal_t *tmp = crm_signals[sig];
crm_signals[sig] = NULL;
crm_trace("Destroying signal %d", sig);
mainloop_destroy_trigger((crm_trigger_t *) tmp);
}
/*!
* \internal
* \brief Add a signal handler to a mainloop
*
* \param[in] sig Signal number to handle
* \param[in] dispatch Signal handler function
*
* \note The true signal handler merely sets a mainloop trigger to call this
* dispatch function via the mainloop. Therefore, the dispatch function
* does not need to be async-safe.
*/
gboolean
mainloop_add_signal(int sig, void (*dispatch) (int sig))
{
GSource *source = NULL;
int priority = G_PRIORITY_HIGH - 1;
if (sig == SIGTERM) {
/* TERM is higher priority than other signals,
* signals are higher priority than other ipc.
* Yes, minus: smaller is "higher"
*/
priority--;
}
if (sig >= NSIG || sig < 0) {
crm_err("Signal %d is out of range", sig);
return FALSE;
} else if (crm_signals[sig] != NULL && crm_signals[sig]->handler == dispatch) {
crm_trace("Signal handler for %d is already installed", sig);
return TRUE;
} else if (crm_signals[sig] != NULL) {
crm_err("Different signal handler for %d is already installed", sig);
return FALSE;
}
CRM_ASSERT(sizeof(crm_signal_t) > sizeof(GSource));
source = g_source_new(&crm_signal_funcs, sizeof(crm_signal_t));
crm_signals[sig] = (crm_signal_t *) mainloop_setup_trigger(source, priority, NULL, NULL);
CRM_ASSERT(crm_signals[sig] != NULL);
crm_signals[sig]->handler = dispatch;
crm_signals[sig]->signal = sig;
if (crm_signal_handler(sig, mainloop_signal_handler) == SIG_ERR) {
mainloop_destroy_signal_entry(sig);
return FALSE;
}
return TRUE;
}
gboolean
mainloop_destroy_signal(int sig)
{
if (sig >= NSIG || sig < 0) {
crm_err("Signal %d is out of range", sig);
return FALSE;
} else if (crm_signal_handler(sig, NULL) == SIG_ERR) {
crm_perror(LOG_ERR, "Could not uninstall signal handler for signal %d", sig);
return FALSE;
} else if (crm_signals[sig] == NULL) {
return TRUE;
}
mainloop_destroy_signal_entry(sig);
return TRUE;
}
static qb_array_t *gio_map = NULL;
void
mainloop_cleanup(void)
{
if (gio_map) {
qb_array_free(gio_map);
}
for (int sig = 0; sig < NSIG; ++sig) {
mainloop_destroy_signal_entry(sig);
}
}
/*
* libqb...
*/
struct gio_to_qb_poll {
int32_t is_used;
guint source;
int32_t events;
void *data;
qb_ipcs_dispatch_fn_t fn;
enum qb_loop_priority p;
};
static gboolean
gio_read_socket(GIOChannel * gio, GIOCondition condition, gpointer data)
{
struct gio_to_qb_poll *adaptor = (struct gio_to_qb_poll *)data;
gint fd = g_io_channel_unix_get_fd(gio);
crm_trace("%p.%d %d", data, fd, condition);
/* if this assert get's hit, then there is a race condition between
* when we destroy a fd and when mainloop actually gives it up */
CRM_ASSERT(adaptor->is_used > 0);
return (adaptor->fn(fd, condition, adaptor->data) == 0);
}
static void
gio_poll_destroy(gpointer data)
{
struct gio_to_qb_poll *adaptor = (struct gio_to_qb_poll *)data;
adaptor->is_used--;
CRM_ASSERT(adaptor->is_used >= 0);
if (adaptor->is_used == 0) {
crm_trace("Marking adaptor %p unused", adaptor);
adaptor->source = 0;
}
}
/*!
* \internal
* \brief Convert libqb's poll priority into GLib's one
*
* \param[in] prio libqb's poll priority (#QB_LOOP_MED assumed as fallback)
*
* \return best matching GLib's priority
*/
static gint
conv_prio_libqb2glib(enum qb_loop_priority prio)
{
switch (prio) {
case QB_LOOP_LOW: return G_PRIORITY_LOW;
case QB_LOOP_HIGH: return G_PRIORITY_HIGH;
default: return G_PRIORITY_DEFAULT; // QB_LOOP_MED
}
}
/*!
* \internal
* \brief Convert libqb's poll priority to rate limiting spec
*
* \param[in] prio libqb's poll priority (#QB_LOOP_MED assumed as fallback)
*
* \return best matching rate limiting spec
* \note This is the inverse of libqb's qb_ipcs_request_rate_limit().
*/
static enum qb_ipcs_rate_limit
conv_libqb_prio2ratelimit(enum qb_loop_priority prio)
{
switch (prio) {
case QB_LOOP_LOW: return QB_IPCS_RATE_SLOW;
case QB_LOOP_HIGH: return QB_IPCS_RATE_FAST;
default: return QB_IPCS_RATE_NORMAL; // QB_LOOP_MED
}
}
static int32_t
gio_poll_dispatch_update(enum qb_loop_priority p, int32_t fd, int32_t evts,
void *data, qb_ipcs_dispatch_fn_t fn, int32_t add)
{
struct gio_to_qb_poll *adaptor;
GIOChannel *channel;
int32_t res = 0;
res = qb_array_index(gio_map, fd, (void **)&adaptor);
if (res < 0) {
crm_err("Array lookup failed for fd=%d: %d", fd, res);
return res;
}
crm_trace("Adding fd=%d to mainloop as adaptor %p", fd, adaptor);
if (add && adaptor->source) {
crm_err("Adaptor for descriptor %d is still in-use", fd);
return -EEXIST;
}
if (!add && !adaptor->is_used) {
crm_err("Adaptor for descriptor %d is not in-use", fd);
return -ENOENT;
}
/* channel is created with ref_count = 1 */
channel = g_io_channel_unix_new(fd);
if (!channel) {
crm_err("No memory left to add fd=%d", fd);
return -ENOMEM;
}
if (adaptor->source) {
g_source_remove(adaptor->source);
adaptor->source = 0;
}
/* Because unlike the poll() API, glib doesn't tell us about HUPs by default */
evts |= (G_IO_HUP | G_IO_NVAL | G_IO_ERR);
adaptor->fn = fn;
adaptor->events = evts;
adaptor->data = data;
adaptor->p = p;
adaptor->is_used++;
adaptor->source =
g_io_add_watch_full(channel, conv_prio_libqb2glib(p), evts,
gio_read_socket, adaptor, gio_poll_destroy);
/* Now that mainloop now holds a reference to channel,
* thanks to g_io_add_watch_full(), drop ours from g_io_channel_unix_new().
*
* This means that channel will be free'd by:
* g_main_context_dispatch()
* -> g_source_destroy_internal()
* -> g_source_callback_unref()
* shortly after gio_poll_destroy() completes
*/
g_io_channel_unref(channel);
crm_trace("Added to mainloop with gsource id=%d", adaptor->source);
if (adaptor->source > 0) {
return 0;
}
return -EINVAL;
}
static int32_t
gio_poll_dispatch_add(enum qb_loop_priority p, int32_t fd, int32_t evts,
void *data, qb_ipcs_dispatch_fn_t fn)
{
return gio_poll_dispatch_update(p, fd, evts, data, fn, QB_TRUE);
}
static int32_t
gio_poll_dispatch_mod(enum qb_loop_priority p, int32_t fd, int32_t evts,
void *data, qb_ipcs_dispatch_fn_t fn)
{
return gio_poll_dispatch_update(p, fd, evts, data, fn, QB_FALSE);
}
static int32_t
gio_poll_dispatch_del(int32_t fd)
{
struct gio_to_qb_poll *adaptor;
crm_trace("Looking for fd=%d", fd);
if (qb_array_index(gio_map, fd, (void **)&adaptor) == 0) {
if (adaptor->source) {
g_source_remove(adaptor->source);
adaptor->source = 0;
}
}
return 0;
}
struct qb_ipcs_poll_handlers gio_poll_funcs = {
.job_add = NULL,
.dispatch_add = gio_poll_dispatch_add,
.dispatch_mod = gio_poll_dispatch_mod,
.dispatch_del = gio_poll_dispatch_del,
};
static enum qb_ipc_type
pick_ipc_type(enum qb_ipc_type requested)
{
- const char *env = getenv("PCMK_ipc_type");
+ const char *env = pcmk__env_option(PCMK__ENV_IPC_TYPE);
if (env && strcmp("shared-mem", env) == 0) {
return QB_IPC_SHM;
} else if (env && strcmp("socket", env) == 0) {
return QB_IPC_SOCKET;
} else if (env && strcmp("posix", env) == 0) {
return QB_IPC_POSIX_MQ;
} else if (env && strcmp("sysv", env) == 0) {
return QB_IPC_SYSV_MQ;
} else if (requested == QB_IPC_NATIVE) {
/* We prefer shared memory because the server never blocks on
* send. If part of a message fits into the socket, libqb
* needs to block until the remainder can be sent also.
* Otherwise the client will wait forever for the remaining
* bytes.
*/
return QB_IPC_SHM;
}
return requested;
}
qb_ipcs_service_t *
mainloop_add_ipc_server(const char *name, enum qb_ipc_type type,
struct qb_ipcs_service_handlers *callbacks)
{
return mainloop_add_ipc_server_with_prio(name, type, callbacks, QB_LOOP_MED);
}
qb_ipcs_service_t *
mainloop_add_ipc_server_with_prio(const char *name, enum qb_ipc_type type,
struct qb_ipcs_service_handlers *callbacks,
enum qb_loop_priority prio)
{
int rc = 0;
qb_ipcs_service_t *server = NULL;
if (gio_map == NULL) {
gio_map = qb_array_create_2(64, sizeof(struct gio_to_qb_poll), 1);
}
server = qb_ipcs_create(name, 0, pick_ipc_type(type), callbacks);
if (server == NULL) {
rc = errno;
crm_err("Could not create %s IPC server: %s (%d)", name, pcmk_rc_str(errno), errno);
return NULL;
}
if (prio != QB_LOOP_MED) {
qb_ipcs_request_rate_limit(server, conv_libqb_prio2ratelimit(prio));
}
/* All clients should use at least ipc_buffer_max as their buffer size */
qb_ipcs_enforce_buffer_size(server, crm_ipc_default_buffer_size());
qb_ipcs_poll_handlers_set(server, &gio_poll_funcs);
rc = qb_ipcs_run(server);
if (rc < 0) {
crm_err("Could not start %s IPC server: %s (%d)", name, pcmk_strerror(rc), rc);
return NULL; // qb_ipcs_run() destroys server on failure
}
return server;
}
void
mainloop_del_ipc_server(qb_ipcs_service_t * server)
{
if (server) {
qb_ipcs_destroy(server);
}
}
struct mainloop_io_s {
char *name;
void *userdata;
int fd;
guint source;
crm_ipc_t *ipc;
GIOChannel *channel;
int (*dispatch_fn_ipc) (const char *buffer, ssize_t length, gpointer userdata);
int (*dispatch_fn_io) (gpointer userdata);
void (*destroy_fn) (gpointer userdata);
};
/*!
* \internal
* \brief I/O watch callback function (GIOFunc)
*
* \param[in] gio I/O channel being watched
* \param[in] condition I/O condition satisfied
* \param[in] data User data passed when source was created
*
* \return G_SOURCE_REMOVE to remove source, G_SOURCE_CONTINUE to keep it
*/
static gboolean
mainloop_gio_callback(GIOChannel *gio, GIOCondition condition, gpointer data)
{
gboolean rc = G_SOURCE_CONTINUE;
mainloop_io_t *client = data;
CRM_ASSERT(client->fd == g_io_channel_unix_get_fd(gio));
if (condition & G_IO_IN) {
if (client->ipc) {
long read_rc = 0L;
int max = 10;
do {
read_rc = crm_ipc_read(client->ipc);
if (read_rc <= 0) {
crm_trace("Could not read IPC message from %s: %s (%ld)",
client->name, pcmk_strerror(read_rc), read_rc);
} else if (client->dispatch_fn_ipc) {
const char *buffer = crm_ipc_buffer(client->ipc);
crm_trace("New %ld-byte IPC message from %s "
"after I/O condition %d",
read_rc, client->name, (int) condition);
if (client->dispatch_fn_ipc(buffer, read_rc, client->userdata) < 0) {
crm_trace("Connection to %s no longer required", client->name);
rc = G_SOURCE_REMOVE;
}
}
} while ((rc == G_SOURCE_CONTINUE) && (read_rc > 0) && --max > 0);
} else {
crm_trace("New I/O event for %s after I/O condition %d",
client->name, (int) condition);
if (client->dispatch_fn_io) {
if (client->dispatch_fn_io(client->userdata) < 0) {
crm_trace("Connection to %s no longer required", client->name);
rc = G_SOURCE_REMOVE;
}
}
}
}
if (client->ipc && !crm_ipc_connected(client->ipc)) {
crm_err("Connection to %s closed " CRM_XS "client=%p condition=%d",
client->name, client, condition);
rc = G_SOURCE_REMOVE;
} else if (condition & (G_IO_HUP | G_IO_NVAL | G_IO_ERR)) {
crm_trace("The connection %s[%p] has been closed (I/O condition=%d)",
client->name, client, condition);
rc = G_SOURCE_REMOVE;
} else if ((condition & G_IO_IN) == 0) {
/*
#define GLIB_SYSDEF_POLLIN =1
#define GLIB_SYSDEF_POLLPRI =2
#define GLIB_SYSDEF_POLLOUT =4
#define GLIB_SYSDEF_POLLERR =8
#define GLIB_SYSDEF_POLLHUP =16
#define GLIB_SYSDEF_POLLNVAL =32
typedef enum
{
G_IO_IN GLIB_SYSDEF_POLLIN,
G_IO_OUT GLIB_SYSDEF_POLLOUT,
G_IO_PRI GLIB_SYSDEF_POLLPRI,
G_IO_ERR GLIB_SYSDEF_POLLERR,
G_IO_HUP GLIB_SYSDEF_POLLHUP,
G_IO_NVAL GLIB_SYSDEF_POLLNVAL
} GIOCondition;
A bitwise combination representing a condition to watch for on an event source.
G_IO_IN There is data to read.
G_IO_OUT Data can be written (without blocking).
G_IO_PRI There is urgent data to read.
G_IO_ERR Error condition.
G_IO_HUP Hung up (the connection has been broken, usually for pipes and sockets).
G_IO_NVAL Invalid request. The file descriptor is not open.
*/
crm_err("Strange condition: %d", condition);
}
/* G_SOURCE_REMOVE results in mainloop_gio_destroy() being called
* just before the source is removed from mainloop
*/
return rc;
}
static void
mainloop_gio_destroy(gpointer c)
{
mainloop_io_t *client = c;
char *c_name = strdup(client->name);
/* client->source is valid but about to be destroyed (ref_count == 0) in gmain.c
* client->channel will still have ref_count > 0... should be == 1
*/
crm_trace("Destroying client %s[%p]", c_name, c);
if (client->ipc) {
crm_ipc_close(client->ipc);
}
if (client->destroy_fn) {
void (*destroy_fn) (gpointer userdata) = client->destroy_fn;
client->destroy_fn = NULL;
destroy_fn(client->userdata);
}
if (client->ipc) {
crm_ipc_t *ipc = client->ipc;
client->ipc = NULL;
crm_ipc_destroy(ipc);
}
crm_trace("Destroyed client %s[%p]", c_name, c);
free(client->name); client->name = NULL;
free(client);
free(c_name);
}
/*!
* \brief Connect to IPC and add it as a main loop source
*
* \param[in,out] ipc IPC connection to add
* \param[in] priority Event source priority to use for connection
* \param[in] userdata Data to register with callbacks
* \param[in] callbacks Dispatch and destroy callbacks for connection
* \param[out] source Newly allocated event source
*
* \return Standard Pacemaker return code
*
* \note On failure, the caller is still responsible for ipc. On success, the
* caller should call mainloop_del_ipc_client() when source is no longer
* needed, which will lead to the disconnection of the IPC later in the
* main loop if it is connected. However the IPC disconnects,
* mainloop_gio_destroy() will free ipc and source after calling the
* destroy callback.
*/
int
pcmk__add_mainloop_ipc(crm_ipc_t *ipc, int priority, void *userdata,
const struct ipc_client_callbacks *callbacks,
mainloop_io_t **source)
{
int rc = pcmk_rc_ok;
int fd = -1;
const char *ipc_name = NULL;
CRM_CHECK((ipc != NULL) && (callbacks != NULL), return EINVAL);
ipc_name = pcmk__s(crm_ipc_name(ipc), "Pacemaker");
rc = pcmk__connect_generic_ipc(ipc);
if (rc != pcmk_rc_ok) {
crm_debug("Connection to %s failed: %s", ipc_name, pcmk_rc_str(rc));
return rc;
}
rc = pcmk__ipc_fd(ipc, &fd);
if (rc != pcmk_rc_ok) {
crm_debug("Could not obtain file descriptor for %s IPC: %s",
ipc_name, pcmk_rc_str(rc));
crm_ipc_close(ipc);
return rc;
}
*source = mainloop_add_fd(ipc_name, priority, fd, userdata, NULL);
if (*source == NULL) {
rc = errno;
crm_ipc_close(ipc);
return rc;
}
(*source)->ipc = ipc;
(*source)->destroy_fn = callbacks->destroy;
(*source)->dispatch_fn_ipc = callbacks->dispatch;
return pcmk_rc_ok;
}
/*!
* \brief Get period for mainloop timer
*
* \param[in] timer Timer
*
* \return Period in ms
*/
guint
pcmk__mainloop_timer_get_period(const mainloop_timer_t *timer)
{
if (timer) {
return timer->period_ms;
}
return 0;
}
mainloop_io_t *
mainloop_add_ipc_client(const char *name, int priority, size_t max_size,
void *userdata, struct ipc_client_callbacks *callbacks)
{
crm_ipc_t *ipc = crm_ipc_new(name, max_size);
mainloop_io_t *source = NULL;
int rc = pcmk__add_mainloop_ipc(ipc, priority, userdata, callbacks,
&source);
if (rc != pcmk_rc_ok) {
if (crm_log_level == LOG_STDOUT) {
fprintf(stderr, "Connection to %s failed: %s",
name, pcmk_rc_str(rc));
}
crm_ipc_destroy(ipc);
if (rc > 0) {
errno = rc;
} else {
errno = ENOTCONN;
}
return NULL;
}
return source;
}
void
mainloop_del_ipc_client(mainloop_io_t * client)
{
mainloop_del_fd(client);
}
crm_ipc_t *
mainloop_get_ipc_client(mainloop_io_t * client)
{
if (client) {
return client->ipc;
}
return NULL;
}
mainloop_io_t *
mainloop_add_fd(const char *name, int priority, int fd, void *userdata,
struct mainloop_fd_callbacks * callbacks)
{
mainloop_io_t *client = NULL;
if (fd >= 0) {
client = calloc(1, sizeof(mainloop_io_t));
if (client == NULL) {
return NULL;
}
client->name = strdup(name);
client->userdata = userdata;
if (callbacks) {
client->destroy_fn = callbacks->destroy;
client->dispatch_fn_io = callbacks->dispatch;
}
client->fd = fd;
client->channel = g_io_channel_unix_new(fd);
client->source =
g_io_add_watch_full(client->channel, priority,
(G_IO_IN | G_IO_HUP | G_IO_NVAL | G_IO_ERR), mainloop_gio_callback,
client, mainloop_gio_destroy);
/* Now that mainloop now holds a reference to channel,
* thanks to g_io_add_watch_full(), drop ours from g_io_channel_unix_new().
*
* This means that channel will be free'd by:
* g_main_context_dispatch() or g_source_remove()
* -> g_source_destroy_internal()
* -> g_source_callback_unref()
* shortly after mainloop_gio_destroy() completes
*/
g_io_channel_unref(client->channel);
crm_trace("Added connection %d for %s[%p].%d", client->source, client->name, client, fd);
} else {
errno = EINVAL;
}
return client;
}
void
mainloop_del_fd(mainloop_io_t * client)
{
if (client != NULL) {
crm_trace("Removing client %s[%p]", client->name, client);
if (client->source) {
/* Results in mainloop_gio_destroy() being called just
* before the source is removed from mainloop
*/
g_source_remove(client->source);
}
}
}
static GList *child_list = NULL;
pid_t
mainloop_child_pid(mainloop_child_t * child)
{
return child->pid;
}
const char *
mainloop_child_name(mainloop_child_t * child)
{
return child->desc;
}
int
mainloop_child_timeout(mainloop_child_t * child)
{
return child->timeout;
}
void *
mainloop_child_userdata(mainloop_child_t * child)
{
return child->privatedata;
}
void
mainloop_clear_child_userdata(mainloop_child_t * child)
{
child->privatedata = NULL;
}
/* good function name */
static void
child_free(mainloop_child_t *child)
{
if (child->timerid != 0) {
crm_trace("Removing timer %d", child->timerid);
g_source_remove(child->timerid);
child->timerid = 0;
}
free(child->desc);
free(child);
}
/* terrible function name */
static int
child_kill_helper(mainloop_child_t *child)
{
int rc;
if (child->flags & mainloop_leave_pid_group) {
crm_debug("Kill pid %d only. leave group intact.", child->pid);
rc = kill(child->pid, SIGKILL);
} else {
crm_debug("Kill pid %d's group", child->pid);
rc = kill(-child->pid, SIGKILL);
}
if (rc < 0) {
if (errno != ESRCH) {
crm_perror(LOG_ERR, "kill(%d, KILL) failed", child->pid);
}
return -errno;
}
return 0;
}
static gboolean
child_timeout_callback(gpointer p)
{
mainloop_child_t *child = p;
int rc = 0;
child->timerid = 0;
if (child->timeout) {
crm_warn("%s process (PID %d) will not die!", child->desc, (int)child->pid);
return FALSE;
}
rc = child_kill_helper(child);
if (rc == -ESRCH) {
/* Nothing left to do. pid doesn't exist */
return FALSE;
}
child->timeout = TRUE;
crm_debug("%s process (PID %d) timed out", child->desc, (int)child->pid);
child->timerid = g_timeout_add(5000, child_timeout_callback, child);
return FALSE;
}
static bool
child_waitpid(mainloop_child_t *child, int flags)
{
int rc = 0;
int core = 0;
int signo = 0;
int status = 0;
int exitcode = 0;
bool callback_needed = true;
rc = waitpid(child->pid, &status, flags);
if (rc == 0) { // WNOHANG in flags, and child status is not available
crm_trace("Child process %d (%s) still active",
child->pid, child->desc);
callback_needed = false;
} else if (rc != child->pid) {
/* According to POSIX, possible conditions:
* - child->pid was non-positive (process group or any child),
* and rc is specific child
* - errno ECHILD (pid does not exist or is not child)
* - errno EINVAL (invalid flags)
* - errno EINTR (caller interrupted by signal)
*
* @TODO Handle these cases more specifically.
*/
signo = SIGCHLD;
exitcode = 1;
crm_notice("Wait for child process %d (%s) interrupted: %s",
child->pid, child->desc, pcmk_rc_str(errno));
} else if (WIFEXITED(status)) {
exitcode = WEXITSTATUS(status);
crm_trace("Child process %d (%s) exited with status %d",
child->pid, child->desc, exitcode);
} else if (WIFSIGNALED(status)) {
signo = WTERMSIG(status);
crm_trace("Child process %d (%s) exited with signal %d (%s)",
child->pid, child->desc, signo, strsignal(signo));
#ifdef WCOREDUMP // AIX, SunOS, maybe others
} else if (WCOREDUMP(status)) {
core = 1;
crm_err("Child process %d (%s) dumped core",
child->pid, child->desc);
#endif
} else { // flags must contain WUNTRACED and/or WCONTINUED to reach this
crm_trace("Child process %d (%s) stopped or continued",
child->pid, child->desc);
callback_needed = false;
}
if (callback_needed && child->callback) {
child->callback(child, child->pid, core, signo, exitcode);
}
return callback_needed;
}
static void
child_death_dispatch(int signal)
{
for (GList *iter = child_list; iter; ) {
GList *saved = iter;
mainloop_child_t *child = iter->data;
iter = iter->next;
if (child_waitpid(child, WNOHANG)) {
crm_trace("Removing completed process %d from child list",
child->pid);
child_list = g_list_remove_link(child_list, saved);
g_list_free(saved);
child_free(child);
}
}
}
static gboolean
child_signal_init(gpointer p)
{
crm_trace("Installed SIGCHLD handler");
/* Do NOT use g_child_watch_add() and friends, they rely on pthreads */
mainloop_add_signal(SIGCHLD, child_death_dispatch);
/* In case they terminated before the signal handler was installed */
child_death_dispatch(SIGCHLD);
return FALSE;
}
gboolean
mainloop_child_kill(pid_t pid)
{
GList *iter;
mainloop_child_t *child = NULL;
mainloop_child_t *match = NULL;
/* It is impossible to block SIGKILL, this allows us to
* call waitpid without WNOHANG flag.*/
int waitflags = 0, rc = 0;
for (iter = child_list; iter != NULL && match == NULL; iter = iter->next) {
child = iter->data;
if (pid == child->pid) {
match = child;
}
}
if (match == NULL) {
return FALSE;
}
rc = child_kill_helper(match);
if(rc == -ESRCH) {
/* It's gone, but hasn't shown up in waitpid() yet. Wait until we get
* SIGCHLD and let handler clean it up as normal (so we get the correct
* return code/status). The blocking alternative would be to call
* child_waitpid(match, 0).
*/
crm_trace("Waiting for signal that child process %d completed",
match->pid);
return TRUE;
} else if(rc != 0) {
/* If KILL for some other reason set the WNOHANG flag since we
* can't be certain what happened.
*/
waitflags = WNOHANG;
}
if (!child_waitpid(match, waitflags)) {
/* not much we can do if this occurs */
return FALSE;
}
child_list = g_list_remove(child_list, match);
child_free(match);
return TRUE;
}
/* Create/Log a new tracked process
* To track a process group, use -pid
*
* @TODO Using a non-positive pid (i.e. any child, or process group) would
* likely not be useful since we will free the child after the first
* completed process.
*/
void
mainloop_child_add_with_flags(pid_t pid, int timeout, const char *desc, void *privatedata, enum mainloop_child_flags flags,
void (*callback) (mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode))
{
static bool need_init = TRUE;
mainloop_child_t *child = calloc(1, sizeof(mainloop_child_t));
child->pid = pid;
child->timerid = 0;
child->timeout = FALSE;
child->privatedata = privatedata;
child->callback = callback;
child->flags = flags;
pcmk__str_update(&child->desc, desc);
if (timeout) {
child->timerid = g_timeout_add(timeout, child_timeout_callback, child);
}
child_list = g_list_append(child_list, child);
if(need_init) {
need_init = FALSE;
/* SIGCHLD processing has to be invoked from mainloop.
* We do not want it to be possible to both add a child pid
* to mainloop, and have the pid's exit callback invoked within
* the same callstack. */
g_timeout_add(1, child_signal_init, NULL);
}
}
void
mainloop_child_add(pid_t pid, int timeout, const char *desc, void *privatedata,
void (*callback) (mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode))
{
mainloop_child_add_with_flags(pid, timeout, desc, privatedata, 0, callback);
}
static gboolean
mainloop_timer_cb(gpointer user_data)
{
int id = 0;
bool repeat = FALSE;
struct mainloop_timer_s *t = user_data;
CRM_ASSERT(t != NULL);
id = t->id;
t->id = 0; /* Ensure it's unset during callbacks so that
* mainloop_timer_running() works as expected
*/
if(t->cb) {
crm_trace("Invoking callbacks for timer %s", t->name);
repeat = t->repeat;
if(t->cb(t->userdata) == FALSE) {
crm_trace("Timer %s complete", t->name);
repeat = FALSE;
}
}
if(repeat) {
/* Restore if repeating */
t->id = id;
}
return repeat;
}
bool
mainloop_timer_running(mainloop_timer_t *t)
{
if(t && t->id != 0) {
return TRUE;
}
return FALSE;
}
void
mainloop_timer_start(mainloop_timer_t *t)
{
mainloop_timer_stop(t);
if(t && t->period_ms > 0) {
crm_trace("Starting timer %s", t->name);
t->id = g_timeout_add(t->period_ms, mainloop_timer_cb, t);
}
}
void
mainloop_timer_stop(mainloop_timer_t *t)
{
if(t && t->id != 0) {
crm_trace("Stopping timer %s", t->name);
g_source_remove(t->id);
t->id = 0;
}
}
guint
mainloop_timer_set_period(mainloop_timer_t *t, guint period_ms)
{
guint last = 0;
if(t) {
last = t->period_ms;
t->period_ms = period_ms;
}
if(t && t->id != 0 && last != t->period_ms) {
mainloop_timer_start(t);
}
return last;
}
mainloop_timer_t *
mainloop_timer_add(const char *name, guint period_ms, bool repeat, GSourceFunc cb, void *userdata)
{
mainloop_timer_t *t = calloc(1, sizeof(mainloop_timer_t));
if(t) {
if(name) {
t->name = crm_strdup_printf("%s-%u-%d", name, period_ms, repeat);
} else {
t->name = crm_strdup_printf("%p-%u-%d", t, period_ms, repeat);
}
t->id = 0;
t->period_ms = period_ms;
t->repeat = repeat;
t->cb = cb;
t->userdata = userdata;
crm_trace("Created timer %s with %p %p", t->name, userdata, t->userdata);
}
return t;
}
void
mainloop_timer_del(mainloop_timer_t *t)
{
if(t) {
crm_trace("Destroying timer %s", t->name);
mainloop_timer_stop(t);
free(t->name);
free(t);
}
}
/*
* Helpers to make sure certain events aren't lost at shutdown
*/
static gboolean
drain_timeout_cb(gpointer user_data)
{
bool *timeout_popped = (bool*) user_data;
*timeout_popped = TRUE;
return FALSE;
}
/*!
* \brief Drain some remaining main loop events then quit it
*
* \param[in,out] mloop Main loop to drain and quit
* \param[in] n Drain up to this many pending events
*/
void
pcmk_quit_main_loop(GMainLoop *mloop, unsigned int n)
{
if ((mloop != NULL) && g_main_loop_is_running(mloop)) {
GMainContext *ctx = g_main_loop_get_context(mloop);
/* Drain up to n events in case some memory clean-up is pending
* (helpful to reduce noise in valgrind output).
*/
for (int i = 0; (i < n) && g_main_context_pending(ctx); ++i) {
g_main_context_dispatch(ctx);
}
g_main_loop_quit(mloop);
}
}
/*!
* \brief Process main loop events while a certain condition is met
*
* \param[in,out] mloop Main loop to process
* \param[in] timer_ms Don't process longer than this amount of time
* \param[in] check Function that returns true if events should be
* processed
*
* \note This function is intended to be called at shutdown if certain important
* events should not be missed. The caller would likely quit the main loop
* or exit after calling this function. The check() function will be
* passed the remaining timeout in milliseconds.
*/
void
pcmk_drain_main_loop(GMainLoop *mloop, guint timer_ms, bool (*check)(guint))
{
bool timeout_popped = FALSE;
guint timer = 0;
GMainContext *ctx = NULL;
CRM_CHECK(mloop && check, return);
ctx = g_main_loop_get_context(mloop);
if (ctx) {
time_t start_time = time(NULL);
timer = g_timeout_add(timer_ms, drain_timeout_cb, &timeout_popped);
while (!timeout_popped
&& check(timer_ms - (time(NULL) - start_time) * 1000)) {
g_main_context_iteration(ctx, TRUE);
}
}
if (!timeout_popped && (timer > 0)) {
g_source_remove(timer);
}
}
// Deprecated functions kept only for backward API compatibility
// LCOV_EXCL_START
#include <crm/common/mainloop_compat.h>
gboolean
crm_signal(int sig, void (*dispatch) (int sig))
{
return crm_signal_handler(sig, dispatch) != SIG_ERR;
}
// LCOV_EXCL_STOP
// End deprecated API
diff --git a/lib/common/options.c b/lib/common/options.c
index cb32b3f25d..2d86ebc760 100644
--- a/lib/common/options.c
+++ b/lib/common/options.c
@@ -1,497 +1,510 @@
/*
* Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef _GNU_SOURCE
# define _GNU_SOURCE
#endif
#include <crm_internal.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <crm/crm.h>
void
pcmk__cli_help(char cmd)
{
if (cmd == 'v' || cmd == '$') {
printf("Pacemaker %s\n", PACEMAKER_VERSION);
printf("Written by Andrew Beekhof and "
"the Pacemaker project contributors\n");
} else if (cmd == '!') {
printf("Pacemaker %s (Build: %s): %s\n", PACEMAKER_VERSION, BUILD_VERSION, CRM_FEATURES);
}
crm_exit(CRM_EX_OK);
while(1); // above does not return
}
/*
* Environment variable option handling
*/
/*!
* \internal
* \brief Get the value of a Pacemaker environment variable option
*
* If an environment variable option is set, with either a PCMK_ or (for
* backward compatibility) HA_ prefix, log and return the value.
*
* \param[in] option Environment variable name (without prefix)
*
* \return Value of environment variable option, or NULL in case of
* option name too long or value not found
*/
const char *
pcmk__env_option(const char *option)
{
const char *const prefixes[] = {"PCMK_", "HA_"};
char env_name[NAME_MAX];
const char *value = NULL;
CRM_CHECK(!pcmk__str_empty(option), return NULL);
for (int i = 0; i < PCMK__NELEM(prefixes); i++) {
int rv = snprintf(env_name, NAME_MAX, "%s%s", prefixes[i], option);
if (rv < 0) {
crm_err("Failed to write %s%s to buffer: %s", prefixes[i], option,
strerror(errno));
return NULL;
}
if (rv >= sizeof(env_name)) {
crm_trace("\"%s%s\" is too long", prefixes[i], option);
continue;
}
value = getenv(env_name);
if (value != NULL) {
crm_trace("Found %s = %s", env_name, value);
return value;
}
}
crm_trace("Nothing found for %s", option);
return NULL;
}
/*!
* \brief Set or unset a Pacemaker environment variable option
*
- * Set an environment variable option with both a PCMK_ and (for
- * backward compatibility) HA_ prefix.
+ * Set an environment variable option with a \c "PCMK_" prefix and optionally
+ * an \c "HA_" prefix for backward compatibility.
*
* \param[in] option Environment variable name (without prefix)
* \param[in] value New value (or NULL to unset)
+ * \param[in] compat If false and \p value is not \c NULL, set only
+ * \c "PCMK_<option>"; otherwise, set (or unset) both
+ * \c "PCMK_<option>" and \c "HA_<option>"
+ *
+ * \note \p compat is ignored when \p value is \c NULL. A \c NULL \p value
+ * means we're unsetting \p option. \c pcmk__get_env_option() checks for
+ * both prefixes, so we want to clear them both.
*/
void
-pcmk__set_env_option(const char *option, const char *value)
+pcmk__set_env_option(const char *option, const char *value, bool compat)
{
+ // @COMPAT Drop support for "HA_" options eventually
const char *const prefixes[] = {"PCMK_", "HA_"};
char env_name[NAME_MAX];
CRM_CHECK(!pcmk__str_empty(option) && (strchr(option, '=') == NULL),
return);
for (int i = 0; i < PCMK__NELEM(prefixes); i++) {
int rv = snprintf(env_name, NAME_MAX, "%s%s", prefixes[i], option);
if (rv < 0) {
crm_err("Failed to write %s%s to buffer: %s", prefixes[i], option,
strerror(errno));
return;
}
if (rv >= sizeof(env_name)) {
crm_trace("\"%s%s\" is too long", prefixes[i], option);
continue;
}
if (value != NULL) {
crm_trace("Setting %s to %s", env_name, value);
rv = setenv(env_name, value, 1);
} else {
crm_trace("Unsetting %s", env_name);
rv = unsetenv(env_name);
}
if (rv < 0) {
crm_err("Failed to %sset %s: %s", (value != NULL)? "" : "un",
env_name, strerror(errno));
}
+
+ if (!compat && (value != NULL)) {
+ // For set, don't proceed to HA_<option> unless compat is enabled
+ break;
+ }
}
}
/*!
* \internal
* \brief Check whether Pacemaker environment variable option is enabled
*
* Given a Pacemaker environment variable option that can either be boolean
* or a list of daemon names, return true if the option is enabled for a given
* daemon.
*
* \param[in] daemon Daemon name (can be NULL)
* \param[in] option Pacemaker environment variable name
*
* \return true if variable is enabled for daemon, otherwise false
*/
bool
pcmk__env_option_enabled(const char *daemon, const char *option)
{
const char *value = pcmk__env_option(option);
return (value != NULL)
&& (crm_is_true(value)
|| ((daemon != NULL) && (strstr(value, daemon) != NULL)));
}
/*
* Cluster option handling
*/
bool
pcmk__valid_interval_spec(const char *value)
{
(void) crm_parse_interval_spec(value);
return errno == 0;
}
bool
pcmk__valid_boolean(const char *value)
{
int tmp;
return crm_str_to_boolean(value, &tmp) == 1;
}
bool
pcmk__valid_number(const char *value)
{
if (value == NULL) {
return false;
} else if (pcmk_str_is_minus_infinity(value) ||
pcmk_str_is_infinity(value)) {
return true;
}
return pcmk__scan_ll(value, NULL, 0LL) == pcmk_rc_ok;
}
bool
pcmk__valid_positive_number(const char *value)
{
long long num = 0LL;
return pcmk_str_is_infinity(value)
|| ((pcmk__scan_ll(value, &num, 0LL) == pcmk_rc_ok) && (num > 0));
}
bool
pcmk__valid_quorum(const char *value)
{
return pcmk__strcase_any_of(value, "stop", "freeze", "ignore", "demote", "suicide", NULL);
}
bool
pcmk__valid_script(const char *value)
{
struct stat st;
if (pcmk__str_eq(value, "/dev/null", pcmk__str_casei)) {
return true;
}
if (stat(value, &st) != 0) {
crm_err("Script %s does not exist", value);
return false;
}
if (S_ISREG(st.st_mode) == 0) {
crm_err("Script %s is not a regular file", value);
return false;
}
if ((st.st_mode & (S_IXUSR | S_IXGRP)) == 0) {
crm_err("Script %s is not executable", value);
return false;
}
return true;
}
bool
pcmk__valid_percentage(const char *value)
{
char *end = NULL;
long number = strtol(value, &end, 10);
if (end && (end[0] != '%')) {
return false;
}
return number >= 0;
}
/*!
* \internal
* \brief Check a table of configured options for a particular option
*
* \param[in,out] options Name/value pairs for configured options
* \param[in] validate If not NULL, validator function for option value
* \param[in] name Option name to look for
* \param[in] old_name Alternative option name to look for
* \param[in] def_value Default to use if option not configured
*
* \return Option value (from supplied options table or default value)
*/
static const char *
cluster_option_value(GHashTable *options, bool (*validate)(const char *),
const char *name, const char *old_name,
const char *def_value)
{
const char *value = NULL;
char *new_value = NULL;
CRM_ASSERT(name != NULL);
if (options) {
value = g_hash_table_lookup(options, name);
if ((value == NULL) && old_name) {
value = g_hash_table_lookup(options, old_name);
if (value != NULL) {
pcmk__config_warn("Support for legacy name '%s' for cluster "
"option '%s' is deprecated and will be "
"removed in a future release",
old_name, name);
// Inserting copy with current name ensures we only warn once
new_value = strdup(value);
g_hash_table_insert(options, strdup(name), new_value);
value = new_value;
}
}
if (value && validate && (validate(value) == FALSE)) {
pcmk__config_err("Using default value for cluster option '%s' "
"because '%s' is invalid", name, value);
value = NULL;
}
if (value) {
return value;
}
}
// No value found, use default
value = def_value;
if (value == NULL) {
crm_trace("No value or default provided for cluster option '%s'",
name);
return NULL;
}
if (validate) {
CRM_CHECK(validate(value) != FALSE,
crm_err("Bug: default value for cluster option '%s' is invalid", name);
return NULL);
}
crm_trace("Using default value '%s' for cluster option '%s'",
value, name);
if (options) {
new_value = strdup(value);
g_hash_table_insert(options, strdup(name), new_value);
value = new_value;
}
return value;
}
/*!
* \internal
* \brief Get the value of a cluster option
*
* \param[in,out] options Name/value pairs for configured options
* \param[in] option_list Possible cluster options
* \param[in] len Length of \p option_list
* \param[in] name (Primary) option name to look for
*
* \return Option value
*/
const char *
pcmk__cluster_option(GHashTable *options,
const pcmk__cluster_option_t *option_list,
int len, const char *name)
{
const char *value = NULL;
for (int lpc = 0; lpc < len; lpc++) {
if (pcmk__str_eq(name, option_list[lpc].name, pcmk__str_casei)) {
value = cluster_option_value(options, option_list[lpc].is_valid,
option_list[lpc].name,
option_list[lpc].alt_name,
option_list[lpc].default_value);
return value;
}
}
CRM_CHECK(FALSE, crm_err("Bug: looking for unknown option '%s'", name));
return NULL;
}
/*!
* \internal
* \brief Add a description element to a meta-data string
*
* \param[in,out] s Meta-data string to add to
* \param[in] tag Name of element to add ("longdesc" or "shortdesc")
* \param[in] desc Textual description to add
* \param[in] values If not \p NULL, the allowed values for the parameter
* \param[in] spaces If not \p NULL, spaces to insert at the beginning of
* each line
*/
static void
add_desc(GString *s, const char *tag, const char *desc, const char *values,
const char *spaces)
{
char *escaped_en = crm_xml_escape(desc);
if (spaces != NULL) {
g_string_append(s, spaces);
}
pcmk__g_strcat(s, "<", tag, " lang=\"en\">", escaped_en, NULL);
if (values != NULL) {
pcmk__g_strcat(s, " Allowed values: ", values, NULL);
}
pcmk__g_strcat(s, "</", tag, ">\n", NULL);
#ifdef ENABLE_NLS
{
static const char *locale = NULL;
char *localized = crm_xml_escape(_(desc));
if (strcmp(escaped_en, localized) != 0) {
if (locale == NULL) {
locale = strtok(setlocale(LC_ALL, NULL), "_");
}
if (spaces != NULL) {
g_string_append(s, spaces);
}
pcmk__g_strcat(s, "<", tag, " lang=\"", locale, "\">", localized,
NULL);
if (values != NULL) {
pcmk__g_strcat(s, _(" Allowed values: "), _(values), NULL);
}
pcmk__g_strcat(s, "</", tag, ">\n", NULL);
}
free(localized);
}
#endif
free(escaped_en);
}
gchar *
pcmk__format_option_metadata(const char *name, const char *desc_short,
const char *desc_long,
pcmk__cluster_option_t *option_list, int len)
{
/* big enough to hold "pacemaker-schedulerd metadata" output */
GString *s = g_string_sized_new(13000);
pcmk__g_strcat(s,
"<?xml version=\"1.0\"?>\n"
"<resource-agent name=\"", name, "\" "
"version=\"" PACEMAKER_VERSION "\">\n"
" <version>" PCMK_OCF_VERSION "</version>\n", NULL);
add_desc(s, "longdesc", desc_long, NULL, " ");
add_desc(s, "shortdesc", desc_short, NULL, " ");
g_string_append(s, " <parameters>\n");
for (int lpc = 0; lpc < len; lpc++) {
const char *opt_name = option_list[lpc].name;
const char *opt_type = option_list[lpc].type;
const char *opt_values = option_list[lpc].values;
const char *opt_default = option_list[lpc].default_value;
const char *opt_desc_short = option_list[lpc].description_short;
const char *opt_desc_long = option_list[lpc].description_long;
// The standard requires long and short parameter descriptions
CRM_ASSERT((opt_desc_short != NULL) || (opt_desc_long != NULL));
if (opt_desc_short == NULL) {
opt_desc_short = opt_desc_long;
} else if (opt_desc_long == NULL) {
opt_desc_long = opt_desc_short;
}
// The standard requires a parameter type
CRM_ASSERT(opt_type != NULL);
pcmk__g_strcat(s, " <parameter name=\"", opt_name, "\">\n", NULL);
add_desc(s, "longdesc", opt_desc_long, opt_values, " ");
add_desc(s, "shortdesc", opt_desc_short, NULL, " ");
pcmk__g_strcat(s, " <content type=\"", opt_type, "\"", NULL);
if (opt_default != NULL) {
pcmk__g_strcat(s, " default=\"", opt_default, "\"", NULL);
}
if ((opt_values != NULL) && (strcmp(opt_type, "select") == 0)) {
char *str = strdup(opt_values);
const char *delim = ", ";
char *ptr = strtok(str, delim);
g_string_append(s, ">\n");
while (ptr != NULL) {
pcmk__g_strcat(s, " <option value=\"", ptr, "\" />\n",
NULL);
ptr = strtok(NULL, delim);
}
g_string_append_printf(s, " </content>\n");
free(str);
} else {
g_string_append(s, "/>\n");
}
g_string_append(s, " </parameter>\n");
}
g_string_append(s, " </parameters>\n</resource-agent>\n");
return g_string_free(s, FALSE);
}
void
pcmk__validate_cluster_options(GHashTable *options,
pcmk__cluster_option_t *option_list, int len)
{
for (int lpc = 0; lpc < len; lpc++) {
cluster_option_value(options, option_list[lpc].is_valid,
option_list[lpc].name,
option_list[lpc].alt_name,
option_list[lpc].default_value);
}
}
diff --git a/lib/common/remote.c b/lib/common/remote.c
index 17ae646994..fe19296e7a 100644
--- a/lib/common/remote.c
+++ b/lib/common/remote.c
@@ -1,1274 +1,1279 @@
/*
* Copyright 2008-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <crm/crm.h>
#include <sys/param.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/tcp.h>
#include <netdb.h>
#include <stdlib.h>
#include <errno.h>
#include <inttypes.h> // PRIx32
#include <glib.h>
#include <bzlib.h>
#include <crm/common/ipc_internal.h>
#include <crm/common/xml.h>
#include <crm/common/mainloop.h>
#include <crm/common/remote_internal.h>
#ifdef HAVE_GNUTLS_GNUTLS_H
# include <gnutls/gnutls.h>
#endif
/* Swab macros from linux/swab.h */
#ifdef HAVE_LINUX_SWAB_H
# include <linux/swab.h>
#else
/*
* casts are necessary for constants, because we never know how for sure
* how U/UL/ULL map to __u16, __u32, __u64. At least not in a portable way.
*/
#define __swab16(x) ((uint16_t)( \
(((uint16_t)(x) & (uint16_t)0x00ffU) << 8) | \
(((uint16_t)(x) & (uint16_t)0xff00U) >> 8)))
#define __swab32(x) ((uint32_t)( \
(((uint32_t)(x) & (uint32_t)0x000000ffUL) << 24) | \
(((uint32_t)(x) & (uint32_t)0x0000ff00UL) << 8) | \
(((uint32_t)(x) & (uint32_t)0x00ff0000UL) >> 8) | \
(((uint32_t)(x) & (uint32_t)0xff000000UL) >> 24)))
#define __swab64(x) ((uint64_t)( \
(((uint64_t)(x) & (uint64_t)0x00000000000000ffULL) << 56) | \
(((uint64_t)(x) & (uint64_t)0x000000000000ff00ULL) << 40) | \
(((uint64_t)(x) & (uint64_t)0x0000000000ff0000ULL) << 24) | \
(((uint64_t)(x) & (uint64_t)0x00000000ff000000ULL) << 8) | \
(((uint64_t)(x) & (uint64_t)0x000000ff00000000ULL) >> 8) | \
(((uint64_t)(x) & (uint64_t)0x0000ff0000000000ULL) >> 24) | \
(((uint64_t)(x) & (uint64_t)0x00ff000000000000ULL) >> 40) | \
(((uint64_t)(x) & (uint64_t)0xff00000000000000ULL) >> 56)))
#endif
#define REMOTE_MSG_VERSION 1
#define ENDIAN_LOCAL 0xBADADBBD
struct remote_header_v0 {
uint32_t endian; /* Detect messages from hosts with different endian-ness */
uint32_t version;
uint64_t id;
uint64_t flags;
uint32_t size_total;
uint32_t payload_offset;
uint32_t payload_compressed;
uint32_t payload_uncompressed;
/* New fields get added here */
} __attribute__ ((packed));
/*!
* \internal
* \brief Retrieve remote message header, in local endianness
*
* Return a pointer to the header portion of a remote connection's message
* buffer, converting the header to local endianness if needed.
*
* \param[in,out] remote Remote connection with new message
*
* \return Pointer to message header, localized if necessary
*/
static struct remote_header_v0 *
localized_remote_header(pcmk__remote_t *remote)
{
struct remote_header_v0 *header = (struct remote_header_v0 *)remote->buffer;
if(remote->buffer_offset < sizeof(struct remote_header_v0)) {
return NULL;
} else if(header->endian != ENDIAN_LOCAL) {
uint32_t endian = __swab32(header->endian);
CRM_LOG_ASSERT(endian == ENDIAN_LOCAL);
if(endian != ENDIAN_LOCAL) {
crm_err("Invalid message detected, endian mismatch: %" PRIx32
" is neither %" PRIx32 " nor the swab'd %" PRIx32,
ENDIAN_LOCAL, header->endian, endian);
return NULL;
}
header->id = __swab64(header->id);
header->flags = __swab64(header->flags);
header->endian = __swab32(header->endian);
header->version = __swab32(header->version);
header->size_total = __swab32(header->size_total);
header->payload_offset = __swab32(header->payload_offset);
header->payload_compressed = __swab32(header->payload_compressed);
header->payload_uncompressed = __swab32(header->payload_uncompressed);
}
return header;
}
#ifdef HAVE_GNUTLS_GNUTLS_H
int
pcmk__tls_client_handshake(pcmk__remote_t *remote, int timeout_ms)
{
int rc = 0;
int pollrc = 0;
time_t time_limit = time(NULL) + timeout_ms / 1000;
do {
rc = gnutls_handshake(*remote->tls_session);
if ((rc == GNUTLS_E_INTERRUPTED) || (rc == GNUTLS_E_AGAIN)) {
pollrc = pcmk__remote_ready(remote, 1000);
if ((pollrc != pcmk_rc_ok) && (pollrc != ETIME)) {
/* poll returned error, there is no hope */
crm_trace("TLS handshake poll failed: %s (%d)",
pcmk_strerror(pollrc), pollrc);
return pcmk_legacy2rc(pollrc);
}
} else if (rc < 0) {
crm_trace("TLS handshake failed: %s (%d)",
gnutls_strerror(rc), rc);
return EPROTO;
} else {
return pcmk_rc_ok;
}
} while (time(NULL) < time_limit);
return ETIME;
}
/*!
* \internal
* \brief Set minimum prime size required by TLS client
*
* \param[in] session TLS session to affect
*/
static void
set_minimum_dh_bits(const gnutls_session_t *session)
{
int dh_min_bits;
- pcmk__scan_min_int(getenv("PCMK_dh_min_bits"), &dh_min_bits, 0);
+ pcmk__scan_min_int(pcmk__env_option(PCMK__ENV_DH_MIN_BITS), &dh_min_bits,
+ 0);
/* This function is deprecated since GnuTLS 3.1.7, in favor of letting
* the priority string imply the DH requirements, but this is the only
* way to give the user control over compatibility with older servers.
*/
if (dh_min_bits > 0) {
crm_info("Requiring server use a Diffie-Hellman prime of at least %d bits",
dh_min_bits);
gnutls_dh_set_prime_bits(*session, dh_min_bits);
}
}
static unsigned int
get_bound_dh_bits(unsigned int dh_bits)
{
int dh_min_bits;
int dh_max_bits;
- pcmk__scan_min_int(getenv("PCMK_dh_min_bits"), &dh_min_bits, 0);
- pcmk__scan_min_int(getenv("PCMK_dh_max_bits"), &dh_max_bits, 0);
+ pcmk__scan_min_int(pcmk__env_option(PCMK__ENV_DH_MIN_BITS), &dh_min_bits,
+ 0);
+ pcmk__scan_min_int(pcmk__env_option(PCMK__ENV_DH_MAX_BITS), &dh_max_bits,
+ 0);
+
if ((dh_max_bits > 0) && (dh_max_bits < dh_min_bits)) {
crm_warn("Ignoring PCMK_dh_max_bits less than PCMK_dh_min_bits");
dh_max_bits = 0;
}
if ((dh_min_bits > 0) && (dh_bits < dh_min_bits)) {
return dh_min_bits;
}
if ((dh_max_bits > 0) && (dh_bits > dh_max_bits)) {
return dh_max_bits;
}
return dh_bits;
}
/*!
* \internal
* \brief Initialize a new TLS session
*
* \param[in] csock Connected socket for TLS session
* \param[in] conn_type GNUTLS_SERVER or GNUTLS_CLIENT
* \param[in] cred_type GNUTLS_CRD_ANON or GNUTLS_CRD_PSK
* \param[in] credentials TLS session credentials
*
* \return Pointer to newly created session object, or NULL on error
*/
gnutls_session_t *
pcmk__new_tls_session(int csock, unsigned int conn_type,
gnutls_credentials_type_t cred_type, void *credentials)
{
int rc = GNUTLS_E_SUCCESS;
const char *prio_base = NULL;
char *prio = NULL;
gnutls_session_t *session = NULL;
/* Determine list of acceptable ciphers, etc. Pacemaker always adds the
* values required for its functionality.
*
* For an example of anonymous authentication, see:
* http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication
*/
- prio_base = getenv("PCMK_tls_priorities");
+ prio_base = pcmk__env_option(PCMK__ENV_TLS_PRIORITIES);
if (prio_base == NULL) {
prio_base = PCMK_GNUTLS_PRIORITIES;
}
prio = crm_strdup_printf("%s:%s", prio_base,
(cred_type == GNUTLS_CRD_ANON)? "+ANON-DH" : "+DHE-PSK:+PSK");
session = gnutls_malloc(sizeof(gnutls_session_t));
if (session == NULL) {
rc = GNUTLS_E_MEMORY_ERROR;
goto error;
}
rc = gnutls_init(session, conn_type);
if (rc != GNUTLS_E_SUCCESS) {
goto error;
}
/* @TODO On the server side, it would be more efficient to cache the
* priority with gnutls_priority_init2() and set it with
* gnutls_priority_set() for all sessions.
*/
rc = gnutls_priority_set_direct(*session, prio, NULL);
if (rc != GNUTLS_E_SUCCESS) {
goto error;
}
if (conn_type == GNUTLS_CLIENT) {
set_minimum_dh_bits(session);
}
gnutls_transport_set_ptr(*session,
(gnutls_transport_ptr_t) GINT_TO_POINTER(csock));
rc = gnutls_credentials_set(*session, cred_type, credentials);
if (rc != GNUTLS_E_SUCCESS) {
goto error;
}
free(prio);
return session;
error:
crm_err("Could not initialize %s TLS %s session: %s "
CRM_XS " rc=%d priority='%s'",
(cred_type == GNUTLS_CRD_ANON)? "anonymous" : "PSK",
(conn_type == GNUTLS_SERVER)? "server" : "client",
gnutls_strerror(rc), rc, prio);
free(prio);
if (session != NULL) {
gnutls_free(session);
}
return NULL;
}
/*!
* \internal
* \brief Initialize Diffie-Hellman parameters for a TLS server
*
* \param[out] dh_params Parameter object to initialize
*
* \return Standard Pacemaker return code
* \todo The current best practice is to allow the client and server to
* negotiate the Diffie-Hellman parameters via a TLS extension (RFC 7919).
* However, we have to support both older versions of GnuTLS (<3.6) that
* don't support the extension on our side, and older Pacemaker versions
* that don't support the extension on the other side. The next best
* practice would be to use a known good prime (see RFC 5114 section 2.2),
* possibly stored in a file distributed with Pacemaker.
*/
int
pcmk__init_tls_dh(gnutls_dh_params_t *dh_params)
{
int rc = GNUTLS_E_SUCCESS;
unsigned int dh_bits = 0;
rc = gnutls_dh_params_init(dh_params);
if (rc != GNUTLS_E_SUCCESS) {
goto error;
}
dh_bits = gnutls_sec_param_to_pk_bits(GNUTLS_PK_DH,
GNUTLS_SEC_PARAM_NORMAL);
if (dh_bits == 0) {
rc = GNUTLS_E_DH_PRIME_UNACCEPTABLE;
goto error;
}
dh_bits = get_bound_dh_bits(dh_bits);
crm_info("Generating Diffie-Hellman parameters with %u-bit prime for TLS",
dh_bits);
rc = gnutls_dh_params_generate2(*dh_params, dh_bits);
if (rc != GNUTLS_E_SUCCESS) {
goto error;
}
return pcmk_rc_ok;
error:
crm_err("Could not initialize Diffie-Hellman parameters for TLS: %s "
CRM_XS " rc=%d", gnutls_strerror(rc), rc);
return EPROTO;
}
/*!
* \internal
* \brief Process handshake data from TLS client
*
* Read as much TLS handshake data as is available.
*
* \param[in] client Client connection
*
* \return Standard Pacemaker return code (of particular interest, EAGAIN
* if some data was successfully read but more data is needed)
*/
int
pcmk__read_handshake_data(const pcmk__client_t *client)
{
int rc = 0;
CRM_ASSERT(client && client->remote && client->remote->tls_session);
do {
rc = gnutls_handshake(*client->remote->tls_session);
} while (rc == GNUTLS_E_INTERRUPTED);
if (rc == GNUTLS_E_AGAIN) {
/* No more data is available at the moment. This function should be
* invoked again once the client sends more.
*/
return EAGAIN;
} else if (rc != GNUTLS_E_SUCCESS) {
crm_err("TLS handshake with remote client failed: %s "
CRM_XS " rc=%d", gnutls_strerror(rc), rc);
return EPROTO;
}
return pcmk_rc_ok;
}
// \return Standard Pacemaker return code
static int
send_tls(gnutls_session_t *session, struct iovec *iov)
{
const char *unsent = iov->iov_base;
size_t unsent_len = iov->iov_len;
ssize_t gnutls_rc;
if (unsent == NULL) {
return EINVAL;
}
crm_trace("Sending TLS message of %llu bytes",
(unsigned long long) unsent_len);
while (true) {
gnutls_rc = gnutls_record_send(*session, unsent, unsent_len);
if (gnutls_rc == GNUTLS_E_INTERRUPTED || gnutls_rc == GNUTLS_E_AGAIN) {
crm_trace("Retrying to send %llu bytes remaining",
(unsigned long long) unsent_len);
} else if (gnutls_rc < 0) {
// Caller can log as error if necessary
crm_info("TLS connection terminated: %s " CRM_XS " rc=%lld",
gnutls_strerror((int) gnutls_rc),
(long long) gnutls_rc);
return ECONNABORTED;
} else if (gnutls_rc < unsent_len) {
crm_trace("Sent %lld of %llu bytes remaining",
(long long) gnutls_rc, (unsigned long long) unsent_len);
unsent_len -= gnutls_rc;
unsent += gnutls_rc;
} else {
crm_trace("Sent all %lld bytes remaining", (long long) gnutls_rc);
break;
}
}
return pcmk_rc_ok;
}
#endif
// \return Standard Pacemaker return code
static int
send_plaintext(int sock, struct iovec *iov)
{
const char *unsent = iov->iov_base;
size_t unsent_len = iov->iov_len;
ssize_t write_rc;
if (unsent == NULL) {
return EINVAL;
}
crm_debug("Sending plaintext message of %llu bytes to socket %d",
(unsigned long long) unsent_len, sock);
while (true) {
write_rc = write(sock, unsent, unsent_len);
if (write_rc < 0) {
int rc = errno;
if ((errno == EINTR) || (errno == EAGAIN)) {
crm_trace("Retrying to send %llu bytes remaining to socket %d",
(unsigned long long) unsent_len, sock);
continue;
}
// Caller can log as error if necessary
crm_info("Could not send message: %s " CRM_XS " rc=%d socket=%d",
pcmk_rc_str(rc), rc, sock);
return rc;
} else if (write_rc < unsent_len) {
crm_trace("Sent %lld of %llu bytes remaining",
(long long) write_rc, (unsigned long long) unsent_len);
unsent += write_rc;
unsent_len -= write_rc;
continue;
} else {
crm_trace("Sent all %lld bytes remaining: %.100s",
(long long) write_rc, (char *) (iov->iov_base));
break;
}
}
return pcmk_rc_ok;
}
// \return Standard Pacemaker return code
static int
remote_send_iovs(pcmk__remote_t *remote, struct iovec *iov, int iovs)
{
int rc = pcmk_rc_ok;
for (int lpc = 0; (lpc < iovs) && (rc == pcmk_rc_ok); lpc++) {
#ifdef HAVE_GNUTLS_GNUTLS_H
if (remote->tls_session) {
rc = send_tls(remote->tls_session, &(iov[lpc]));
continue;
}
#endif
if (remote->tcp_socket) {
rc = send_plaintext(remote->tcp_socket, &(iov[lpc]));
} else {
rc = ESOCKTNOSUPPORT;
}
}
return rc;
}
/*!
* \internal
* \brief Send an XML message over a Pacemaker Remote connection
*
* \param[in,out] remote Pacemaker Remote connection to use
* \param[in] msg XML to send
*
* \return Standard Pacemaker return code
*/
int
pcmk__remote_send_xml(pcmk__remote_t *remote, const xmlNode *msg)
{
int rc = pcmk_rc_ok;
static uint64_t id = 0;
char *xml_text = NULL;
struct iovec iov[2];
struct remote_header_v0 *header;
CRM_CHECK((remote != NULL) && (msg != NULL), return EINVAL);
xml_text = dump_xml_unformatted(msg);
CRM_CHECK(xml_text != NULL, return EINVAL);
header = calloc(1, sizeof(struct remote_header_v0));
CRM_ASSERT(header != NULL);
iov[0].iov_base = header;
iov[0].iov_len = sizeof(struct remote_header_v0);
iov[1].iov_base = xml_text;
iov[1].iov_len = 1 + strlen(xml_text);
id++;
header->id = id;
header->endian = ENDIAN_LOCAL;
header->version = REMOTE_MSG_VERSION;
header->payload_offset = iov[0].iov_len;
header->payload_uncompressed = iov[1].iov_len;
header->size_total = iov[0].iov_len + iov[1].iov_len;
rc = remote_send_iovs(remote, iov, 2);
if (rc != pcmk_rc_ok) {
crm_err("Could not send remote message: %s " CRM_XS " rc=%d",
pcmk_rc_str(rc), rc);
}
free(iov[0].iov_base);
free(iov[1].iov_base);
return rc;
}
/*!
* \internal
* \brief Obtain the XML from the currently buffered remote connection message
*
* \param[in,out] remote Remote connection possibly with message available
*
* \return Newly allocated XML object corresponding to message data, or NULL
* \note This effectively removes the message from the connection buffer.
*/
xmlNode *
pcmk__remote_message_xml(pcmk__remote_t *remote)
{
xmlNode *xml = NULL;
struct remote_header_v0 *header = localized_remote_header(remote);
if (header == NULL) {
return NULL;
}
/* Support compression on the receiving end now, in case we ever want to add it later */
if (header->payload_compressed) {
int rc = 0;
unsigned int size_u = 1 + header->payload_uncompressed;
char *uncompressed = calloc(1, header->payload_offset + size_u);
crm_trace("Decompressing message data %d bytes into %d bytes",
header->payload_compressed, size_u);
rc = BZ2_bzBuffToBuffDecompress(uncompressed + header->payload_offset, &size_u,
remote->buffer + header->payload_offset,
header->payload_compressed, 1, 0);
rc = pcmk__bzlib2rc(rc);
if (rc != pcmk_rc_ok && header->version > REMOTE_MSG_VERSION) {
crm_warn("Couldn't decompress v%d message, we only understand v%d",
header->version, REMOTE_MSG_VERSION);
free(uncompressed);
return NULL;
} else if (rc != pcmk_rc_ok) {
crm_err("Decompression failed: %s " CRM_XS " rc=%d",
pcmk_rc_str(rc), rc);
free(uncompressed);
return NULL;
}
CRM_ASSERT(size_u == header->payload_uncompressed);
memcpy(uncompressed, remote->buffer, header->payload_offset); /* Preserve the header */
remote->buffer_size = header->payload_offset + size_u;
free(remote->buffer);
remote->buffer = uncompressed;
header = localized_remote_header(remote);
}
/* take ownership of the buffer */
remote->buffer_offset = 0;
CRM_LOG_ASSERT(remote->buffer[sizeof(struct remote_header_v0) + header->payload_uncompressed - 1] == 0);
xml = string2xml(remote->buffer + header->payload_offset);
if (xml == NULL && header->version > REMOTE_MSG_VERSION) {
crm_warn("Couldn't parse v%d message, we only understand v%d",
header->version, REMOTE_MSG_VERSION);
} else if (xml == NULL) {
crm_err("Couldn't parse: '%.120s'", remote->buffer + header->payload_offset);
}
return xml;
}
static int
get_remote_socket(const pcmk__remote_t *remote)
{
#ifdef HAVE_GNUTLS_GNUTLS_H
if (remote->tls_session) {
void *sock_ptr = gnutls_transport_get_ptr(*remote->tls_session);
return GPOINTER_TO_INT(sock_ptr);
}
#endif
if (remote->tcp_socket) {
return remote->tcp_socket;
}
crm_err("Remote connection type undetermined (bug?)");
return -1;
}
/*!
* \internal
* \brief Wait for a remote session to have data to read
*
* \param[in] remote Connection to check
* \param[in] timeout_ms Maximum time (in ms) to wait
*
* \return Standard Pacemaker return code (of particular interest, pcmk_rc_ok if
* there is data ready to be read, and ETIME if there is no data within
* the specified timeout)
*/
int
pcmk__remote_ready(const pcmk__remote_t *remote, int timeout_ms)
{
struct pollfd fds = { 0, };
int sock = 0;
int rc = 0;
time_t start;
int timeout = timeout_ms;
sock = get_remote_socket(remote);
if (sock <= 0) {
crm_trace("No longer connected");
return ENOTCONN;
}
start = time(NULL);
errno = 0;
do {
fds.fd = sock;
fds.events = POLLIN;
/* If we got an EINTR while polling, and we have a
* specific timeout we are trying to honor, attempt
* to adjust the timeout to the closest second. */
if (errno == EINTR && (timeout > 0)) {
timeout = timeout_ms - ((time(NULL) - start) * 1000);
if (timeout < 1000) {
timeout = 1000;
}
}
rc = poll(&fds, 1, timeout);
} while (rc < 0 && errno == EINTR);
if (rc < 0) {
return errno;
}
return (rc == 0)? ETIME : pcmk_rc_ok;
}
/*!
* \internal
* \brief Read bytes from non-blocking remote connection
*
* \param[in,out] remote Remote connection to read
*
* \return Standard Pacemaker return code (of particular interest, pcmk_rc_ok if
* a full message has been received, or EAGAIN for a partial message)
* \note Use only with non-blocking sockets after polling the socket.
* \note This function will return when the socket read buffer is empty or an
* error is encountered.
*/
static int
read_available_remote_data(pcmk__remote_t *remote)
{
int rc = pcmk_rc_ok;
size_t read_len = sizeof(struct remote_header_v0);
struct remote_header_v0 *header = localized_remote_header(remote);
bool received = false;
ssize_t read_rc;
if(header) {
/* Stop at the end of the current message */
read_len = header->size_total;
}
/* automatically grow the buffer when needed */
if(remote->buffer_size < read_len) {
remote->buffer_size = 2 * read_len;
crm_trace("Expanding buffer to %llu bytes",
(unsigned long long) remote->buffer_size);
remote->buffer = pcmk__realloc(remote->buffer, remote->buffer_size + 1);
}
#ifdef HAVE_GNUTLS_GNUTLS_H
if (!received && remote->tls_session) {
read_rc = gnutls_record_recv(*(remote->tls_session),
remote->buffer + remote->buffer_offset,
remote->buffer_size - remote->buffer_offset);
if (read_rc == GNUTLS_E_INTERRUPTED) {
rc = EINTR;
} else if (read_rc == GNUTLS_E_AGAIN) {
rc = EAGAIN;
} else if (read_rc < 0) {
crm_debug("TLS receive failed: %s (%lld)",
gnutls_strerror(read_rc), (long long) read_rc);
rc = EIO;
}
received = true;
}
#endif
if (!received && remote->tcp_socket) {
read_rc = read(remote->tcp_socket,
remote->buffer + remote->buffer_offset,
remote->buffer_size - remote->buffer_offset);
if (read_rc < 0) {
rc = errno;
}
received = true;
}
if (!received) {
crm_err("Remote connection type undetermined (bug?)");
return ESOCKTNOSUPPORT;
}
/* process any errors. */
if (read_rc > 0) {
remote->buffer_offset += read_rc;
/* always null terminate buffer, the +1 to alloc always allows for this. */
remote->buffer[remote->buffer_offset] = '\0';
crm_trace("Received %lld more bytes (%llu total)",
(long long) read_rc,
(unsigned long long) remote->buffer_offset);
} else if ((rc == EINTR) || (rc == EAGAIN)) {
crm_trace("No data available for non-blocking remote read: %s (%d)",
pcmk_rc_str(rc), rc);
} else if (read_rc == 0) {
crm_debug("End of remote data encountered after %llu bytes",
(unsigned long long) remote->buffer_offset);
return ENOTCONN;
} else {
crm_debug("Error receiving remote data after %llu bytes: %s (%d)",
(unsigned long long) remote->buffer_offset,
pcmk_rc_str(rc), rc);
return ENOTCONN;
}
header = localized_remote_header(remote);
if(header) {
if(remote->buffer_offset < header->size_total) {
crm_trace("Read partial remote message (%llu of %u bytes)",
(unsigned long long) remote->buffer_offset,
header->size_total);
} else {
crm_trace("Read full remote message of %llu bytes",
(unsigned long long) remote->buffer_offset);
return pcmk_rc_ok;
}
}
return EAGAIN;
}
/*!
* \internal
* \brief Read one message from a remote connection
*
* \param[in,out] remote Remote connection to read
* \param[in] timeout_ms Fail if message not read in this many milliseconds
* (10s will be used if 0, and 60s if negative)
*
* \return Standard Pacemaker return code
*/
int
pcmk__read_remote_message(pcmk__remote_t *remote, int timeout_ms)
{
int rc = pcmk_rc_ok;
time_t start = time(NULL);
int remaining_timeout = 0;
if (timeout_ms == 0) {
timeout_ms = 10000;
} else if (timeout_ms < 0) {
timeout_ms = 60000;
}
remaining_timeout = timeout_ms;
while (remaining_timeout > 0) {
crm_trace("Waiting for remote data (%d ms of %d ms timeout remaining)",
remaining_timeout, timeout_ms);
rc = pcmk__remote_ready(remote, remaining_timeout);
if (rc == ETIME) {
crm_err("Timed out (%d ms) while waiting for remote data",
remaining_timeout);
return rc;
} else if (rc != pcmk_rc_ok) {
crm_debug("Wait for remote data aborted (will retry): %s "
CRM_XS " rc=%d", pcmk_rc_str(rc), rc);
} else {
rc = read_available_remote_data(remote);
if (rc == pcmk_rc_ok) {
return rc;
} else if (rc == EAGAIN) {
crm_trace("Waiting for more remote data");
} else {
crm_debug("Could not receive remote data: %s " CRM_XS " rc=%d",
pcmk_rc_str(rc), rc);
}
}
// Don't waste time retrying after fatal errors
if ((rc == ENOTCONN) || (rc == ESOCKTNOSUPPORT)) {
return rc;
}
remaining_timeout = timeout_ms - ((time(NULL) - start) * 1000);
}
return ETIME;
}
struct tcp_async_cb_data {
int sock;
int timeout_ms;
time_t start;
void *userdata;
void (*callback) (void *userdata, int rc, int sock);
};
// \return TRUE if timer should be rescheduled, FALSE otherwise
static gboolean
check_connect_finished(gpointer userdata)
{
struct tcp_async_cb_data *cb_data = userdata;
int rc;
fd_set rset, wset;
struct timeval ts = { 0, };
if (cb_data->start == 0) {
// Last connect() returned success immediately
rc = pcmk_rc_ok;
goto dispatch_done;
}
// If the socket is ready for reading or writing, the connect succeeded
FD_ZERO(&rset);
FD_SET(cb_data->sock, &rset);
wset = rset;
rc = select(cb_data->sock + 1, &rset, &wset, NULL, &ts);
if (rc < 0) { // select() error
rc = errno;
if ((rc == EINPROGRESS) || (rc == EAGAIN)) {
if ((time(NULL) - cb_data->start) < (cb_data->timeout_ms / 1000)) {
return TRUE; // There is time left, so reschedule timer
} else {
rc = ETIMEDOUT;
}
}
crm_trace("Could not check socket %d for connection success: %s (%d)",
cb_data->sock, pcmk_rc_str(rc), rc);
} else if (rc == 0) { // select() timeout
if ((time(NULL) - cb_data->start) < (cb_data->timeout_ms / 1000)) {
return TRUE; // There is time left, so reschedule timer
}
crm_debug("Timed out while waiting for socket %d connection success",
cb_data->sock);
rc = ETIMEDOUT;
// select() returned number of file descriptors that are ready
} else if (FD_ISSET(cb_data->sock, &rset)
|| FD_ISSET(cb_data->sock, &wset)) {
// The socket is ready; check it for connection errors
int error = 0;
socklen_t len = sizeof(error);
if (getsockopt(cb_data->sock, SOL_SOCKET, SO_ERROR, &error, &len) < 0) {
rc = errno;
crm_trace("Couldn't check socket %d for connection errors: %s (%d)",
cb_data->sock, pcmk_rc_str(rc), rc);
} else if (error != 0) {
rc = error;
crm_trace("Socket %d connected with error: %s (%d)",
cb_data->sock, pcmk_rc_str(rc), rc);
} else {
rc = pcmk_rc_ok;
}
} else { // Should not be possible
crm_trace("select() succeeded, but socket %d not in resulting "
"read/write sets", cb_data->sock);
rc = EAGAIN;
}
dispatch_done:
if (rc == pcmk_rc_ok) {
crm_trace("Socket %d is connected", cb_data->sock);
} else {
close(cb_data->sock);
cb_data->sock = -1;
}
if (cb_data->callback) {
cb_data->callback(cb_data->userdata, rc, cb_data->sock);
}
free(cb_data);
return FALSE; // Do not reschedule timer
}
/*!
* \internal
* \brief Attempt to connect socket, calling callback when done
*
* Set a given socket non-blocking, then attempt to connect to it,
* retrying periodically until success or a timeout is reached.
* Call a caller-supplied callback function when completed.
*
* \param[in] sock Newly created socket
* \param[in] addr Socket address information for connect
* \param[in] addrlen Size of socket address information in bytes
* \param[in] timeout_ms Fail if not connected within this much time
* \param[out] timer_id If not NULL, store retry timer ID here
* \param[in] userdata User data to pass to callback
* \param[in] callback Function to call when connection attempt completes
*
* \return Standard Pacemaker return code
*/
static int
connect_socket_retry(int sock, const struct sockaddr *addr, socklen_t addrlen,
int timeout_ms, int *timer_id, void *userdata,
void (*callback) (void *userdata, int rc, int sock))
{
int rc = 0;
int interval = 500;
int timer;
struct tcp_async_cb_data *cb_data = NULL;
rc = pcmk__set_nonblocking(sock);
if (rc != pcmk_rc_ok) {
crm_warn("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
pcmk_rc_str(rc), rc);
return rc;
}
rc = connect(sock, addr, addrlen);
if (rc < 0 && (errno != EINPROGRESS) && (errno != EAGAIN)) {
rc = errno;
crm_warn("Could not connect socket: %s " CRM_XS " rc=%d",
pcmk_rc_str(rc), rc);
return rc;
}
cb_data = calloc(1, sizeof(struct tcp_async_cb_data));
cb_data->userdata = userdata;
cb_data->callback = callback;
cb_data->sock = sock;
cb_data->timeout_ms = timeout_ms;
if (rc == 0) {
/* The connect was successful immediately, we still return to mainloop
* and let this callback get called later. This avoids the user of this api
* to have to account for the fact the callback could be invoked within this
* function before returning. */
cb_data->start = 0;
interval = 1;
} else {
cb_data->start = time(NULL);
}
/* This timer function does a non-blocking poll on the socket to see if we
* can use it. Once we can, the connect has completed. This method allows us
* to connect without blocking the mainloop.
*
* @TODO Use a mainloop fd callback for this instead of polling. Something
* about the way mainloop is currently polling prevents this from
* working at the moment though. (See connect(2) regarding EINPROGRESS
* for possible new handling needed.)
*/
crm_trace("Scheduling check in %dms for whether connect to fd %d finished",
interval, sock);
timer = g_timeout_add(interval, check_connect_finished, cb_data);
if (timer_id) {
*timer_id = timer;
}
// timer callback should be taking care of cb_data
// cppcheck-suppress memleak
return pcmk_rc_ok;
}
/*!
* \internal
* \brief Attempt once to connect socket and set it non-blocking
*
* \param[in] sock Newly created socket
* \param[in] addr Socket address information for connect
* \param[in] addrlen Size of socket address information in bytes
*
* \return Standard Pacemaker return code
*/
static int
connect_socket_once(int sock, const struct sockaddr *addr, socklen_t addrlen)
{
int rc = connect(sock, addr, addrlen);
if (rc < 0) {
rc = errno;
crm_warn("Could not connect socket: %s " CRM_XS " rc=%d",
pcmk_rc_str(rc), rc);
return rc;
}
rc = pcmk__set_nonblocking(sock);
if (rc != pcmk_rc_ok) {
crm_warn("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
pcmk_rc_str(rc), rc);
return rc;
}
return pcmk_ok;
}
/*!
* \internal
* \brief Connect to server at specified TCP port
*
* \param[in] host Name of server to connect to
* \param[in] port Server port to connect to
* \param[in] timeout_ms If asynchronous, fail if not connected in this time
* \param[out] timer_id If asynchronous and this is non-NULL, retry timer ID
* will be put here (for ease of cancelling by caller)
* \param[out] sock_fd Where to store socket file descriptor
* \param[in] userdata If asynchronous, data to pass to callback
* \param[in] callback If NULL, attempt a single synchronous connection,
* otherwise retry asynchronously then call this
*
* \return Standard Pacemaker return code
*/
int
pcmk__connect_remote(const char *host, int port, int timeout, int *timer_id,
int *sock_fd, void *userdata,
void (*callback) (void *userdata, int rc, int sock))
{
char buffer[INET6_ADDRSTRLEN];
struct addrinfo *res = NULL;
struct addrinfo *rp = NULL;
struct addrinfo hints;
const char *server = host;
int rc;
int sock = -1;
CRM_CHECK((host != NULL) && (sock_fd != NULL), return EINVAL);
// Get host's IP address(es)
memset(&hints, 0, sizeof(struct addrinfo));
hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
hints.ai_socktype = SOCK_STREAM;
hints.ai_flags = AI_CANONNAME;
rc = getaddrinfo(server, NULL, &hints, &res);
rc = pcmk__gaierror2rc(rc);
if (rc != pcmk_rc_ok) {
crm_err("Unable to get IP address info for %s: %s",
server, pcmk_rc_str(rc));
goto async_cleanup;
}
if (!res || !res->ai_addr) {
crm_err("Unable to get IP address info for %s: no result", server);
rc = ENOTCONN;
goto async_cleanup;
}
// getaddrinfo() returns a list of host's addresses, try them in order
for (rp = res; rp != NULL; rp = rp->ai_next) {
struct sockaddr *addr = rp->ai_addr;
if (!addr) {
continue;
}
if (rp->ai_canonname) {
server = res->ai_canonname;
}
crm_debug("Got canonical name %s for %s", server, host);
sock = socket(rp->ai_family, SOCK_STREAM, IPPROTO_TCP);
if (sock == -1) {
rc = errno;
crm_warn("Could not create socket for remote connection to %s:%d: "
"%s " CRM_XS " rc=%d", server, port, pcmk_rc_str(rc), rc);
continue;
}
/* Set port appropriately for address family */
/* (void*) casts avoid false-positive compiler alignment warnings */
if (addr->sa_family == AF_INET6) {
((struct sockaddr_in6 *)(void*)addr)->sin6_port = htons(port);
} else {
((struct sockaddr_in *)(void*)addr)->sin_port = htons(port);
}
memset(buffer, 0, PCMK__NELEM(buffer));
pcmk__sockaddr2str(addr, buffer);
crm_info("Attempting remote connection to %s:%d", buffer, port);
if (callback) {
if (connect_socket_retry(sock, rp->ai_addr, rp->ai_addrlen, timeout,
timer_id, userdata, callback) == pcmk_rc_ok) {
goto async_cleanup; /* Success for now, we'll hear back later in the callback */
}
} else if (connect_socket_once(sock, rp->ai_addr,
rp->ai_addrlen) == pcmk_rc_ok) {
break; /* Success */
}
// Connect failed
close(sock);
sock = -1;
rc = ENOTCONN;
}
async_cleanup:
if (res) {
freeaddrinfo(res);
}
*sock_fd = sock;
return rc;
}
/*!
* \internal
* \brief Convert an IP address (IPv4 or IPv6) to a string for logging
*
* \param[in] sa Socket address for IP
* \param[out] s Storage for at least INET6_ADDRSTRLEN bytes
*
* \note sa The socket address can be a pointer to struct sockaddr_in (IPv4),
* struct sockaddr_in6 (IPv6) or struct sockaddr_storage (either),
* as long as its sa_family member is set correctly.
*/
void
pcmk__sockaddr2str(const void *sa, char *s)
{
switch (((const struct sockaddr *) sa)->sa_family) {
case AF_INET:
inet_ntop(AF_INET, &(((const struct sockaddr_in *) sa)->sin_addr),
s, INET6_ADDRSTRLEN);
break;
case AF_INET6:
inet_ntop(AF_INET6,
&(((const struct sockaddr_in6 *) sa)->sin6_addr),
s, INET6_ADDRSTRLEN);
break;
default:
strcpy(s, "<invalid>");
}
}
/*!
* \internal
* \brief Accept a client connection on a remote server socket
*
* \param[in] ssock Server socket file descriptor being listened on
* \param[out] csock Where to put new client socket's file descriptor
*
* \return Standard Pacemaker return code
*/
int
pcmk__accept_remote_connection(int ssock, int *csock)
{
int rc;
struct sockaddr_storage addr;
socklen_t laddr = sizeof(addr);
char addr_str[INET6_ADDRSTRLEN];
/* accept the connection */
memset(&addr, 0, sizeof(addr));
*csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
if (*csock == -1) {
rc = errno;
crm_err("Could not accept remote client connection: %s "
CRM_XS " rc=%d", pcmk_rc_str(rc), rc);
return rc;
}
pcmk__sockaddr2str(&addr, addr_str);
crm_info("Accepted new remote client connection from %s", addr_str);
rc = pcmk__set_nonblocking(*csock);
if (rc != pcmk_rc_ok) {
crm_err("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
pcmk_rc_str(rc), rc);
close(*csock);
*csock = -1;
return rc;
}
#ifdef TCP_USER_TIMEOUT
if (pcmk__get_sbd_timeout() > 0) {
// Time to fail and retry before watchdog
unsigned int optval = (unsigned int) pcmk__get_sbd_timeout() / 2;
rc = setsockopt(*csock, SOL_TCP, TCP_USER_TIMEOUT,
&optval, sizeof(optval));
if (rc < 0) {
rc = errno;
crm_err("Could not set TCP timeout to %d ms on remote connection: "
"%s " CRM_XS " rc=%d", optval, pcmk_rc_str(rc), rc);
close(*csock);
*csock = -1;
return rc;
}
}
#endif
return rc;
}
/*!
* \brief Get the default remote connection TCP port on this host
*
* \return Remote connection TCP port number
*/
int
crm_default_remote_port(void)
{
static int port = 0;
if (port == 0) {
- const char *env = getenv("PCMK_remote_port");
+ const char *env = pcmk__env_option(PCMK__ENV_REMOTE_PORT);
if (env) {
errno = 0;
port = strtol(env, NULL, 10);
if (errno || (port < 1) || (port > 65535)) {
- crm_warn("Environment variable PCMK_remote_port has invalid value '%s', using %d instead",
+ crm_warn("Environment variable PCMK_" PCMK__ENV_REMOTE_PORT
+ " has invalid value '%s', using %d instead",
env, DEFAULT_REMOTE_PORT);
port = DEFAULT_REMOTE_PORT;
}
} else {
port = DEFAULT_REMOTE_PORT;
}
}
return port;
}
diff --git a/lib/common/tests/options/pcmk__set_env_option_test.c b/lib/common/tests/options/pcmk__set_env_option_test.c
index 753bf74863..22fd795052 100644
--- a/lib/common/tests/options/pcmk__set_env_option_test.c
+++ b/lib/common/tests/options/pcmk__set_env_option_test.c
@@ -1,154 +1,181 @@
/*
- * Copyright 2022 the Pacemaker project contributors
+ * Copyright 2022-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <crm/common/unittest_internal.h>
#include "mock_private.h"
static void
bad_input_string(void **state)
{
// Bad setenv()/unsetenv() input: NULL, empty, or containing '='
// Never call setenv()
pcmk__mock_setenv = true;
- pcmk__set_env_option(NULL, "new_value");
- pcmk__set_env_option("", "new_value");
- pcmk__set_env_option("name=val", "new_value");
+ pcmk__set_env_option(NULL, "new_value", true);
+ pcmk__set_env_option("", "new_value", true);
+ pcmk__set_env_option("name=val", "new_value", true);
pcmk__mock_setenv = false;
// Never call unsetenv()
pcmk__mock_unsetenv = true;
- pcmk__set_env_option(NULL, NULL);
- pcmk__set_env_option("", NULL);
- pcmk__set_env_option("name=val", NULL);
+ pcmk__set_env_option(NULL, NULL, true);
+ pcmk__set_env_option("", NULL, true);
+ pcmk__set_env_option("name=val", NULL, true);
pcmk__mock_unsetenv = false;
}
static void
input_too_long_for_both(void **state)
{
/* pcmk__set_env_option() wants to set "PCMK_<option>" and "HA_<option>". If
* "PCMK_<option>" is too long for the buffer, it simply moves on to
* "HA_<option>". A string of length (NAME_MAX - 3) will set us just over
* the edge for both tries.
*/
char long_opt[NAME_MAX - 2];
for (int i = 0; i < NAME_MAX - 3; i++) {
long_opt[i] = 'a';
}
long_opt[NAME_MAX - 3] = '\0';
// Never call setenv() or unsetenv()
pcmk__mock_setenv = true;
- pcmk__set_env_option(long_opt, "new_value");
+ pcmk__set_env_option(long_opt, "new_value", true);
pcmk__mock_setenv = false;
pcmk__mock_unsetenv = true;
- pcmk__set_env_option(long_opt, NULL);
+ pcmk__set_env_option(long_opt, NULL, true);
pcmk__mock_unsetenv = false;
}
static void
input_too_long_for_pcmk(void **state)
{
/* If an input is too long to set "PCMK_<option>", make sure we fall through
* to try to set "HA_<option>".
*
* A string of length (NAME_MAX - 5) will set us just over the edge for
* "PCMK_<option>", while still short enough for "HA_<option>" to fit.
*/
char long_opt[NAME_MAX - 4];
char buf[NAME_MAX];
for (int i = 0; i < NAME_MAX - 5; i++) {
long_opt[i] = 'a';
}
long_opt[NAME_MAX - 5] = '\0';
snprintf(buf, NAME_MAX, "HA_%s", long_opt);
// Call setenv() for "HA_" only
pcmk__mock_setenv = true;
expect_string(__wrap_setenv, name, buf);
expect_string(__wrap_setenv, value, "new_value");
expect_value(__wrap_setenv, overwrite, 1);
will_return(__wrap_setenv, 0);
- pcmk__set_env_option(long_opt, "new_value");
+ pcmk__set_env_option(long_opt, "new_value", true);
pcmk__mock_setenv = false;
// Call unsetenv() for "HA_" only
pcmk__mock_unsetenv = true;
expect_string(__wrap_unsetenv, name, buf);
will_return(__wrap_unsetenv, 0);
- pcmk__set_env_option(long_opt, NULL);
+ pcmk__set_env_option(long_opt, NULL, true);
pcmk__mock_unsetenv = false;
}
static void
valid_inputs_set(void **state)
{
// Make sure we set "PCMK_<option>" and "HA_<option>"
pcmk__mock_setenv = true;
expect_string(__wrap_setenv, name, "PCMK_env_var");
expect_string(__wrap_setenv, value, "new_value");
expect_value(__wrap_setenv, overwrite, 1);
will_return(__wrap_setenv, 0);
expect_string(__wrap_setenv, name, "HA_env_var");
expect_string(__wrap_setenv, value, "new_value");
expect_value(__wrap_setenv, overwrite, 1);
will_return(__wrap_setenv, 0);
- pcmk__set_env_option("env_var", "new_value");
+ pcmk__set_env_option("env_var", "new_value", true);
// Empty string is also a valid value
expect_string(__wrap_setenv, name, "PCMK_env_var");
expect_string(__wrap_setenv, value, "");
expect_value(__wrap_setenv, overwrite, 1);
will_return(__wrap_setenv, 0);
expect_string(__wrap_setenv, name, "HA_env_var");
expect_string(__wrap_setenv, value, "");
expect_value(__wrap_setenv, overwrite, 1);
will_return(__wrap_setenv, 0);
- pcmk__set_env_option("env_var", "");
+ pcmk__set_env_option("env_var", "", true);
pcmk__mock_setenv = false;
}
static void
valid_inputs_unset(void **state)
{
// Make sure we unset "PCMK_<option>" and "HA_<option>"
pcmk__mock_unsetenv = true;
expect_string(__wrap_unsetenv, name, "PCMK_env_var");
will_return(__wrap_unsetenv, 0);
expect_string(__wrap_unsetenv, name, "HA_env_var");
will_return(__wrap_unsetenv, 0);
- pcmk__set_env_option("env_var", NULL);
+ pcmk__set_env_option("env_var", NULL, true);
+
+ pcmk__mock_unsetenv = false;
+}
+
+static void
+disable_compat(void **state)
+{
+ // Make sure we set only "PCMK_<option>" and not "HA_<option>"
+ pcmk__mock_setenv = true;
+
+ expect_string(__wrap_setenv, name, "PCMK_env_var");
+ expect_string(__wrap_setenv, value, "new_value");
+ expect_value(__wrap_setenv, overwrite, 1);
+ will_return(__wrap_setenv, 0);
+ pcmk__set_env_option("env_var", "new_value", false);
+
+ pcmk__mock_setenv = false;
+
+ // Make sure we clear both "PCMK_<option>" and "HA_<option>"
+ pcmk__mock_unsetenv = true;
+
+ expect_string(__wrap_unsetenv, name, "PCMK_env_var");
+ will_return(__wrap_unsetenv, 0);
+ expect_string(__wrap_unsetenv, name, "HA_env_var");
+ will_return(__wrap_unsetenv, 0);
+ pcmk__set_env_option("env_var", NULL, false);
pcmk__mock_unsetenv = false;
}
PCMK__UNIT_TEST(NULL, NULL,
cmocka_unit_test(bad_input_string),
cmocka_unit_test(input_too_long_for_both),
cmocka_unit_test(input_too_long_for_pcmk),
cmocka_unit_test(valid_inputs_set),
- cmocka_unit_test(valid_inputs_unset))
+ cmocka_unit_test(valid_inputs_unset),
+ cmocka_unit_test(disable_compat))
diff --git a/lib/common/watchdog.c b/lib/common/watchdog.c
index 6c67555c53..e5692140ad 100644
--- a/lib/common/watchdog.c
+++ b/lib/common/watchdog.c
@@ -1,307 +1,310 @@
/*
* Copyright 2013-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <sched.h>
#include <sys/ioctl.h>
#include <sys/reboot.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <ctype.h>
#include <dirent.h>
#include <signal.h>
static pid_t sbd_pid = 0;
static void
sysrq_trigger(char t)
{
#if HAVE_LINUX_PROCFS
FILE *procf;
// Root can always write here, regardless of kernel.sysrq value
procf = fopen("/proc/sysrq-trigger", "a");
if (!procf) {
crm_perror(LOG_WARNING, "Opening sysrq-trigger failed");
return;
}
crm_info("sysrq-trigger: %c", t);
fprintf(procf, "%c\n", t);
fclose(procf);
#endif // HAVE_LINUX_PROCFS
return;
}
/*!
* \internal
* \brief Panic the local host (if root) or tell pacemakerd to do so
*/
static void
panic_local(void)
{
int rc = pcmk_ok;
uid_t uid = geteuid();
pid_t ppid = getppid();
+ const char *panic_action = pcmk__env_option(PCMK__ENV_PANIC_ACTION);
if(uid != 0 && ppid > 1) {
/* We're a non-root pacemaker daemon (pacemaker-based,
* pacemaker-controld, pacemaker-schedulerd, pacemaker-attrd, etc.) with
* the original pacemakerd parent.
*
* Of these, only the controller is likely to be initiating resets.
*/
crm_emerg("Signaling parent %lld to panic", (long long) ppid);
crm_exit(CRM_EX_PANIC);
return;
} else if (uid != 0) {
#if HAVE_LINUX_PROCFS
/*
* No permissions, and no pacemakerd parent to escalate to.
* Track down the new pacemakerd process and send a signal instead.
*/
union sigval signal_value;
memset(&signal_value, 0, sizeof(signal_value));
ppid = pcmk__procfs_pid_of("pacemakerd");
crm_emerg("Signaling pacemakerd[%lld] to panic", (long long) ppid);
if(ppid > 1 && sigqueue(ppid, SIGQUIT, signal_value) < 0) {
crm_perror(LOG_EMERG, "Cannot signal pacemakerd[%lld] to panic",
(long long) ppid);
}
#endif // HAVE_LINUX_PROCFS
/* The best we can do now is die */
crm_exit(CRM_EX_PANIC);
return;
}
/* We're either pacemakerd, or a pacemaker daemon running as root */
- if (pcmk__str_eq("crash", getenv("PCMK_panic_action"), pcmk__str_casei)) {
+ if (pcmk__str_eq(panic_action, "crash", pcmk__str_casei)) {
sysrq_trigger('c');
- } else if (pcmk__str_eq("sync-crash", getenv("PCMK_panic_action"), pcmk__str_casei)) {
+
+ } else if (pcmk__str_eq(panic_action, "sync-crash", pcmk__str_casei)) {
sync();
sysrq_trigger('c');
+
} else {
- if (pcmk__str_eq("sync-reboot", getenv("PCMK_panic_action"), pcmk__str_casei)) {
+ if (pcmk__str_eq(panic_action, "sync-reboot", pcmk__str_casei)) {
sync();
}
sysrq_trigger('b');
}
/* reboot(RB_HALT_SYSTEM); rc = errno; */
reboot(RB_AUTOBOOT);
rc = errno;
crm_emerg("Reboot failed, escalating to parent %lld: %s " CRM_XS " rc=%d",
(long long) ppid, pcmk_rc_str(rc), rc);
if(ppid > 1) {
/* child daemon */
exit(CRM_EX_PANIC);
} else {
/* pacemakerd or orphan child */
exit(CRM_EX_FATAL);
}
}
/*!
* \internal
* \brief Tell sbd to kill the local host, then exit
*/
static void
panic_sbd(void)
{
union sigval signal_value;
pid_t ppid = getppid();
crm_emerg("Signaling sbd[%lld] to panic", (long long) sbd_pid);
memset(&signal_value, 0, sizeof(signal_value));
/* TODO: Arrange for a slightly less brutal option? */
if(sigqueue(sbd_pid, SIGKILL, signal_value) < 0) {
crm_perror(LOG_EMERG, "Cannot signal sbd[%lld] to terminate",
(long long) sbd_pid);
panic_local();
}
if(ppid > 1) {
/* child daemon */
exit(CRM_EX_PANIC);
} else {
/* pacemakerd or orphan child */
exit(CRM_EX_FATAL);
}
}
/*!
* \internal
* \brief Panic the local host
*
* Panic the local host either by sbd (if running), directly, or by asking
* pacemakerd. If trace logging this function, exit instead.
*
* \param[in] origin Function caller (for logging only)
*/
void
pcmk__panic(const char *origin)
{
/* Ensure sbd_pid is set */
(void) pcmk__locate_sbd();
pcmk__if_tracing(
{
// getppid() == 1 means our original parent no longer exists
crm_emerg("Shutting down instead of panicking the node "
CRM_XS " origin=%s sbd=%lld parent=%d",
origin, (long long) sbd_pid, getppid());
crm_exit(CRM_EX_FATAL);
return;
},
{}
);
if(sbd_pid > 1) {
crm_emerg("Signaling sbd[%lld] to panic the system: %s",
(long long) sbd_pid, origin);
panic_sbd();
} else {
crm_emerg("Panicking the system directly: %s", origin);
panic_local();
}
}
/*!
* \internal
* \brief Return the process ID of sbd (or 0 if it is not running)
*/
pid_t
pcmk__locate_sbd(void)
{
char *pidfile = NULL;
char *sbd_path = NULL;
int rc;
if(sbd_pid > 1) {
return sbd_pid;
}
/* Look for the pid file */
pidfile = crm_strdup_printf(PCMK_RUN_DIR "/sbd.pid");
sbd_path = crm_strdup_printf("%s/sbd", SBIN_DIR);
/* Read the pid file */
rc = pcmk__pidfile_matches(pidfile, 0, sbd_path, &sbd_pid);
if (rc == pcmk_rc_ok) {
crm_trace("SBD detected at pid %lld (via PID file %s)",
(long long) sbd_pid, pidfile);
#if HAVE_LINUX_PROCFS
} else {
/* Fall back to /proc for systems that support it */
sbd_pid = pcmk__procfs_pid_of("sbd");
crm_trace("SBD detected at pid %lld (via procfs)",
(long long) sbd_pid);
#endif // HAVE_LINUX_PROCFS
}
if(sbd_pid < 0) {
sbd_pid = 0;
crm_trace("SBD not detected");
}
free(pidfile);
free(sbd_path);
return sbd_pid;
}
long
pcmk__get_sbd_timeout(void)
{
static long sbd_timeout = -2;
if (sbd_timeout == -2) {
sbd_timeout = crm_get_msec(getenv("SBD_WATCHDOG_TIMEOUT"));
}
return sbd_timeout;
}
bool
pcmk__get_sbd_sync_resource_startup(void)
{
static int sync_resource_startup = PCMK__SBD_SYNC_DEFAULT;
static bool checked_sync_resource_startup = false;
if (!checked_sync_resource_startup) {
const char *sync_env = getenv("SBD_SYNC_RESOURCE_STARTUP");
if (sync_env == NULL) {
crm_trace("Defaulting to %sstart-up synchronization with sbd",
(PCMK__SBD_SYNC_DEFAULT? "" : "no "));
} else if (crm_str_to_boolean(sync_env, &sync_resource_startup) < 0) {
crm_warn("Defaulting to %sstart-up synchronization with sbd "
"because environment value '%s' is invalid",
(PCMK__SBD_SYNC_DEFAULT? "" : "no "), sync_env);
}
checked_sync_resource_startup = true;
}
return sync_resource_startup != 0;
}
long
pcmk__auto_watchdog_timeout(void)
{
long sbd_timeout = pcmk__get_sbd_timeout();
return (sbd_timeout <= 0)? 0 : (2 * sbd_timeout);
}
bool
pcmk__valid_sbd_timeout(const char *value)
{
long st_timeout = value? crm_get_msec(value) : 0;
if (st_timeout < 0) {
st_timeout = pcmk__auto_watchdog_timeout();
crm_debug("Using calculated value %ld for stonith-watchdog-timeout (%s)",
st_timeout, value);
}
if (st_timeout == 0) {
crm_debug("Watchdog may be enabled but stonith-watchdog-timeout is disabled (%s)",
value? value : "default");
} else if (pcmk__locate_sbd() == 0) {
crm_emerg("Shutting down: stonith-watchdog-timeout configured (%s) "
"but SBD not active", (value? value : "auto"));
crm_exit(CRM_EX_FATAL);
return false;
} else {
long sbd_timeout = pcmk__get_sbd_timeout();
if (st_timeout < sbd_timeout) {
crm_emerg("Shutting down: stonith-watchdog-timeout (%s) too short "
"(must be >%ldms)", value, sbd_timeout);
crm_exit(CRM_EX_FATAL);
return false;
}
crm_info("Watchdog configured with stonith-watchdog-timeout %s and SBD timeout %ldms",
value, sbd_timeout);
}
return true;
}
diff --git a/lib/common/xml.c b/lib/common/xml.c
index 355cf4cb29..272d09241d 100644
--- a/lib/common/xml.c
+++ b/lib/common/xml.c
@@ -1,2736 +1,2736 @@
/*
* Copyright 2004-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <stdio.h>
#include <sys/types.h>
#include <unistd.h>
#include <time.h>
#include <string.h>
#include <stdlib.h>
#include <stdarg.h>
#include <bzlib.h>
#include <libxml/parser.h>
#include <libxml/tree.h>
#include <libxml/xmlIO.h> /* xmlAllocOutputBuffer */
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <crm/common/xml_internal.h> // PCMK__XML_LOG_BASE, etc.
#include "crmcommon_private.h"
// Define this as 1 in development to get insanely verbose trace messages
#ifndef XML_PARSER_DEBUG
#define XML_PARSER_DEBUG 0
#endif
/* @TODO XML_PARSE_RECOVER allows some XML errors to be silently worked around
* by libxml2, which is potentially ambiguous and dangerous. We should drop it
* when we can break backward compatibility with configurations that might be
* relying on it (i.e. pacemaker 3.0.0).
*
* It might be a good idea to have a transitional period where we first try
* parsing without XML_PARSE_RECOVER, and if that fails, try parsing again with
* it, logging a warning if it succeeds.
*/
#define PCMK__XML_PARSE_OPTS_WITHOUT_RECOVER (XML_PARSE_NOBLANKS)
#define PCMK__XML_PARSE_OPTS_WITH_RECOVER (XML_PARSE_NOBLANKS | XML_PARSE_RECOVER)
bool
pcmk__tracking_xml_changes(xmlNode *xml, bool lazy)
{
if(xml == NULL || xml->doc == NULL || xml->doc->_private == NULL) {
return FALSE;
} else if (!pcmk_is_set(((xml_doc_private_t *)xml->doc->_private)->flags,
pcmk__xf_tracking)) {
return FALSE;
} else if (lazy && !pcmk_is_set(((xml_doc_private_t *)xml->doc->_private)->flags,
pcmk__xf_lazy)) {
return FALSE;
}
return TRUE;
}
static inline void
set_parent_flag(xmlNode *xml, long flag)
{
for(; xml; xml = xml->parent) {
xml_node_private_t *nodepriv = xml->_private;
if (nodepriv == NULL) {
/* During calls to xmlDocCopyNode(), _private will be unset for parent nodes */
} else {
pcmk__set_xml_flags(nodepriv, flag);
}
}
}
void
pcmk__set_xml_doc_flag(xmlNode *xml, enum xml_private_flags flag)
{
if(xml && xml->doc && xml->doc->_private){
/* During calls to xmlDocCopyNode(), xml->doc may be unset */
xml_doc_private_t *docpriv = xml->doc->_private;
pcmk__set_xml_flags(docpriv, flag);
}
}
// Mark document, element, and all element's parents as changed
void
pcmk__mark_xml_node_dirty(xmlNode *xml)
{
pcmk__set_xml_doc_flag(xml, pcmk__xf_dirty);
set_parent_flag(xml, pcmk__xf_dirty);
}
// Clear flags on XML node and its children
static void
reset_xml_node_flags(xmlNode *xml)
{
xmlNode *cIter = NULL;
xml_node_private_t *nodepriv = xml->_private;
if (nodepriv) {
nodepriv->flags = 0;
}
for (cIter = pcmk__xml_first_child(xml); cIter != NULL;
cIter = pcmk__xml_next(cIter)) {
reset_xml_node_flags(cIter);
}
}
// Set xpf_created flag on XML node and any children
void
pcmk__mark_xml_created(xmlNode *xml)
{
xmlNode *cIter = NULL;
xml_node_private_t *nodepriv = NULL;
CRM_ASSERT(xml != NULL);
nodepriv = xml->_private;
if (nodepriv && pcmk__tracking_xml_changes(xml, FALSE)) {
if (!pcmk_is_set(nodepriv->flags, pcmk__xf_created)) {
pcmk__set_xml_flags(nodepriv, pcmk__xf_created);
pcmk__mark_xml_node_dirty(xml);
}
for (cIter = pcmk__xml_first_child(xml); cIter != NULL;
cIter = pcmk__xml_next(cIter)) {
pcmk__mark_xml_created(cIter);
}
}
}
#define XML_DOC_PRIVATE_MAGIC 0x81726354UL
#define XML_NODE_PRIVATE_MAGIC 0x54637281UL
// Free an XML object previously marked as deleted
static void
free_deleted_object(void *data)
{
if(data) {
pcmk__deleted_xml_t *deleted_obj = data;
free(deleted_obj->path);
free(deleted_obj);
}
}
// Free and NULL user, ACLs, and deleted objects in an XML node's private data
static void
reset_xml_private_data(xml_doc_private_t *docpriv)
{
if (docpriv != NULL) {
CRM_ASSERT(docpriv->check == XML_DOC_PRIVATE_MAGIC);
free(docpriv->user);
docpriv->user = NULL;
if (docpriv->acls != NULL) {
pcmk__free_acls(docpriv->acls);
docpriv->acls = NULL;
}
if(docpriv->deleted_objs) {
g_list_free_full(docpriv->deleted_objs, free_deleted_object);
docpriv->deleted_objs = NULL;
}
}
}
// Free all private data associated with an XML node
static void
free_private_data(xmlNode *node)
{
/* Note:
This function frees private data assosciated with an XML node,
unless the function is being called as a result of internal
XSLT cleanup.
That could happen through, for example, the following chain of
function calls:
xsltApplyStylesheetInternal
-> xsltFreeTransformContext
-> xsltFreeRVTs
-> xmlFreeDoc
And in that case, the node would fulfill three conditions:
1. It would be a standalone document (i.e. it wouldn't be
part of a document)
2. It would have a space-prefixed name (for reference, please
see xsltInternals.h: XSLT_MARK_RES_TREE_FRAG)
3. It would carry its own payload in the _private field.
We do not free data in this circumstance to avoid a failed
assertion on the XML_*_PRIVATE_MAGIC later.
*/
if (node->name == NULL || node->name[0] != ' ') {
if (node->_private) {
if (node->type == XML_DOCUMENT_NODE) {
reset_xml_private_data(node->_private);
} else {
CRM_ASSERT(((xml_node_private_t *) node->_private)->check
== XML_NODE_PRIVATE_MAGIC);
/* nothing dynamically allocated nested */
}
free(node->_private);
node->_private = NULL;
}
}
}
// Allocate and initialize private data for an XML node
static void
new_private_data(xmlNode *node)
{
switch (node->type) {
case XML_DOCUMENT_NODE: {
xml_doc_private_t *docpriv = NULL;
docpriv = calloc(1, sizeof(xml_doc_private_t));
CRM_ASSERT(docpriv != NULL);
docpriv->check = XML_DOC_PRIVATE_MAGIC;
/* Flags will be reset if necessary when tracking is enabled */
pcmk__set_xml_flags(docpriv, pcmk__xf_dirty|pcmk__xf_created);
node->_private = docpriv;
break;
}
case XML_ELEMENT_NODE:
case XML_ATTRIBUTE_NODE:
case XML_COMMENT_NODE: {
xml_node_private_t *nodepriv = NULL;
nodepriv = calloc(1, sizeof(xml_node_private_t));
CRM_ASSERT(nodepriv != NULL);
nodepriv->check = XML_NODE_PRIVATE_MAGIC;
/* Flags will be reset if necessary when tracking is enabled */
pcmk__set_xml_flags(nodepriv, pcmk__xf_dirty|pcmk__xf_created);
node->_private = nodepriv;
if (pcmk__tracking_xml_changes(node, FALSE)) {
/* XML_ELEMENT_NODE doesn't get picked up here, node->doc is
* not hooked up at the point we are called
*/
pcmk__mark_xml_node_dirty(node);
}
break;
}
case XML_TEXT_NODE:
case XML_DTD_NODE:
case XML_CDATA_SECTION_NODE:
break;
default:
/* Ignore */
crm_trace("Ignoring %p %d", node, node->type);
CRM_LOG_ASSERT(node->type == XML_ELEMENT_NODE);
break;
}
}
void
xml_track_changes(xmlNode * xml, const char *user, xmlNode *acl_source, bool enforce_acls)
{
xml_accept_changes(xml);
crm_trace("Tracking changes%s to %p", enforce_acls?" with ACLs":"", xml);
pcmk__set_xml_doc_flag(xml, pcmk__xf_tracking);
if(enforce_acls) {
if(acl_source == NULL) {
acl_source = xml;
}
pcmk__set_xml_doc_flag(xml, pcmk__xf_acl_enabled);
pcmk__unpack_acl(acl_source, xml, user);
pcmk__apply_acl(xml);
}
}
bool xml_tracking_changes(xmlNode * xml)
{
return (xml != NULL) && (xml->doc != NULL) && (xml->doc->_private != NULL)
&& pcmk_is_set(((xml_doc_private_t *)(xml->doc->_private))->flags,
pcmk__xf_tracking);
}
bool xml_document_dirty(xmlNode *xml)
{
return (xml != NULL) && (xml->doc != NULL) && (xml->doc->_private != NULL)
&& pcmk_is_set(((xml_doc_private_t *)(xml->doc->_private))->flags,
pcmk__xf_dirty);
}
/*!
* \internal
* \brief Return ordinal position of an XML node among its siblings
*
* \param[in] xml XML node to check
* \param[in] ignore_if_set Don't count siblings with this flag set
*
* \return Ordinal position of \p xml (starting with 0)
*/
int
pcmk__xml_position(const xmlNode *xml, enum xml_private_flags ignore_if_set)
{
int position = 0;
for (const xmlNode *cIter = xml; cIter->prev; cIter = cIter->prev) {
xml_node_private_t *nodepriv = ((xmlNode*)cIter->prev)->_private;
if (!pcmk_is_set(nodepriv->flags, ignore_if_set)) {
position++;
}
}
return position;
}
// Remove all attributes marked as deleted from an XML node
static void
accept_attr_deletions(xmlNode *xml)
{
// Clear XML node's flags
((xml_node_private_t *) xml->_private)->flags = pcmk__xf_none;
// Remove this XML node's attributes that were marked as deleted
pcmk__xe_remove_matching_attrs(xml, pcmk__marked_as_deleted, NULL);
// Recursively do the same for this XML node's children
for (xmlNodePtr cIter = pcmk__xml_first_child(xml); cIter != NULL;
cIter = pcmk__xml_next(cIter)) {
accept_attr_deletions(cIter);
}
}
/*!
* \internal
* \brief Find first child XML node matching another given XML node
*
* \param[in] haystack XML whose children should be checked
* \param[in] needle XML to match (comment content or element name and ID)
* \param[in] exact If true and needle is a comment, position must match
*/
xmlNode *
pcmk__xml_match(const xmlNode *haystack, const xmlNode *needle, bool exact)
{
CRM_CHECK(needle != NULL, return NULL);
if (needle->type == XML_COMMENT_NODE) {
return pcmk__xc_match(haystack, needle, exact);
} else {
const char *id = ID(needle);
const char *attr = (id == NULL)? NULL : XML_ATTR_ID;
return pcmk__xe_match(haystack, (const char *) needle->name, attr, id);
}
}
void
xml_accept_changes(xmlNode * xml)
{
xmlNode *top = NULL;
xml_doc_private_t *docpriv = NULL;
if(xml == NULL) {
return;
}
crm_trace("Accepting changes to %p", xml);
docpriv = xml->doc->_private;
top = xmlDocGetRootElement(xml->doc);
reset_xml_private_data(xml->doc->_private);
if (!pcmk_is_set(docpriv->flags, pcmk__xf_dirty)) {
docpriv->flags = pcmk__xf_none;
return;
}
docpriv->flags = pcmk__xf_none;
accept_attr_deletions(top);
}
xmlNode *
find_xml_node(const xmlNode *root, const char *search_path, gboolean must_find)
{
xmlNode *a_child = NULL;
const char *name = (root == NULL)? "<NULL>" : (const char *) root->name;
if (search_path == NULL) {
crm_warn("Will never find <NULL>");
return NULL;
}
for (a_child = pcmk__xml_first_child(root); a_child != NULL;
a_child = pcmk__xml_next(a_child)) {
if (strcmp((const char *)a_child->name, search_path) == 0) {
return a_child;
}
}
if (must_find) {
crm_warn("Could not find %s in %s.", search_path, name);
} else if (root != NULL) {
crm_trace("Could not find %s in %s.", search_path, name);
} else {
crm_trace("Could not find %s in <NULL>.", search_path);
}
return NULL;
}
#define attr_matches(c, n, v) pcmk__str_eq(crm_element_value((c), (n)), \
(v), pcmk__str_none)
/*!
* \internal
* \brief Find first XML child element matching given criteria
*
* \param[in] parent XML element to search
* \param[in] node_name If not NULL, only match children of this type
* \param[in] attr_n If not NULL, only match children with an attribute
* of this name.
* \param[in] attr_v If \p attr_n and this are not NULL, only match children
* with an attribute named \p attr_n and this value
*
* \return Matching XML child element, or NULL if none found
*/
xmlNode *
pcmk__xe_match(const xmlNode *parent, const char *node_name,
const char *attr_n, const char *attr_v)
{
CRM_CHECK(parent != NULL, return NULL);
CRM_CHECK(attr_v == NULL || attr_n != NULL, return NULL);
for (xmlNode *child = pcmk__xml_first_child(parent); child != NULL;
child = pcmk__xml_next(child)) {
if (pcmk__str_eq(node_name, (const char *) (child->name),
pcmk__str_null_matches)
&& ((attr_n == NULL) ||
(attr_v == NULL && xmlHasProp(child, (pcmkXmlStr) attr_n)) ||
(attr_v != NULL && attr_matches(child, attr_n, attr_v)))) {
return child;
}
}
crm_trace("XML child node <%s%s%s%s%s> not found in %s",
(node_name? node_name : "(any)"),
(attr_n? " " : ""),
(attr_n? attr_n : ""),
(attr_n? "=" : ""),
(attr_n? attr_v : ""),
(const char *) parent->name);
return NULL;
}
void
copy_in_properties(xmlNode *target, const xmlNode *src)
{
if (src == NULL) {
crm_warn("No node to copy properties from");
} else if (target == NULL) {
crm_err("No node to copy properties into");
} else {
for (xmlAttrPtr a = pcmk__xe_first_attr(src); a != NULL; a = a->next) {
const char *p_name = (const char *) a->name;
const char *p_value = pcmk__xml_attr_value(a);
expand_plus_plus(target, p_name, p_value);
if (xml_acl_denied(target)) {
crm_trace("Cannot copy %s=%s to %s", p_name, p_value, target->name);
return;
}
}
}
return;
}
/*!
* \brief Parse integer assignment statements on this node and all its child
* nodes
*
* \param[in,out] target Root XML node to be processed
*
* \note This function is recursive
*/
void
fix_plus_plus_recursive(xmlNode *target)
{
/* TODO: Remove recursion and use xpath searches for value++ */
xmlNode *child = NULL;
for (xmlAttrPtr a = pcmk__xe_first_attr(target); a != NULL; a = a->next) {
const char *p_name = (const char *) a->name;
const char *p_value = pcmk__xml_attr_value(a);
expand_plus_plus(target, p_name, p_value);
}
for (child = pcmk__xml_first_child(target); child != NULL;
child = pcmk__xml_next(child)) {
fix_plus_plus_recursive(child);
}
}
/*!
* \brief Update current XML attribute value per parsed integer assignment
statement
*
* \param[in,out] target an XML node, containing a XML attribute that is
* initialized to some numeric value, to be processed
* \param[in] name name of the XML attribute, e.g. X, whose value
* should be updated
* \param[in] value assignment statement, e.g. "X++" or
* "X+=5", to be applied to the initialized value.
*
* \note The original XML attribute value is treated as 0 if non-numeric and
* truncated to be an integer if decimal-point-containing.
* \note The final XML attribute value is truncated to not exceed 1000000.
* \note Undefined behavior if unexpected input.
*/
void
expand_plus_plus(xmlNode * target, const char *name, const char *value)
{
int offset = 1;
int name_len = 0;
int int_value = 0;
int value_len = 0;
const char *old_value = NULL;
if (target == NULL || value == NULL || name == NULL) {
return;
}
old_value = crm_element_value(target, name);
if (old_value == NULL) {
/* if no previous value, set unexpanded */
goto set_unexpanded;
} else if (strstr(value, name) != value) {
goto set_unexpanded;
}
name_len = strlen(name);
value_len = strlen(value);
if (value_len < (name_len + 2)
|| value[name_len] != '+' || (value[name_len + 1] != '+' && value[name_len + 1] != '=')) {
goto set_unexpanded;
}
/* if we are expanding ourselves,
* then no previous value was set and leave int_value as 0
*/
if (old_value != value) {
int_value = char2score(old_value);
}
if (value[name_len + 1] != '+') {
const char *offset_s = value + (name_len + 2);
offset = char2score(offset_s);
}
int_value += offset;
if (int_value > INFINITY) {
int_value = (int)INFINITY;
}
crm_xml_add_int(target, name, int_value);
return;
set_unexpanded:
if (old_value == value) {
/* the old value is already set, nothing to do */
return;
}
crm_xml_add(target, name, value);
return;
}
/*!
* \internal
* \brief Remove an XML element's attributes that match some criteria
*
* \param[in,out] element XML element to modify
* \param[in] match If not NULL, only remove attributes for which
* this function returns true
* \param[in,out] user_data Data to pass to \p match
*/
void
pcmk__xe_remove_matching_attrs(xmlNode *element,
bool (*match)(xmlAttrPtr, void *),
void *user_data)
{
xmlAttrPtr next = NULL;
for (xmlAttrPtr a = pcmk__xe_first_attr(element); a != NULL; a = next) {
next = a->next; // Grab now because attribute might get removed
if ((match == NULL) || match(a, user_data)) {
if (!pcmk__check_acl(element, NULL, pcmk__xf_acl_write)) {
crm_trace("ACLs prevent removal of attributes (%s and "
"possibly others) from %s element",
(const char *) a->name, (const char *) element->name);
return; // ACLs apply to element, not particular attributes
}
if (pcmk__tracking_xml_changes(element, false)) {
// Leave (marked for removal) until after diff is calculated
set_parent_flag(element, pcmk__xf_dirty);
pcmk__set_xml_flags((xml_node_private_t *) a->_private,
pcmk__xf_deleted);
} else {
xmlRemoveProp(a);
}
}
}
}
xmlNode *
add_node_copy(xmlNode * parent, xmlNode * src_node)
{
xmlNode *child = NULL;
CRM_CHECK((parent != NULL) && (src_node != NULL), return NULL);
child = xmlDocCopyNode(src_node, parent->doc, 1);
if (child == NULL) {
return NULL;
}
xmlAddChild(parent, child);
pcmk__mark_xml_created(child);
return child;
}
xmlNode *
create_xml_node(xmlNode * parent, const char *name)
{
xmlDoc *doc = NULL;
xmlNode *node = NULL;
if (pcmk__str_empty(name)) {
CRM_CHECK(name != NULL && name[0] == 0, return NULL);
return NULL;
}
if (parent == NULL) {
doc = xmlNewDoc((pcmkXmlStr) "1.0");
if (doc == NULL) {
return NULL;
}
node = xmlNewDocRawNode(doc, NULL, (pcmkXmlStr) name, NULL);
if (node == NULL) {
xmlFreeDoc(doc);
return NULL;
}
xmlDocSetRootElement(doc, node);
} else {
node = xmlNewChild(parent, NULL, (pcmkXmlStr) name, NULL);
if (node == NULL) {
return NULL;
}
}
pcmk__mark_xml_created(node);
return node;
}
xmlNode *
pcmk_create_xml_text_node(xmlNode * parent, const char *name, const char *content)
{
xmlNode *node = create_xml_node(parent, name);
if (node != NULL) {
xmlNodeSetContent(node, (pcmkXmlStr) content);
}
return node;
}
xmlNode *
pcmk_create_html_node(xmlNode * parent, const char *element_name, const char *id,
const char *class_name, const char *text)
{
xmlNode *node = pcmk_create_xml_text_node(parent, element_name, text);
if (class_name != NULL) {
crm_xml_add(node, "class", class_name);
}
if (id != NULL) {
crm_xml_add(node, "id", id);
}
return node;
}
/*!
* Free an XML element and all of its children, removing it from its parent
*
* \param[in,out] xml XML element to free
*/
void
pcmk_free_xml_subtree(xmlNode *xml)
{
xmlUnlinkNode(xml); // Detaches from parent and siblings
xmlFreeNode(xml); // Frees
}
static void
free_xml_with_position(xmlNode * child, int position)
{
if (child != NULL) {
xmlNode *top = NULL;
xmlDoc *doc = child->doc;
xml_node_private_t *nodepriv = child->_private;
xml_doc_private_t *docpriv = NULL;
if (doc != NULL) {
top = xmlDocGetRootElement(doc);
}
if (doc != NULL && top == child) {
/* Free everything */
xmlFreeDoc(doc);
} else if (pcmk__check_acl(child, NULL, pcmk__xf_acl_write) == FALSE) {
GString *xpath = NULL;
pcmk__if_tracing({}, return);
xpath = pcmk__element_xpath(child);
qb_log_from_external_source(__func__, __FILE__,
"Cannot remove %s %x", LOG_TRACE,
__LINE__, 0, (const char *) xpath->str,
nodepriv->flags);
g_string_free(xpath, TRUE);
return;
} else {
if (doc && pcmk__tracking_xml_changes(child, FALSE)
&& !pcmk_is_set(nodepriv->flags, pcmk__xf_created)) {
GString *xpath = pcmk__element_xpath(child);
if (xpath != NULL) {
pcmk__deleted_xml_t *deleted_obj = NULL;
crm_trace("Deleting %s %p from %p",
(const char *) xpath->str, child, doc);
deleted_obj = calloc(1, sizeof(pcmk__deleted_xml_t));
deleted_obj->path = strdup((const char *) xpath->str);
CRM_ASSERT(deleted_obj->path != NULL);
g_string_free(xpath, TRUE);
deleted_obj->position = -1;
/* Record the "position" only for XML comments for now */
if (child->type == XML_COMMENT_NODE) {
if (position >= 0) {
deleted_obj->position = position;
} else {
deleted_obj->position = pcmk__xml_position(child,
pcmk__xf_skip);
}
}
docpriv = doc->_private;
docpriv->deleted_objs = g_list_append(docpriv->deleted_objs, deleted_obj);
pcmk__set_xml_doc_flag(child, pcmk__xf_dirty);
}
}
pcmk_free_xml_subtree(child);
}
}
}
void
free_xml(xmlNode * child)
{
free_xml_with_position(child, -1);
}
xmlNode *
copy_xml(xmlNode * src)
{
xmlDoc *doc = xmlNewDoc((pcmkXmlStr) "1.0");
xmlNode *copy = xmlDocCopyNode(src, doc, 1);
CRM_ASSERT(copy != NULL);
xmlDocSetRootElement(doc, copy);
return copy;
}
xmlNode *
string2xml(const char *input)
{
xmlNode *xml = NULL;
xmlDocPtr output = NULL;
xmlParserCtxtPtr ctxt = NULL;
xmlErrorPtr last_error = NULL;
if (input == NULL) {
crm_err("Can't parse NULL input");
return NULL;
}
/* create a parser context */
ctxt = xmlNewParserCtxt();
CRM_CHECK(ctxt != NULL, return NULL);
xmlCtxtResetLastError(ctxt);
xmlSetGenericErrorFunc(ctxt, pcmk__log_xmllib_err);
output = xmlCtxtReadDoc(ctxt, (pcmkXmlStr) input, NULL, NULL,
PCMK__XML_PARSE_OPTS_WITHOUT_RECOVER);
if (output == NULL) {
output = xmlCtxtReadDoc(ctxt, (pcmkXmlStr) input, NULL, NULL,
PCMK__XML_PARSE_OPTS_WITH_RECOVER);
if (output) {
crm_warn("Successfully recovered from XML errors "
"(note: a future release will treat this as a fatal failure)");
}
}
if (output) {
xml = xmlDocGetRootElement(output);
}
last_error = xmlCtxtGetLastError(ctxt);
if (last_error && last_error->code != XML_ERR_OK) {
/* crm_abort(__FILE__,__func__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
/*
* http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
* http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
*/
crm_warn("Parsing failed (domain=%d, level=%d, code=%d): %s",
last_error->domain, last_error->level, last_error->code, last_error->message);
if (last_error->code == XML_ERR_DOCUMENT_EMPTY) {
CRM_LOG_ASSERT("Cannot parse an empty string");
} else if (last_error->code != XML_ERR_DOCUMENT_END) {
crm_err("Couldn't%s parse %d chars: %s", xml ? " fully" : "", (int)strlen(input),
input);
if (xml != NULL) {
crm_log_xml_err(xml, "Partial");
}
} else {
int len = strlen(input);
int lpc = 0;
while(lpc < len) {
crm_warn("Parse error[+%.3d]: %.80s", lpc, input+lpc);
lpc += 80;
}
CRM_LOG_ASSERT("String parsing error");
}
}
xmlFreeParserCtxt(ctxt);
return xml;
}
xmlNode *
stdin2xml(void)
{
size_t data_length = 0;
size_t read_chars = 0;
char *xml_buffer = NULL;
xmlNode *xml_obj = NULL;
do {
xml_buffer = pcmk__realloc(xml_buffer, data_length + PCMK__BUFFER_SIZE);
read_chars = fread(xml_buffer + data_length, 1, PCMK__BUFFER_SIZE,
stdin);
data_length += read_chars;
} while (read_chars == PCMK__BUFFER_SIZE);
if (data_length == 0) {
crm_warn("No XML supplied on stdin");
free(xml_buffer);
return NULL;
}
xml_buffer[data_length] = '\0';
xml_obj = string2xml(xml_buffer);
free(xml_buffer);
crm_log_xml_trace(xml_obj, "Created fragment");
return xml_obj;
}
static char *
decompress_file(const char *filename)
{
char *buffer = NULL;
int rc = 0;
size_t length = 0, read_len = 0;
BZFILE *bz_file = NULL;
FILE *input = fopen(filename, "r");
if (input == NULL) {
crm_perror(LOG_ERR, "Could not open %s for reading", filename);
return NULL;
}
bz_file = BZ2_bzReadOpen(&rc, input, 0, 0, NULL, 0);
rc = pcmk__bzlib2rc(rc);
if (rc != pcmk_rc_ok) {
crm_err("Could not prepare to read compressed %s: %s "
CRM_XS " rc=%d", filename, pcmk_rc_str(rc), rc);
BZ2_bzReadClose(&rc, bz_file);
fclose(input);
return NULL;
}
rc = BZ_OK;
// cppcheck seems not to understand the abort-logic in pcmk__realloc
// cppcheck-suppress memleak
while (rc == BZ_OK) {
buffer = pcmk__realloc(buffer, PCMK__BUFFER_SIZE + length + 1);
read_len = BZ2_bzRead(&rc, bz_file, buffer + length, PCMK__BUFFER_SIZE);
crm_trace("Read %ld bytes from file: %d", (long)read_len, rc);
if (rc == BZ_OK || rc == BZ_STREAM_END) {
length += read_len;
}
}
buffer[length] = '\0';
rc = pcmk__bzlib2rc(rc);
if (rc != pcmk_rc_ok) {
crm_err("Could not read compressed %s: %s " CRM_XS " rc=%d",
filename, pcmk_rc_str(rc), rc);
free(buffer);
buffer = NULL;
}
BZ2_bzReadClose(&rc, bz_file);
fclose(input);
return buffer;
}
/*!
* \internal
* \brief Remove XML text nodes from specified XML and all its children
*
* \param[in,out] xml XML to strip text from
*/
void
pcmk__strip_xml_text(xmlNode *xml)
{
xmlNode *iter = xml->children;
while (iter) {
xmlNode *next = iter->next;
switch (iter->type) {
case XML_TEXT_NODE:
/* Remove it */
pcmk_free_xml_subtree(iter);
break;
case XML_ELEMENT_NODE:
/* Search it */
pcmk__strip_xml_text(iter);
break;
default:
/* Leave it */
break;
}
iter = next;
}
}
xmlNode *
filename2xml(const char *filename)
{
xmlNode *xml = NULL;
xmlDocPtr output = NULL;
bool uncompressed = true;
xmlParserCtxtPtr ctxt = NULL;
xmlErrorPtr last_error = NULL;
/* create a parser context */
ctxt = xmlNewParserCtxt();
CRM_CHECK(ctxt != NULL, return NULL);
xmlCtxtResetLastError(ctxt);
xmlSetGenericErrorFunc(ctxt, pcmk__log_xmllib_err);
if (filename) {
uncompressed = !pcmk__ends_with_ext(filename, ".bz2");
}
if (pcmk__str_eq(filename, "-", pcmk__str_null_matches)) {
/* STDIN_FILENO == fileno(stdin) */
output = xmlCtxtReadFd(ctxt, STDIN_FILENO, "unknown.xml", NULL,
PCMK__XML_PARSE_OPTS_WITHOUT_RECOVER);
if (output == NULL) {
output = xmlCtxtReadFd(ctxt, STDIN_FILENO, "unknown.xml", NULL,
PCMK__XML_PARSE_OPTS_WITH_RECOVER);
if (output) {
crm_warn("Successfully recovered from XML errors "
"(note: a future release will treat this as a fatal failure)");
}
}
} else if (uncompressed) {
output = xmlCtxtReadFile(ctxt, filename, NULL,
PCMK__XML_PARSE_OPTS_WITHOUT_RECOVER);
if (output == NULL) {
output = xmlCtxtReadFile(ctxt, filename, NULL,
PCMK__XML_PARSE_OPTS_WITH_RECOVER);
if (output) {
crm_warn("Successfully recovered from XML errors "
"(note: a future release will treat this as a fatal failure)");
}
}
} else {
char *input = decompress_file(filename);
output = xmlCtxtReadDoc(ctxt, (pcmkXmlStr) input, NULL, NULL,
PCMK__XML_PARSE_OPTS_WITHOUT_RECOVER);
if (output == NULL) {
output = xmlCtxtReadDoc(ctxt, (pcmkXmlStr) input, NULL, NULL,
PCMK__XML_PARSE_OPTS_WITH_RECOVER);
if (output) {
crm_warn("Successfully recovered from XML errors "
"(note: a future release will treat this as a fatal failure)");
}
}
free(input);
}
if (output && (xml = xmlDocGetRootElement(output))) {
pcmk__strip_xml_text(xml);
}
last_error = xmlCtxtGetLastError(ctxt);
if (last_error && last_error->code != XML_ERR_OK) {
/* crm_abort(__FILE__,__func__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
/*
* http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
* http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
*/
crm_err("Parsing failed (domain=%d, level=%d, code=%d): %s",
last_error->domain, last_error->level, last_error->code, last_error->message);
if (last_error && last_error->code != XML_ERR_OK) {
crm_err("Couldn't%s parse %s", xml ? " fully" : "", filename);
if (xml != NULL) {
crm_log_xml_err(xml, "Partial");
}
}
}
xmlFreeParserCtxt(ctxt);
return xml;
}
/*!
* \internal
* \brief Add a "last written" attribute to an XML element, set to current time
*
* \param[in,out] xe XML element to add attribute to
*
* \return Value that was set, or NULL on error
*/
const char *
pcmk__xe_add_last_written(xmlNode *xe)
{
char *now_s = pcmk__epoch2str(NULL, 0);
const char *result = NULL;
result = crm_xml_add(xe, XML_CIB_ATTR_WRITTEN,
pcmk__s(now_s, "Could not determine current time"));
free(now_s);
return result;
}
/*!
* \brief Sanitize a string so it is usable as an XML ID
*
* \param[in,out] id String to sanitize
*/
void
crm_xml_sanitize_id(char *id)
{
char *c;
for (c = id; *c; ++c) {
/* @TODO Sanitize more comprehensively */
switch (*c) {
case ':':
case '#':
*c = '.';
}
}
}
/*!
* \brief Set the ID of an XML element using a format
*
* \param[in,out] xml XML element
* \param[in] fmt printf-style format
* \param[in] ... any arguments required by format
*/
void
crm_xml_set_id(xmlNode *xml, const char *format, ...)
{
va_list ap;
int len = 0;
char *id = NULL;
/* equivalent to crm_strdup_printf() */
va_start(ap, format);
len = vasprintf(&id, format, ap);
va_end(ap);
CRM_ASSERT(len > 0);
crm_xml_sanitize_id(id);
crm_xml_add(xml, XML_ATTR_ID, id);
free(id);
}
/*!
* \internal
* \brief Write XML to a file stream
*
* \param[in] xml XML to write
* \param[in] filename Name of file being written (for logging only)
* \param[in,out] stream Open file stream corresponding to filename
* \param[in] compress Whether to compress XML before writing
* \param[out] nbytes Number of bytes written
*
* \return Standard Pacemaker return code
*/
static int
write_xml_stream(const xmlNode *xml, const char *filename, FILE *stream,
bool compress, unsigned int *nbytes)
{
int rc = pcmk_rc_ok;
char *buffer = NULL;
*nbytes = 0;
crm_log_xml_trace(xml, "writing");
buffer = dump_xml_formatted(xml);
CRM_CHECK(buffer && strlen(buffer),
crm_log_xml_warn(xml, "formatting failed");
rc = pcmk_rc_error;
goto bail);
if (compress) {
unsigned int in = 0;
BZFILE *bz_file = NULL;
rc = BZ_OK;
bz_file = BZ2_bzWriteOpen(&rc, stream, 5, 0, 30);
rc = pcmk__bzlib2rc(rc);
if (rc != pcmk_rc_ok) {
crm_warn("Not compressing %s: could not prepare file stream: %s "
CRM_XS " rc=%d", filename, pcmk_rc_str(rc), rc);
} else {
BZ2_bzWrite(&rc, bz_file, buffer, strlen(buffer));
rc = pcmk__bzlib2rc(rc);
if (rc != pcmk_rc_ok) {
crm_warn("Not compressing %s: could not compress data: %s "
CRM_XS " rc=%d errno=%d",
filename, pcmk_rc_str(rc), rc, errno);
}
}
if (rc == pcmk_rc_ok) {
BZ2_bzWriteClose(&rc, bz_file, 0, &in, nbytes);
rc = pcmk__bzlib2rc(rc);
if (rc != pcmk_rc_ok) {
crm_warn("Not compressing %s: could not write compressed data: %s "
CRM_XS " rc=%d errno=%d",
filename, pcmk_rc_str(rc), rc, errno);
*nbytes = 0; // retry without compression
} else {
crm_trace("Compressed XML for %s from %u bytes to %u",
filename, in, *nbytes);
}
}
rc = pcmk_rc_ok; // Either true, or we'll retry without compression
}
if (*nbytes == 0) {
rc = fprintf(stream, "%s", buffer);
if (rc < 0) {
rc = errno;
crm_perror(LOG_ERR, "writing %s", filename);
} else {
*nbytes = (unsigned int) rc;
rc = pcmk_rc_ok;
}
}
bail:
if (fflush(stream) != 0) {
rc = errno;
crm_perror(LOG_ERR, "flushing %s", filename);
}
/* Don't report error if the file does not support synchronization */
if (fsync(fileno(stream)) < 0 && errno != EROFS && errno != EINVAL) {
rc = errno;
crm_perror(LOG_ERR, "synchronizing %s", filename);
}
fclose(stream);
crm_trace("Saved %d bytes to %s as XML", *nbytes, filename);
free(buffer);
return rc;
}
/*!
* \brief Write XML to a file descriptor
*
* \param[in] xml XML to write
* \param[in] filename Name of file being written (for logging only)
* \param[in] fd Open file descriptor corresponding to filename
* \param[in] compress Whether to compress XML before writing
*
* \return Number of bytes written on success, -errno otherwise
*/
int
write_xml_fd(const xmlNode *xml, const char *filename, int fd,
gboolean compress)
{
FILE *stream = NULL;
unsigned int nbytes = 0;
int rc = pcmk_rc_ok;
CRM_CHECK((xml != NULL) && (fd > 0), return -EINVAL);
stream = fdopen(fd, "w");
if (stream == NULL) {
return -errno;
}
rc = write_xml_stream(xml, filename, stream, compress, &nbytes);
if (rc != pcmk_rc_ok) {
return pcmk_rc2legacy(rc);
}
return (int) nbytes;
}
/*!
* \brief Write XML to a file
*
* \param[in] xml XML to write
* \param[in] filename Name of file to write
* \param[in] compress Whether to compress XML before writing
*
* \return Number of bytes written on success, -errno otherwise
*/
int
write_xml_file(const xmlNode *xml, const char *filename, gboolean compress)
{
FILE *stream = NULL;
unsigned int nbytes = 0;
int rc = pcmk_rc_ok;
CRM_CHECK((xml != NULL) && (filename != NULL), return -EINVAL);
stream = fopen(filename, "w");
if (stream == NULL) {
return -errno;
}
rc = write_xml_stream(xml, filename, stream, compress, &nbytes);
if (rc != pcmk_rc_ok) {
return pcmk_rc2legacy(rc);
}
return (int) nbytes;
}
// Replace a portion of a dynamically allocated string (reallocating memory)
static char *
replace_text(char *text, int start, size_t *length, const char *replace)
{
size_t offset = strlen(replace) - 1; // We have space for 1 char already
*length += offset;
text = pcmk__realloc(text, *length);
for (size_t lpc = (*length) - 1; lpc > (start + offset); lpc--) {
text[lpc] = text[lpc - offset];
}
memcpy(text + start, replace, offset + 1);
return text;
}
/*!
* \brief Replace special characters with their XML escape sequences
*
* \param[in] text Text to escape
*
* \return Newly allocated string equivalent to \p text but with special
* characters replaced with XML escape sequences (or NULL if \p text
* is NULL)
*/
char *
crm_xml_escape(const char *text)
{
size_t length;
char *copy;
/*
* When xmlCtxtReadDoc() parses &lt; and friends in a
* value, it converts them to their human readable
* form.
*
* If one uses xmlNodeDump() to convert it back to a
* string, all is well, because special characters are
* converted back to their escape sequences.
*
* However xmlNodeDump() is randomly dog slow, even with the same
* input. So we need to replicate the escaping in our custom
* version so that the result can be re-parsed by xmlCtxtReadDoc()
* when necessary.
*/
if (text == NULL) {
return NULL;
}
length = 1 + strlen(text);
copy = strdup(text);
CRM_ASSERT(copy != NULL);
for (size_t index = 0; index < length; index++) {
if(copy[index] & 0x80 && copy[index+1] & 0x80){
index++;
break;
}
switch (copy[index]) {
case 0:
break;
case '<':
copy = replace_text(copy, index, &length, "&lt;");
break;
case '>':
copy = replace_text(copy, index, &length, "&gt;");
break;
case '"':
copy = replace_text(copy, index, &length, "&quot;");
break;
case '\'':
copy = replace_text(copy, index, &length, "&apos;");
break;
case '&':
copy = replace_text(copy, index, &length, "&amp;");
break;
case '\t':
/* Might as well just expand to a few spaces... */
copy = replace_text(copy, index, &length, " ");
break;
case '\n':
copy = replace_text(copy, index, &length, "\\n");
break;
case '\r':
copy = replace_text(copy, index, &length, "\\r");
break;
default:
/* Check for and replace non-printing characters with their octal equivalent */
if(copy[index] < ' ' || copy[index] > '~') {
char *replace = crm_strdup_printf("\\%.3o", copy[index]);
copy = replace_text(copy, index, &length, replace);
free(replace);
}
}
}
return copy;
}
/*!
* \internal
* \brief Append a string representation of an XML element to a buffer
*
* \param[in] data XML whose representation to append
* \param[in] options Group of \p pcmk__xml_fmt_options flags
* \param[in,out] buffer Where to append the content (must not be \p NULL)
* \param[in] depth Current indentation level
*/
static void
dump_xml_element(const xmlNode *data, uint32_t options, GString *buffer,
int depth)
{
bool pretty = pcmk_is_set(options, pcmk__xml_fmt_pretty);
bool filtered = pcmk_is_set(options, pcmk__xml_fmt_filtered);
int spaces = pretty? (2 * depth) : 0;
for (int lpc = 0; lpc < spaces; lpc++) {
g_string_append_c(buffer, ' ');
}
pcmk__g_strcat(buffer, "<", data->name, NULL);
for (const xmlAttr *attr = pcmk__xe_first_attr(data); attr != NULL;
attr = attr->next) {
if (!filtered || !pcmk__xa_filterable((const char *) (attr->name))) {
pcmk__dump_xml_attr(attr, buffer);
}
}
if (data->children == NULL) {
g_string_append(buffer, "/>");
} else {
g_string_append_c(buffer, '>');
}
if (pretty) {
g_string_append_c(buffer, '\n');
}
if (data->children) {
for (const xmlNode *child = data->children; child != NULL;
child = child->next) {
pcmk__xml2text(child, options, buffer, depth + 1);
}
for (int lpc = 0; lpc < spaces; lpc++) {
g_string_append_c(buffer, ' ');
}
pcmk__g_strcat(buffer, "</", data->name, ">", NULL);
if (pretty) {
g_string_append_c(buffer, '\n');
}
}
}
/*!
* \internal
* \brief Append XML text content to a buffer
*
* \param[in] data XML whose content to append
* \param[in] options Group of \p xml_log_options flags
* \param[in,out] buffer Where to append the content (must not be \p NULL)
* \param[in] depth Current indentation level
*/
static void
dump_xml_text(const xmlNode *data, uint32_t options, GString *buffer,
int depth)
{
/* @COMPAT: Remove when log_data_element() is removed. There are no internal
* code paths to this, except through the deprecated log_data_element().
*/
bool pretty = pcmk_is_set(options, pcmk__xml_fmt_pretty);
int spaces = pretty? (2 * depth) : 0;
for (int lpc = 0; lpc < spaces; lpc++) {
g_string_append_c(buffer, ' ');
}
g_string_append(buffer, (const gchar *) data->content);
if (pretty) {
g_string_append_c(buffer, '\n');
}
}
/*!
* \internal
* \brief Append XML CDATA content to a buffer
*
* \param[in] data XML whose content to append
* \param[in] options Group of \p pcmk__xml_fmt_options flags
* \param[in,out] buffer Where to append the content (must not be \p NULL)
* \param[in] depth Current indentation level
*/
static void
dump_xml_cdata(const xmlNode *data, uint32_t options, GString *buffer,
int depth)
{
bool pretty = pcmk_is_set(options, pcmk__xml_fmt_pretty);
int spaces = pretty? (2 * depth) : 0;
for (int lpc = 0; lpc < spaces; lpc++) {
g_string_append_c(buffer, ' ');
}
pcmk__g_strcat(buffer, "<![CDATA[", (const char *) data->content, "]]>",
NULL);
if (pretty) {
g_string_append_c(buffer, '\n');
}
}
/*!
* \internal
* \brief Append an XML comment to a buffer
*
* \param[in] data XML whose content to append
* \param[in] options Group of \p pcmk__xml_fmt_options flags
* \param[in,out] buffer Where to append the content (must not be \p NULL)
* \param[in] depth Current indentation level
*/
static void
dump_xml_comment(const xmlNode *data, uint32_t options, GString *buffer,
int depth)
{
bool pretty = pcmk_is_set(options, pcmk__xml_fmt_pretty);
int spaces = pretty? (2 * depth) : 0;
for (int lpc = 0; lpc < spaces; lpc++) {
g_string_append_c(buffer, ' ');
}
pcmk__g_strcat(buffer, "<!--", (const char *) data->content, "-->", NULL);
if (pretty) {
g_string_append_c(buffer, '\n');
}
}
/*!
* \internal
* \brief Get a string representation of an XML element type
*
* \param[in] type XML element type
*
* \return String representation of \p type
*/
static const char *
xml_element_type2str(xmlElementType type)
{
static const char *const element_type_names[] = {
[XML_ELEMENT_NODE] = "element",
[XML_ATTRIBUTE_NODE] = "attribute",
[XML_TEXT_NODE] = "text",
[XML_CDATA_SECTION_NODE] = "CDATA section",
[XML_ENTITY_REF_NODE] = "entity reference",
[XML_ENTITY_NODE] = "entity",
[XML_PI_NODE] = "PI",
[XML_COMMENT_NODE] = "comment",
[XML_DOCUMENT_NODE] = "document",
[XML_DOCUMENT_TYPE_NODE] = "document type",
[XML_DOCUMENT_FRAG_NODE] = "document fragment",
[XML_NOTATION_NODE] = "notation",
[XML_HTML_DOCUMENT_NODE] = "HTML document",
[XML_DTD_NODE] = "DTD",
[XML_ELEMENT_DECL] = "element declaration",
[XML_ATTRIBUTE_DECL] = "attribute declaration",
[XML_ENTITY_DECL] = "entity declaration",
[XML_NAMESPACE_DECL] = "namespace declaration",
[XML_XINCLUDE_START] = "XInclude start",
[XML_XINCLUDE_END] = "XInclude end",
};
if ((type < 0) || (type >= PCMK__NELEM(element_type_names))) {
return "unrecognized type";
}
return element_type_names[type];
}
/*!
* \internal
* \brief Create a text representation of an XML object
*
* \param[in] data XML to convert
* \param[in] options Group of \p pcmk__xml_fmt_options flags
* \param[in,out] buffer Where to store the text (must not be \p NULL)
* \param[in] depth Current indentation level
*/
void
pcmk__xml2text(const xmlNode *data, uint32_t options, GString *buffer,
int depth)
{
if (data == NULL) {
crm_trace("Nothing to dump");
return;
}
CRM_ASSERT(buffer != NULL);
CRM_CHECK(depth >= 0, depth = 0);
switch(data->type) {
case XML_ELEMENT_NODE:
/* Handle below */
dump_xml_element(data, options, buffer, depth);
break;
case XML_TEXT_NODE:
if (pcmk_is_set(options, pcmk__xml_fmt_text)) {
dump_xml_text(data, options, buffer, depth);
}
break;
case XML_COMMENT_NODE:
dump_xml_comment(data, options, buffer, depth);
break;
case XML_CDATA_SECTION_NODE:
dump_xml_cdata(data, options, buffer, depth);
break;
default:
crm_warn("Cannot convert XML %s node to text " CRM_XS " type=%d",
xml_element_type2str(data->type), data->type);
break;
}
}
char *
dump_xml_formatted_with_text(const xmlNode *xml)
{
/* libxml's xmlNodeDumpOutput() would work here since we're not specifically
* filtering out any nodes. However, use pcmk__xml2text() for consistency,
* to escape attribute values, and to allow a const argument.
*/
char *buffer = NULL;
GString *g_buffer = g_string_sized_new(1024);
pcmk__xml2text(xml, pcmk__xml_fmt_pretty|pcmk__xml_fmt_text, g_buffer, 0);
pcmk__str_update(&buffer, g_buffer->str);
g_string_free(g_buffer, TRUE);
return buffer;
}
char *
dump_xml_formatted(const xmlNode *xml)
{
char *buffer = NULL;
GString *g_buffer = g_string_sized_new(1024);
pcmk__xml2text(xml, pcmk__xml_fmt_pretty, g_buffer, 0);
pcmk__str_update(&buffer, g_buffer->str);
g_string_free(g_buffer, TRUE);
return buffer;
}
char *
dump_xml_unformatted(const xmlNode *xml)
{
char *buffer = NULL;
GString *g_buffer = g_string_sized_new(1024);
pcmk__xml2text(xml, 0, g_buffer, 0);
pcmk__str_update(&buffer, g_buffer->str);
g_string_free(g_buffer, TRUE);
return buffer;
}
int
pcmk__xml2fd(int fd, xmlNode *cur)
{
bool success;
xmlOutputBuffer *fd_out = xmlOutputBufferCreateFd(fd, NULL);
CRM_ASSERT(fd_out != NULL);
xmlNodeDumpOutput(fd_out, cur->doc, cur, 0, pcmk__xml_fmt_pretty, NULL);
success = xmlOutputBufferWrite(fd_out, sizeof("\n") - 1, "\n") != -1;
success = xmlOutputBufferClose(fd_out) != -1 && success;
if (!success) {
return EIO;
}
fsync(fd);
return pcmk_rc_ok;
}
void
xml_remove_prop(xmlNode * obj, const char *name)
{
if (crm_element_value(obj, name) == NULL) {
return;
}
if (pcmk__check_acl(obj, NULL, pcmk__xf_acl_write) == FALSE) {
crm_trace("Cannot remove %s from %s", name, obj->name);
} else if (pcmk__tracking_xml_changes(obj, FALSE)) {
/* Leave in place (marked for removal) until after the diff is calculated */
xmlAttr *attr = xmlHasProp(obj, (pcmkXmlStr) name);
xml_node_private_t *nodepriv = attr->_private;
set_parent_flag(obj, pcmk__xf_dirty);
pcmk__set_xml_flags(nodepriv, pcmk__xf_deleted);
} else {
xmlUnsetProp(obj, (pcmkXmlStr) name);
}
}
void
save_xml_to_file(const xmlNode *xml, const char *desc, const char *filename)
{
char *f = NULL;
if (filename == NULL) {
char *uuid = crm_generate_uuid();
f = crm_strdup_printf("%s/%s", pcmk__get_tmpdir(), uuid);
filename = f;
free(uuid);
}
crm_info("Saving %s to %s", desc, filename);
write_xml_file(xml, filename, FALSE);
free(f);
}
/*!
* \internal
* \brief Set a flag on all attributes of an XML element
*
* \param[in,out] xml XML node to set flags on
* \param[in] flag XML private flag to set
*/
static void
set_attrs_flag(xmlNode *xml, enum xml_private_flags flag)
{
for (xmlAttr *attr = pcmk__xe_first_attr(xml); attr; attr = attr->next) {
pcmk__set_xml_flags((xml_node_private_t *) (attr->_private), flag);
}
}
/*!
* \internal
* \brief Add an XML attribute to a node, marked as deleted
*
* When calculating XML changes, we need to know when an attribute has been
* deleted. Add the attribute back to the new XML, so that we can check the
* removal against ACLs, and mark it as deleted for later removal after
* differences have been calculated.
*
* \param[in,out] new_xml XML to modify
* \param[in] element Name of XML element that changed (for logging)
* \param[in] attr_name Name of attribute that was deleted
* \param[in] old_value Value of attribute that was deleted
*/
static void
mark_attr_deleted(xmlNode *new_xml, const char *element, const char *attr_name,
const char *old_value)
{
xml_doc_private_t *docpriv = new_xml->doc->_private;
xmlAttr *attr = NULL;
xml_node_private_t *nodepriv;
// Prevent the dirty flag being set recursively upwards
pcmk__clear_xml_flags(docpriv, pcmk__xf_tracking);
// Restore the old value (and the tracking flag)
attr = xmlSetProp(new_xml, (pcmkXmlStr) attr_name, (pcmkXmlStr) old_value);
pcmk__set_xml_flags(docpriv, pcmk__xf_tracking);
// Reset flags (so the attribute doesn't appear as newly created)
nodepriv = attr->_private;
nodepriv->flags = 0;
// Check ACLs and mark restored value for later removal
xml_remove_prop(new_xml, attr_name);
crm_trace("XML attribute %s=%s was removed from %s",
attr_name, old_value, element);
}
/*
* \internal
* \brief Check ACLs for a changed XML attribute
*/
static void
mark_attr_changed(xmlNode *new_xml, const char *element, const char *attr_name,
const char *old_value)
{
char *vcopy = crm_element_value_copy(new_xml, attr_name);
crm_trace("XML attribute %s was changed from '%s' to '%s' in %s",
attr_name, old_value, vcopy, element);
// Restore the original value
xmlSetProp(new_xml, (pcmkXmlStr) attr_name, (pcmkXmlStr) old_value);
// Change it back to the new value, to check ACLs
crm_xml_add(new_xml, attr_name, vcopy);
free(vcopy);
}
/*!
* \internal
* \brief Mark an XML attribute as having changed position
*
* \param[in,out] new_xml XML to modify
* \param[in] element Name of XML element that changed (for logging)
* \param[in,out] old_attr Attribute that moved, in original XML
* \param[in,out] new_attr Attribute that moved, in \p new_xml
* \param[in] p_old Ordinal position of \p old_attr in original XML
* \param[in] p_new Ordinal position of \p new_attr in \p new_xml
*/
static void
mark_attr_moved(xmlNode *new_xml, const char *element, xmlAttr *old_attr,
xmlAttr *new_attr, int p_old, int p_new)
{
xml_node_private_t *nodepriv = new_attr->_private;
crm_trace("XML attribute %s moved from position %d to %d in %s",
old_attr->name, p_old, p_new, element);
// Mark document, element, and all element's parents as changed
pcmk__mark_xml_node_dirty(new_xml);
// Mark attribute as changed
pcmk__set_xml_flags(nodepriv, pcmk__xf_dirty|pcmk__xf_moved);
nodepriv = (p_old > p_new)? old_attr->_private : new_attr->_private;
pcmk__set_xml_flags(nodepriv, pcmk__xf_skip);
}
/*!
* \internal
* \brief Calculate differences in all previously existing XML attributes
*
* \param[in,out] old_xml Original XML to compare
* \param[in,out] new_xml New XML to compare
*/
static void
xml_diff_old_attrs(xmlNode *old_xml, xmlNode *new_xml)
{
xmlAttr *attr_iter = pcmk__xe_first_attr(old_xml);
while (attr_iter != NULL) {
const char *name = (const char *) attr_iter->name;
xmlAttr *old_attr = attr_iter;
xmlAttr *new_attr = xmlHasProp(new_xml, attr_iter->name);
const char *old_value = pcmk__xml_attr_value(attr_iter);
attr_iter = attr_iter->next;
if (new_attr == NULL) {
mark_attr_deleted(new_xml, (const char *) old_xml->name, name,
old_value);
} else {
xml_node_private_t *nodepriv = new_attr->_private;
int new_pos = pcmk__xml_position((xmlNode*) new_attr,
pcmk__xf_skip);
int old_pos = pcmk__xml_position((xmlNode*) old_attr,
pcmk__xf_skip);
const char *new_value = crm_element_value(new_xml, name);
// This attribute isn't new
pcmk__clear_xml_flags(nodepriv, pcmk__xf_created);
if (strcmp(new_value, old_value) != 0) {
mark_attr_changed(new_xml, (const char *) old_xml->name, name,
old_value);
} else if ((old_pos != new_pos)
&& !pcmk__tracking_xml_changes(new_xml, TRUE)) {
mark_attr_moved(new_xml, (const char *) old_xml->name,
old_attr, new_attr, old_pos, new_pos);
}
}
}
}
/*!
* \internal
* \brief Check all attributes in new XML for creation
*
* For each of a given XML element's attributes marked as newly created, accept
* (and mark as dirty) or reject the creation according to ACLs.
*
* \param[in,out] new_xml XML to check
*/
static void
mark_created_attrs(xmlNode *new_xml)
{
xmlAttr *attr_iter = pcmk__xe_first_attr(new_xml);
while (attr_iter != NULL) {
xmlAttr *new_attr = attr_iter;
xml_node_private_t *nodepriv = attr_iter->_private;
attr_iter = attr_iter->next;
if (pcmk_is_set(nodepriv->flags, pcmk__xf_created)) {
const char *attr_name = (const char *) new_attr->name;
crm_trace("Created new attribute %s=%s in %s",
attr_name, pcmk__xml_attr_value(new_attr),
new_xml->name);
/* Check ACLs (we can't use the remove-then-create trick because it
* would modify the attribute position).
*/
if (pcmk__check_acl(new_xml, attr_name, pcmk__xf_acl_write)) {
pcmk__mark_xml_attr_dirty(new_attr);
} else {
// Creation was not allowed, so remove the attribute
xmlUnsetProp(new_xml, new_attr->name);
}
}
}
}
/*!
* \internal
* \brief Calculate differences in attributes between two XML nodes
*
* \param[in,out] old_xml Original XML to compare
* \param[in,out] new_xml New XML to compare
*/
static void
xml_diff_attrs(xmlNode *old_xml, xmlNode *new_xml)
{
set_attrs_flag(new_xml, pcmk__xf_created); // cleared later if not really new
xml_diff_old_attrs(old_xml, new_xml);
mark_created_attrs(new_xml);
}
/*!
* \internal
* \brief Add an XML child element to a node, marked as deleted
*
* When calculating XML changes, we need to know when a child element has been
* deleted. Add the child back to the new XML, so that we can check the removal
* against ACLs, and mark it as deleted for later removal after differences have
* been calculated.
*
* \param[in,out] old_child Child element from original XML
* \param[in,out] new_parent New XML to add marked copy to
*/
static void
mark_child_deleted(xmlNode *old_child, xmlNode *new_parent)
{
// Re-create the child element so we can check ACLs
xmlNode *candidate = add_node_copy(new_parent, old_child);
// Clear flags on new child and its children
reset_xml_node_flags(candidate);
// Check whether ACLs allow the deletion
pcmk__apply_acl(xmlDocGetRootElement(candidate->doc));
// Remove the child again (which will track it in document's deleted_objs)
free_xml_with_position(candidate,
pcmk__xml_position(old_child, pcmk__xf_skip));
if (pcmk__xml_match(new_parent, old_child, true) == NULL) {
pcmk__set_xml_flags((xml_node_private_t *) (old_child->_private),
pcmk__xf_skip);
}
}
static void
mark_child_moved(xmlNode *old_child, xmlNode *new_parent, xmlNode *new_child,
int p_old, int p_new)
{
xml_node_private_t *nodepriv = new_child->_private;
crm_trace("Child element %s with id='%s' moved from position %d to %d under %s",
new_child->name, (ID(new_child)? ID(new_child) : "<no id>"),
p_old, p_new, new_parent->name);
pcmk__mark_xml_node_dirty(new_parent);
pcmk__set_xml_flags(nodepriv, pcmk__xf_moved);
if (p_old > p_new) {
nodepriv = old_child->_private;
} else {
nodepriv = new_child->_private;
}
pcmk__set_xml_flags(nodepriv, pcmk__xf_skip);
}
// Given original and new XML, mark new XML portions that have changed
static void
mark_xml_changes(xmlNode *old_xml, xmlNode *new_xml, bool check_top)
{
xmlNode *cIter = NULL;
xml_node_private_t *nodepriv = NULL;
CRM_CHECK(new_xml != NULL, return);
if (old_xml == NULL) {
pcmk__mark_xml_created(new_xml);
pcmk__apply_creation_acl(new_xml, check_top);
return;
}
nodepriv = new_xml->_private;
CRM_CHECK(nodepriv != NULL, return);
if(nodepriv->flags & pcmk__xf_processed) {
/* Avoid re-comparing nodes */
return;
}
pcmk__set_xml_flags(nodepriv, pcmk__xf_processed);
xml_diff_attrs(old_xml, new_xml);
// Check for differences in the original children
for (cIter = pcmk__xml_first_child(old_xml); cIter != NULL; ) {
xmlNode *old_child = cIter;
xmlNode *new_child = pcmk__xml_match(new_xml, cIter, true);
cIter = pcmk__xml_next(cIter);
if(new_child) {
mark_xml_changes(old_child, new_child, TRUE);
} else {
mark_child_deleted(old_child, new_xml);
}
}
// Check for moved or created children
for (cIter = pcmk__xml_first_child(new_xml); cIter != NULL; ) {
xmlNode *new_child = cIter;
xmlNode *old_child = pcmk__xml_match(old_xml, cIter, true);
cIter = pcmk__xml_next(cIter);
if(old_child == NULL) {
// This is a newly created child
nodepriv = new_child->_private;
pcmk__set_xml_flags(nodepriv, pcmk__xf_skip);
mark_xml_changes(old_child, new_child, TRUE);
} else {
/* Check for movement, we already checked for differences */
int p_new = pcmk__xml_position(new_child, pcmk__xf_skip);
int p_old = pcmk__xml_position(old_child, pcmk__xf_skip);
if(p_old != p_new) {
mark_child_moved(old_child, new_xml, new_child, p_old, p_new);
}
}
}
}
void
xml_calculate_significant_changes(xmlNode *old_xml, xmlNode *new_xml)
{
pcmk__set_xml_doc_flag(new_xml, pcmk__xf_lazy);
xml_calculate_changes(old_xml, new_xml);
}
// Called functions may set the \p pcmk__xf_skip flag on parts of \p old_xml
void
xml_calculate_changes(xmlNode *old_xml, xmlNode *new_xml)
{
CRM_CHECK((old_xml != NULL) && (new_xml != NULL)
&& pcmk__xe_is(old_xml, (const char *) new_xml->name)
&& pcmk__str_eq(ID(old_xml), ID(new_xml), pcmk__str_none),
return);
if(xml_tracking_changes(new_xml) == FALSE) {
xml_track_changes(new_xml, NULL, NULL, FALSE);
}
mark_xml_changes(old_xml, new_xml, FALSE);
}
gboolean
can_prune_leaf(xmlNode * xml_node)
{
xmlNode *cIter = NULL;
gboolean can_prune = TRUE;
CRM_CHECK(xml_node != NULL, return FALSE);
if (pcmk__strcase_any_of((const char *) xml_node->name,
XML_TAG_RESOURCE_REF, XML_CIB_TAG_OBJ_REF,
XML_ACL_TAG_ROLE_REF, XML_ACL_TAG_ROLE_REFv1,
NULL)) {
return FALSE;
}
for (xmlAttrPtr a = pcmk__xe_first_attr(xml_node); a != NULL; a = a->next) {
const char *p_name = (const char *) a->name;
if (strcmp(p_name, XML_ATTR_ID) == 0) {
continue;
}
can_prune = FALSE;
}
cIter = pcmk__xml_first_child(xml_node);
while (cIter) {
xmlNode *child = cIter;
cIter = pcmk__xml_next(cIter);
if (can_prune_leaf(child)) {
free_xml(child);
} else {
can_prune = FALSE;
}
}
return can_prune;
}
/*!
* \internal
* \brief Find a comment with matching content in specified XML
*
* \param[in] root XML to search
* \param[in] search_comment Comment whose content should be searched for
* \param[in] exact If true, comment must also be at same position
*/
xmlNode *
pcmk__xc_match(const xmlNode *root, const xmlNode *search_comment, bool exact)
{
xmlNode *a_child = NULL;
int search_offset = pcmk__xml_position(search_comment, pcmk__xf_skip);
CRM_CHECK(search_comment->type == XML_COMMENT_NODE, return NULL);
for (a_child = pcmk__xml_first_child(root); a_child != NULL;
a_child = pcmk__xml_next(a_child)) {
if (exact) {
int offset = pcmk__xml_position(a_child, pcmk__xf_skip);
xml_node_private_t *nodepriv = a_child->_private;
if (offset < search_offset) {
continue;
} else if (offset > search_offset) {
return NULL;
}
if (pcmk_is_set(nodepriv->flags, pcmk__xf_skip)) {
continue;
}
}
if (a_child->type == XML_COMMENT_NODE
&& pcmk__str_eq((const char *)a_child->content, (const char *)search_comment->content, pcmk__str_casei)) {
return a_child;
} else if (exact) {
return NULL;
}
}
return NULL;
}
/*!
* \internal
* \brief Make one XML comment match another (in content)
*
* \param[in,out] parent If \p target is NULL and this is not, add or update
* comment child of this XML node that matches \p update
* \param[in,out] target If not NULL, update this XML comment node
* \param[in] update Make comment content match this (must not be NULL)
*
* \note At least one of \parent and \target must be non-NULL
*/
void
pcmk__xc_update(xmlNode *parent, xmlNode *target, xmlNode *update)
{
CRM_CHECK(update != NULL, return);
CRM_CHECK(update->type == XML_COMMENT_NODE, return);
if (target == NULL) {
target = pcmk__xc_match(parent, update, false);
}
if (target == NULL) {
add_node_copy(parent, update);
} else if (!pcmk__str_eq((const char *)target->content, (const char *)update->content, pcmk__str_casei)) {
xmlFree(target->content);
target->content = xmlStrdup(update->content);
}
}
/*!
* \internal
* \brief Make one XML tree match another (in children and attributes)
*
* \param[in,out] parent If \p target is NULL and this is not, add or update
* child of this XML node that matches \p update
* \param[in,out] target If not NULL, update this XML
* \param[in] update Make the desired XML match this (must not be NULL)
* \param[in] as_diff If false, expand "++" when making attributes match
*
* \note At least one of \p parent and \p target must be non-NULL
*/
void
pcmk__xml_update(xmlNode *parent, xmlNode *target, xmlNode *update,
bool as_diff)
{
xmlNode *a_child = NULL;
const char *object_name = NULL,
*object_href = NULL,
*object_href_val = NULL;
#if XML_PARSER_DEBUG
crm_log_xml_trace(update, "update:");
crm_log_xml_trace(target, "target:");
#endif
CRM_CHECK(update != NULL, return);
if (update->type == XML_COMMENT_NODE) {
pcmk__xc_update(parent, target, update);
return;
}
object_name = (const char *) update->name;
object_href_val = ID(update);
if (object_href_val != NULL) {
object_href = XML_ATTR_ID;
} else {
object_href_val = crm_element_value(update, XML_ATTR_IDREF);
object_href = (object_href_val == NULL) ? NULL : XML_ATTR_IDREF;
}
CRM_CHECK(object_name != NULL, return);
CRM_CHECK(target != NULL || parent != NULL, return);
if (target == NULL) {
target = pcmk__xe_match(parent, object_name,
object_href, object_href_val);
}
if (target == NULL) {
target = create_xml_node(parent, object_name);
CRM_CHECK(target != NULL, return);
#if XML_PARSER_DEBUG
crm_trace("Added <%s%s%s%s%s/>", pcmk__s(object_name, "<null>"),
object_href ? " " : "",
object_href ? object_href : "",
object_href ? "=" : "",
object_href ? object_href_val : "");
} else {
crm_trace("Found node <%s%s%s%s%s/> to update",
pcmk__s(object_name, "<null>"),
object_href ? " " : "",
object_href ? object_href : "",
object_href ? "=" : "",
object_href ? object_href_val : "");
#endif
}
CRM_CHECK(pcmk__xe_is(target, (const char *) update->name), return);
if (as_diff == FALSE) {
/* So that expand_plus_plus() gets called */
copy_in_properties(target, update);
} else {
/* No need for expand_plus_plus(), just raw speed */
for (xmlAttrPtr a = pcmk__xe_first_attr(update); a != NULL;
a = a->next) {
const char *p_value = pcmk__xml_attr_value(a);
/* Remove it first so the ordering of the update is preserved */
xmlUnsetProp(target, a->name);
xmlSetProp(target, a->name, (pcmkXmlStr) p_value);
}
}
for (a_child = pcmk__xml_first_child(update); a_child != NULL;
a_child = pcmk__xml_next(a_child)) {
#if XML_PARSER_DEBUG
crm_trace("Updating child <%s%s%s%s%s/>",
pcmk__s(object_name, "<null>"),
object_href ? " " : "",
object_href ? object_href : "",
object_href ? "=" : "",
object_href ? object_href_val : "");
#endif
pcmk__xml_update(target, NULL, a_child, as_diff);
}
#if XML_PARSER_DEBUG
crm_trace("Finished with <%s%s%s%s%s/>", pcmk__s(object_name, "<null>"),
object_href ? " " : "",
object_href ? object_href : "",
object_href ? "=" : "",
object_href ? object_href_val : "");
#endif
}
gboolean
update_xml_child(xmlNode * child, xmlNode * to_update)
{
gboolean can_update = TRUE;
xmlNode *child_of_child = NULL;
CRM_CHECK(child != NULL, return FALSE);
CRM_CHECK(to_update != NULL, return FALSE);
if (!pcmk__xe_is(to_update, (const char *) child->name)) {
can_update = FALSE;
} else if (!pcmk__str_eq(ID(to_update), ID(child), pcmk__str_none)) {
can_update = FALSE;
} else if (can_update) {
#if XML_PARSER_DEBUG
crm_log_xml_trace(child, "Update match found...");
#endif
pcmk__xml_update(NULL, child, to_update, false);
}
for (child_of_child = pcmk__xml_first_child(child); child_of_child != NULL;
child_of_child = pcmk__xml_next(child_of_child)) {
/* only update the first one */
if (can_update) {
break;
}
can_update = update_xml_child(child_of_child, to_update);
}
return can_update;
}
int
find_xml_children(xmlNode ** children, xmlNode * root,
const char *tag, const char *field, const char *value, gboolean search_matches)
{
int match_found = 0;
CRM_CHECK(root != NULL, return FALSE);
CRM_CHECK(children != NULL, return FALSE);
if ((tag != NULL) && !pcmk__xe_is(root, tag)) {
} else if (value != NULL && !pcmk__str_eq(value, crm_element_value(root, field), pcmk__str_casei)) {
} else {
if (*children == NULL) {
*children = create_xml_node(NULL, __func__);
}
add_node_copy(*children, root);
match_found = 1;
}
if (search_matches || match_found == 0) {
xmlNode *child = NULL;
for (child = pcmk__xml_first_child(root); child != NULL;
child = pcmk__xml_next(child)) {
match_found += find_xml_children(children, child, tag, field, value, search_matches);
}
}
return match_found;
}
gboolean
replace_xml_child(xmlNode * parent, xmlNode * child, xmlNode * update, gboolean delete_only)
{
gboolean can_delete = FALSE;
xmlNode *child_of_child = NULL;
const char *up_id = NULL;
const char *child_id = NULL;
const char *right_val = NULL;
CRM_CHECK(child != NULL, return FALSE);
CRM_CHECK(update != NULL, return FALSE);
up_id = ID(update);
child_id = ID(child);
if (up_id == NULL || (child_id && strcmp(child_id, up_id) == 0)) {
can_delete = TRUE;
}
if (!pcmk__xe_is(update, (const char *) child->name)) {
can_delete = FALSE;
}
if (can_delete && delete_only) {
for (xmlAttrPtr a = pcmk__xe_first_attr(update); a != NULL;
a = a->next) {
const char *p_name = (const char *) a->name;
const char *p_value = pcmk__xml_attr_value(a);
right_val = crm_element_value(child, p_name);
if (!pcmk__str_eq(p_value, right_val, pcmk__str_casei)) {
can_delete = FALSE;
}
}
}
if (can_delete && parent != NULL) {
crm_log_xml_trace(child, "Delete match found...");
if (delete_only || update == NULL) {
free_xml(child);
} else {
xmlNode *old = child;
xmlNode *new = xmlCopyNode(update, 1);
CRM_ASSERT(new != NULL);
// May be unnecessary but avoids slight changes to some test outputs
reset_xml_node_flags(new);
old = xmlReplaceNode(old, new);
if (xml_tracking_changes(new)) {
// Replaced sections may have included relevant ACLs
pcmk__apply_acl(new);
}
xml_calculate_changes(old, new);
xmlFreeNode(old);
}
return TRUE;
} else if (can_delete) {
crm_log_xml_debug(child, "Cannot delete the search root");
can_delete = FALSE;
}
child_of_child = pcmk__xml_first_child(child);
while (child_of_child) {
xmlNode *next = pcmk__xml_next(child_of_child);
can_delete = replace_xml_child(child, child_of_child, update, delete_only);
/* only delete the first one */
if (can_delete) {
child_of_child = NULL;
} else {
child_of_child = next;
}
}
return can_delete;
}
xmlNode *
sorted_xml(xmlNode *input, xmlNode *parent, gboolean recursive)
{
xmlNode *child = NULL;
GSList *nvpairs = NULL;
xmlNode *result = NULL;
CRM_CHECK(input != NULL, return NULL);
result = create_xml_node(parent, (const char *) input->name);
nvpairs = pcmk_xml_attrs2nvpairs(input);
nvpairs = pcmk_sort_nvpairs(nvpairs);
pcmk_nvpairs2xml_attrs(nvpairs, result);
pcmk_free_nvpairs(nvpairs);
for (child = pcmk__xml_first_child(input); child != NULL;
child = pcmk__xml_next(child)) {
if (recursive) {
sorted_xml(child, result, recursive);
} else {
add_node_copy(result, child);
}
}
return result;
}
xmlNode *
first_named_child(const xmlNode *parent, const char *name)
{
xmlNode *match = NULL;
for (match = pcmk__xe_first_child(parent); match != NULL;
match = pcmk__xe_next(match)) {
/*
* name == NULL gives first child regardless of name; this is
* semantically incorrect in this function, but may be necessary
* due to prior use of xml_child_iter_filter
*/
if (pcmk__str_eq(name, (const char *)match->name, pcmk__str_null_matches)) {
return match;
}
}
return NULL;
}
/*!
* \brief Get next instance of same XML tag
*
* \param[in] sibling XML tag to start from
*
* \return Next sibling XML tag with same name
*/
xmlNode *
crm_next_same_xml(const xmlNode *sibling)
{
xmlNode *match = pcmk__xe_next(sibling);
while (match != NULL) {
if (pcmk__xe_is(match, (const char *) sibling->name)) {
return match;
}
match = pcmk__xe_next(match);
}
return NULL;
}
void
crm_xml_init(void)
{
static bool init = true;
if(init) {
init = false;
/* The default allocator XML_BUFFER_ALLOC_EXACT does far too many
* pcmk__realloc()s and it can take upwards of 18 seconds (yes, seconds)
* to dump a 28kb tree which XML_BUFFER_ALLOC_DOUBLEIT can do in
* less than 1 second.
*/
xmlSetBufferAllocationScheme(XML_BUFFER_ALLOC_DOUBLEIT);
/* Populate and free the _private field when nodes are created and destroyed */
xmlDeregisterNodeDefault(free_private_data);
xmlRegisterNodeDefault(new_private_data);
crm_schema_init();
}
}
void
crm_xml_cleanup(void)
{
crm_schema_cleanup();
xmlCleanupParser();
}
#define XPATH_MAX 512
xmlNode *
expand_idref(xmlNode * input, xmlNode * top)
{
const char *ref = NULL;
xmlNode *result = input;
if (result == NULL) {
return NULL;
} else if (top == NULL) {
top = input;
}
ref = crm_element_value(result, XML_ATTR_IDREF);
if (ref != NULL) {
char *xpath_string = crm_strdup_printf("//%s[@" XML_ATTR_ID "='%s']",
result->name, ref);
result = get_xpath_object(xpath_string, top, LOG_ERR);
if (result == NULL) {
char *nodePath = (char *)xmlGetNodePath(top);
crm_err("No match for %s found in %s: Invalid configuration",
xpath_string, pcmk__s(nodePath, "unrecognizable path"));
free(nodePath);
}
free(xpath_string);
}
return result;
}
char *
pcmk__xml_artefact_root(enum pcmk__xml_artefact_ns ns)
{
static const char *base = NULL;
char *ret = NULL;
if (base == NULL) {
- base = getenv("PCMK_schema_directory");
+ base = pcmk__env_option(PCMK__ENV_SCHEMA_DIRECTORY);
}
if (pcmk__str_empty(base)) {
base = CRM_SCHEMA_DIRECTORY;
}
switch (ns) {
case pcmk__xml_artefact_ns_legacy_rng:
case pcmk__xml_artefact_ns_legacy_xslt:
ret = strdup(base);
break;
case pcmk__xml_artefact_ns_base_rng:
case pcmk__xml_artefact_ns_base_xslt:
ret = crm_strdup_printf("%s/base", base);
break;
default:
crm_err("XML artefact family specified as %u not recognized", ns);
}
return ret;
}
char *
pcmk__xml_artefact_path(enum pcmk__xml_artefact_ns ns, const char *filespec)
{
char *base = pcmk__xml_artefact_root(ns), *ret = NULL;
switch (ns) {
case pcmk__xml_artefact_ns_legacy_rng:
case pcmk__xml_artefact_ns_base_rng:
ret = crm_strdup_printf("%s/%s.rng", base, filespec);
break;
case pcmk__xml_artefact_ns_legacy_xslt:
case pcmk__xml_artefact_ns_base_xslt:
ret = crm_strdup_printf("%s/%s.xsl", base, filespec);
break;
default:
crm_err("XML artefact family specified as %u not recognized", ns);
}
free(base);
return ret;
}
void
pcmk__xe_set_propv(xmlNodePtr node, va_list pairs)
{
while (true) {
const char *name, *value;
name = va_arg(pairs, const char *);
if (name == NULL) {
return;
}
value = va_arg(pairs, const char *);
if (value != NULL) {
crm_xml_add(node, name, value);
}
}
}
void
pcmk__xe_set_props(xmlNodePtr node, ...)
{
va_list pairs;
va_start(pairs, node);
pcmk__xe_set_propv(node, pairs);
va_end(pairs);
}
int
pcmk__xe_foreach_child(xmlNode *xml, const char *child_element_name,
int (*handler)(xmlNode *xml, void *userdata),
void *userdata)
{
xmlNode *children = (xml? xml->children : NULL);
CRM_ASSERT(handler != NULL);
for (xmlNode *node = children; node != NULL; node = node->next) {
if (node->type == XML_ELEMENT_NODE &&
pcmk__str_eq(child_element_name, (const char *) node->name, pcmk__str_null_matches)) {
int rc = handler(node, userdata);
if (rc != pcmk_rc_ok) {
return rc;
}
}
}
return pcmk_rc_ok;
}
// Deprecated functions kept only for backward API compatibility
// LCOV_EXCL_START
#include <crm/common/xml_compat.h>
xmlNode *
find_entity(xmlNode *parent, const char *node_name, const char *id)
{
return pcmk__xe_match(parent, node_name,
((id == NULL)? id : XML_ATTR_ID), id);
}
void
crm_destroy_xml(gpointer data)
{
free_xml(data);
}
xmlDoc *
getDocPtr(xmlNode *node)
{
xmlDoc *doc = NULL;
CRM_CHECK(node != NULL, return NULL);
doc = node->doc;
if (doc == NULL) {
doc = xmlNewDoc((pcmkXmlStr) "1.0");
xmlDocSetRootElement(doc, node);
}
return doc;
}
int
add_node_nocopy(xmlNode *parent, const char *name, xmlNode *child)
{
add_node_copy(parent, child);
free_xml(child);
return 1;
}
gboolean
xml_has_children(const xmlNode * xml_root)
{
if (xml_root != NULL && xml_root->children != NULL) {
return TRUE;
}
return FALSE;
}
// LCOV_EXCL_STOP
// End deprecated API
diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c
index 44cc52bb92..400d3b0935 100644
--- a/lib/lrmd/lrmd_client.c
+++ b/lib/lrmd/lrmd_client.c
@@ -1,2570 +1,2570 @@
/*
* Copyright 2012-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h> // uint32_t, uint64_t
#include <stdarg.h>
#include <string.h>
#include <ctype.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <glib.h>
#include <dirent.h>
#include <crm/crm.h>
#include <crm/lrmd.h>
#include <crm/lrmd_internal.h>
#include <crm/services.h>
#include <crm/services_internal.h>
#include <crm/common/mainloop.h>
#include <crm/common/ipc_internal.h>
#include <crm/common/remote_internal.h>
#include <crm/msg_xml.h>
#include <crm/stonith-ng.h>
#include <crm/fencing/internal.h>
#ifdef HAVE_GNUTLS_GNUTLS_H
# include <gnutls/gnutls.h>
#endif
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#include <netdb.h>
#define MAX_TLS_RECV_WAIT 10000
CRM_TRACE_INIT_DATA(lrmd);
static int lrmd_api_disconnect(lrmd_t * lrmd);
static int lrmd_api_is_connected(lrmd_t * lrmd);
/* IPC proxy functions */
int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
#ifdef HAVE_GNUTLS_GNUTLS_H
# define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000 /* 5 seconds */
gnutls_psk_client_credentials_t psk_cred_s;
static void lrmd_tls_disconnect(lrmd_t * lrmd);
static int global_remote_msg_id = 0;
static void lrmd_tls_connection_destroy(gpointer userdata);
#endif
typedef struct lrmd_private_s {
uint64_t type;
char *token;
mainloop_io_t *source;
/* IPC parameters */
crm_ipc_t *ipc;
pcmk__remote_t *remote;
/* Extra TLS parameters */
char *remote_nodename;
#ifdef HAVE_GNUTLS_GNUTLS_H
char *server;
int port;
gnutls_psk_client_credentials_t psk_cred_c;
/* while the async connection is occurring, this is the id
* of the connection timeout timer. */
int async_timer;
int sock;
/* since tls requires a round trip across the network for a
* request/reply, there are times where we just want to be able
* to send a request from the client and not wait around (or even care
* about) what the reply is. */
int expected_late_replies;
GList *pending_notify;
crm_trigger_t *process_notify;
#endif
lrmd_event_callback callback;
/* Internal IPC proxy msg passing for remote guests */
void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
void *proxy_callback_userdata;
char *peer_version;
} lrmd_private_t;
static lrmd_list_t *
lrmd_list_add(lrmd_list_t * head, const char *value)
{
lrmd_list_t *p, *end;
p = calloc(1, sizeof(lrmd_list_t));
p->val = strdup(value);
end = head;
while (end && end->next) {
end = end->next;
}
if (end) {
end->next = p;
} else {
head = p;
}
return head;
}
void
lrmd_list_freeall(lrmd_list_t * head)
{
lrmd_list_t *p;
while (head) {
char *val = (char *)head->val;
p = head->next;
free(val);
free(head);
head = p;
}
}
lrmd_key_value_t *
lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
{
lrmd_key_value_t *p, *end;
p = calloc(1, sizeof(lrmd_key_value_t));
p->key = strdup(key);
p->value = strdup(value);
end = head;
while (end && end->next) {
end = end->next;
}
if (end) {
end->next = p;
} else {
head = p;
}
return head;
}
void
lrmd_key_value_freeall(lrmd_key_value_t * head)
{
lrmd_key_value_t *p;
while (head) {
p = head->next;
free(head->key);
free(head->value);
free(head);
head = p;
}
}
/*!
* \brief Create a new lrmd_event_data_t object
*
* \param[in] rsc_id ID of resource involved in event
* \param[in] task Action name
* \param[in] interval_ms Action interval
*
* \return Newly allocated and initialized lrmd_event_data_t
* \note This functions asserts on memory errors, so the return value is
* guaranteed to be non-NULL. The caller is responsible for freeing the
* result with lrmd_free_event().
*/
lrmd_event_data_t *
lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms)
{
lrmd_event_data_t *event = calloc(1, sizeof(lrmd_event_data_t));
CRM_ASSERT(event != NULL);
pcmk__str_update((char **) &event->rsc_id, rsc_id);
pcmk__str_update((char **) &event->op_type, task);
event->interval_ms = interval_ms;
return event;
}
lrmd_event_data_t *
lrmd_copy_event(lrmd_event_data_t * event)
{
lrmd_event_data_t *copy = NULL;
copy = calloc(1, sizeof(lrmd_event_data_t));
copy->type = event->type;
pcmk__str_update((char **) &copy->rsc_id, event->rsc_id);
pcmk__str_update((char **) &copy->op_type, event->op_type);
pcmk__str_update((char **) &copy->user_data, event->user_data);
copy->call_id = event->call_id;
copy->timeout = event->timeout;
copy->interval_ms = event->interval_ms;
copy->start_delay = event->start_delay;
copy->rsc_deleted = event->rsc_deleted;
copy->rc = event->rc;
copy->op_status = event->op_status;
pcmk__str_update((char **) &copy->output, event->output);
copy->t_run = event->t_run;
copy->t_rcchange = event->t_rcchange;
copy->exec_time = event->exec_time;
copy->queue_time = event->queue_time;
copy->connection_rc = event->connection_rc;
copy->params = pcmk__str_table_dup(event->params);
pcmk__str_update((char **) &copy->remote_nodename, event->remote_nodename);
pcmk__str_update((char **) &copy->exit_reason, event->exit_reason);
return copy;
}
/*!
* \brief Free an executor event
*
* \param[in,out] Executor event object to free
*/
void
lrmd_free_event(lrmd_event_data_t *event)
{
if (event == NULL) {
return;
}
// @TODO Why are these const char *?
free((void *) event->rsc_id);
free((void *) event->op_type);
free((void *) event->user_data);
free((void *) event->remote_nodename);
lrmd__reset_result(event);
if (event->params != NULL) {
g_hash_table_destroy(event->params);
}
free(event);
}
static void
lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
{
const char *type;
const char *proxy_session = crm_element_value(msg, F_LRMD_IPC_SESSION);
lrmd_private_t *native = lrmd->lrmd_private;
lrmd_event_data_t event = { 0, };
if (proxy_session != NULL) {
/* this is proxy business */
lrmd_internal_proxy_dispatch(lrmd, msg);
return;
} else if (!native->callback) {
/* no callback set */
crm_trace("notify event received but client has not set callback");
return;
}
event.remote_nodename = native->remote_nodename;
type = crm_element_value(msg, F_LRMD_OPERATION);
crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);
if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
event.type = lrmd_event_register;
} else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
event.type = lrmd_event_unregister;
} else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
time_t epoch = 0;
crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
crm_element_value_ms(msg, F_LRMD_RSC_INTERVAL, &event.interval_ms);
crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);
crm_element_value_epoch(msg, F_LRMD_RSC_RUN_TIME, &epoch);
event.t_run = (unsigned int) epoch;
crm_element_value_epoch(msg, F_LRMD_RSC_RCCHANGE_TIME, &epoch);
event.t_rcchange = (unsigned int) epoch;
crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);
event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
event.type = lrmd_event_exec_complete;
/* output and exit_reason may be freed by a callback */
event.output = crm_element_value_copy(msg, F_LRMD_RSC_OUTPUT);
lrmd__set_result(&event, event.rc, event.op_status,
crm_element_value(msg, F_LRMD_RSC_EXIT_REASON));
event.params = xml2list(msg);
} else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
event.type = lrmd_event_new_client;
} else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
event.type = lrmd_event_poke;
} else {
return;
}
crm_trace("op %s notify event received", type);
native->callback(&event);
if (event.params) {
g_hash_table_destroy(event.params);
}
lrmd__reset_result(&event);
}
// \return Always 0, to indicate that IPC mainloop source should be kept
static int
lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
if (native->callback != NULL) {
xmlNode *msg = string2xml(buffer);
lrmd_dispatch_internal(lrmd, msg);
free_xml(msg);
}
return 0;
}
#ifdef HAVE_GNUTLS_GNUTLS_H
static void
lrmd_free_xml(gpointer userdata)
{
free_xml((xmlNode *) userdata);
}
static bool
remote_executor_connected(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
return (native->remote->tls_session != NULL);
}
/*!
* \internal
* \brief TLS dispatch function (for both trigger and file descriptor sources)
*
* \param[in,out] userdata API connection
*
* \return Always return a nonnegative value, which as a file descriptor
* dispatch function means keep the mainloop source, and as a
* trigger dispatch function, 0 means remove the trigger from the
* mainloop while 1 means keep it (and job completed)
*/
static int
lrmd_tls_dispatch(gpointer userdata)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *xml = NULL;
int rc = pcmk_rc_ok;
if (!remote_executor_connected(lrmd)) {
crm_trace("TLS dispatch triggered after disconnect");
return 0;
}
crm_trace("TLS dispatch triggered");
/* First check if there are any pending notifies to process that came
* while we were waiting for replies earlier. */
if (native->pending_notify) {
GList *iter = NULL;
crm_trace("Processing pending notifies");
for (iter = native->pending_notify; iter; iter = iter->next) {
lrmd_dispatch_internal(lrmd, iter->data);
}
g_list_free_full(native->pending_notify, lrmd_free_xml);
native->pending_notify = NULL;
}
/* Next read the current buffer and see if there are any messages to handle. */
switch (pcmk__remote_ready(native->remote, 0)) {
case pcmk_rc_ok:
rc = pcmk__read_remote_message(native->remote, -1);
xml = pcmk__remote_message_xml(native->remote);
break;
case ETIME:
// Nothing to read, check if a full message is already in buffer
xml = pcmk__remote_message_xml(native->remote);
break;
default:
rc = ENOTCONN;
break;
}
while (xml) {
const char *msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
lrmd_dispatch_internal(lrmd, xml);
} else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
if (native->expected_late_replies > 0) {
native->expected_late_replies--;
} else {
int reply_id = 0;
crm_element_value_int(xml, F_LRMD_CALLID, &reply_id);
/* if this happens, we want to know about it */
crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
}
}
free_xml(xml);
xml = pcmk__remote_message_xml(native->remote);
}
if (rc == ENOTCONN) {
crm_info("Lost %s executor connection while reading data",
(native->remote_nodename? native->remote_nodename : "local"));
lrmd_tls_disconnect(lrmd);
return 0;
}
return 1;
}
#endif
/* Not used with mainloop */
int
lrmd_poll(lrmd_t * lrmd, int timeout)
{
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
return crm_ipc_ready(native->ipc);
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
if (native->pending_notify) {
return 1;
} else {
int rc = pcmk__remote_ready(native->remote, 0);
switch (rc) {
case pcmk_rc_ok:
return 1;
case ETIME:
return 0;
default:
return pcmk_rc2legacy(rc);
}
}
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
return -EPROTONOSUPPORT;
}
}
/* Not used with mainloop */
bool
lrmd_dispatch(lrmd_t * lrmd)
{
lrmd_private_t *private = NULL;
CRM_ASSERT(lrmd != NULL);
private = lrmd->lrmd_private;
switch (private->type) {
case pcmk__client_ipc:
while (crm_ipc_ready(private->ipc)) {
if (crm_ipc_read(private->ipc) > 0) {
const char *msg = crm_ipc_buffer(private->ipc);
lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
}
}
break;
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
lrmd_tls_dispatch(lrmd);
break;
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
private->type);
}
if (lrmd_api_is_connected(lrmd) == FALSE) {
crm_err("Connection closed");
return FALSE;
}
return TRUE;
}
static xmlNode *
lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
enum lrmd_call_options options)
{
xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");
CRM_CHECK(op_msg != NULL, return NULL);
CRM_CHECK(token != NULL, return NULL);
crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");
crm_xml_add(op_msg, F_TYPE, T_LRMD);
crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
crm_xml_add(op_msg, F_LRMD_OPERATION, op);
crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);
crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);
if (data != NULL) {
add_message_xml(op_msg, F_LRMD_CALLDATA, data);
}
crm_trace("Created executor %s command with call options %.8lx (%d)",
op, (long)options, options);
return op_msg;
}
static void
lrmd_ipc_connection_destroy(gpointer userdata)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
crm_info("Disconnected from local executor");
break;
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
crm_info("Disconnected from remote executor on %s",
native->remote_nodename);
break;
#endif
default:
crm_err("Unsupported executor connection type %d (bug?)",
native->type);
}
/* Prevent these from being cleaned up in lrmd_api_disconnect() */
native->ipc = NULL;
native->source = NULL;
if (native->callback) {
lrmd_event_data_t event = { 0, };
event.type = lrmd_event_disconnect;
event.remote_nodename = native->remote_nodename;
native->callback(&event);
}
}
#ifdef HAVE_GNUTLS_GNUTLS_H
static void
lrmd_tls_connection_destroy(gpointer userdata)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
crm_info("TLS connection destroyed");
if (native->remote->tls_session) {
gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
gnutls_deinit(*native->remote->tls_session);
gnutls_free(native->remote->tls_session);
}
if (native->psk_cred_c) {
gnutls_psk_free_client_credentials(native->psk_cred_c);
}
if (native->sock) {
close(native->sock);
}
if (native->process_notify) {
mainloop_destroy_trigger(native->process_notify);
native->process_notify = NULL;
}
if (native->pending_notify) {
g_list_free_full(native->pending_notify, lrmd_free_xml);
native->pending_notify = NULL;
}
free(native->remote->buffer);
free(native->remote->start_state);
native->remote->buffer = NULL;
native->remote->start_state = NULL;
native->source = 0;
native->sock = 0;
native->psk_cred_c = NULL;
native->remote->tls_session = NULL;
native->sock = 0;
if (native->callback) {
lrmd_event_data_t event = { 0, };
event.remote_nodename = native->remote_nodename;
event.type = lrmd_event_disconnect;
native->callback(&event);
}
return;
}
// \return Standard Pacemaker return code
int
lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
const char *msg_type)
{
crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);
return pcmk__remote_send_xml(session, msg);
}
// \return Standard Pacemaker return code
static int
read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
xmlNode **reply)
{
lrmd_private_t *native = lrmd->lrmd_private;
time_t start = time(NULL);
const char *msg_type = NULL;
int reply_id = 0;
int remaining_timeout = 0;
int rc = pcmk_rc_ok;
/* A timeout of 0 here makes no sense. We have to wait a period of time
* for the response to come back. If -1 or 0, default to 10 seconds. */
if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
total_timeout = MAX_TLS_RECV_WAIT;
}
for (*reply = NULL; *reply == NULL; ) {
*reply = pcmk__remote_message_xml(native->remote);
if (*reply == NULL) {
/* read some more off the tls buffer if we still have time left. */
if (remaining_timeout) {
remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
} else {
remaining_timeout = total_timeout;
}
if (remaining_timeout <= 0) {
return ETIME;
}
rc = pcmk__read_remote_message(native->remote, remaining_timeout);
if (rc != pcmk_rc_ok) {
return rc;
}
*reply = pcmk__remote_message_xml(native->remote);
if (*reply == NULL) {
return ENOMSG;
}
}
crm_element_value_int(*reply, F_LRMD_REMOTE_MSG_ID, &reply_id);
msg_type = crm_element_value(*reply, F_LRMD_REMOTE_MSG_TYPE);
if (!msg_type) {
crm_err("Empty msg type received while waiting for reply");
free_xml(*reply);
*reply = NULL;
} else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
/* got a notify while waiting for reply, trigger the notify to be processed later */
crm_info("queueing notify");
native->pending_notify = g_list_append(native->pending_notify, *reply);
if (native->process_notify) {
crm_info("notify trigger set.");
mainloop_set_trigger(native->process_notify);
}
*reply = NULL;
} else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
/* msg isn't a reply, make some noise */
crm_err("Expected a reply, got %s", msg_type);
free_xml(*reply);
*reply = NULL;
} else if (reply_id != expected_reply_id) {
if (native->expected_late_replies > 0) {
native->expected_late_replies--;
} else {
crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
}
free_xml(*reply);
*reply = NULL;
}
}
if (native->remote->buffer && native->process_notify) {
mainloop_set_trigger(native->process_notify);
}
return rc;
}
// \return Standard Pacemaker return code
static int
send_remote_message(lrmd_t *lrmd, xmlNode *msg)
{
int rc = pcmk_rc_ok;
lrmd_private_t *native = lrmd->lrmd_private;
global_remote_msg_id++;
if (global_remote_msg_id <= 0) {
global_remote_msg_id = 1;
}
rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
"request");
if (rc != pcmk_rc_ok) {
crm_err("Disconnecting because TLS message could not be sent to "
"Pacemaker Remote: %s", pcmk_rc_str(rc));
lrmd_tls_disconnect(lrmd);
}
return rc;
}
static int
lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
{
int rc = 0;
xmlNode *xml = NULL;
if (!remote_executor_connected(lrmd)) {
return -ENOTCONN;
}
rc = send_remote_message(lrmd, msg);
if (rc != pcmk_rc_ok) {
return pcmk_rc2legacy(rc);
}
rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
if (rc != pcmk_rc_ok) {
crm_err("Disconnecting remote after request %d reply not received: %s "
CRM_XS " rc=%d timeout=%dms",
global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
lrmd_tls_disconnect(lrmd);
}
if (reply) {
*reply = xml;
} else {
free_xml(xml);
}
return pcmk_rc2legacy(rc);
}
#endif
static int
lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
break;
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
break;
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
return rc;
}
static int
lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
break;
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
rc = send_remote_message(lrmd, msg);
if (rc == pcmk_rc_ok) {
/* we don't want to wait around for the reply, but
* since the request/reply protocol needs to behave the same
* as libqb, a reply will eventually come later anyway. */
native->expected_late_replies++;
}
rc = pcmk_rc2legacy(rc);
break;
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
return rc;
}
static int
lrmd_api_is_connected(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
return crm_ipc_connected(native->ipc);
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
return remote_executor_connected(lrmd);
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
return 0;
}
}
/*!
* \internal
* \brief Send a prepared API command to the executor
*
* \param[in,out] lrmd Existing connection to the executor
* \param[in] op Name of API command to send
* \param[in] data Command data XML to add to the sent command
* \param[out] output_data If expecting a reply, it will be stored here
* \param[in] timeout Timeout in milliseconds (if 0, defaults to
* a sensible value per the type of connection,
* standard vs. pacemaker remote);
* also propagated to the command XML
* \param[in] call_options Call options to pass to server when sending
* \param[in] expect_reply If TRUE, wait for a reply from the server;
* must be TRUE for IPC (as opposed to TLS) clients
*
* \return pcmk_ok on success, -errno on error
*/
static int
lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
xmlNode **output_data, int timeout,
enum lrmd_call_options options, gboolean expect_reply)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *op_msg = NULL;
xmlNode *op_reply = NULL;
if (!lrmd_api_is_connected(lrmd)) {
return -ENOTCONN;
}
if (op == NULL) {
crm_err("No operation specified");
return -EINVAL;
}
CRM_CHECK(native->token != NULL,;
);
crm_trace("Sending %s op to executor", op);
op_msg = lrmd_create_op(native->token, op, data, timeout, options);
if (op_msg == NULL) {
return -EINVAL;
}
if (expect_reply) {
rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
} else {
rc = lrmd_send_xml_no_reply(lrmd, op_msg);
goto done;
}
if (rc < 0) {
crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
goto done;
} else if(op_reply == NULL) {
rc = -ENOMSG;
goto done;
}
rc = pcmk_ok;
crm_trace("%s op reply received", op);
if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
rc = -ENOMSG;
goto done;
}
crm_log_xml_trace(op_reply, "Reply");
if (output_data) {
*output_data = op_reply;
op_reply = NULL; /* Prevent subsequent free */
}
done:
if (lrmd_api_is_connected(lrmd) == FALSE) {
crm_err("Executor disconnected");
}
free_xml(op_msg);
free_xml(op_reply);
return rc;
}
static int
lrmd_api_poke_connection(lrmd_t * lrmd)
{
int rc;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
(native->type == pcmk__client_ipc));
free_xml(data);
return rc < 0 ? rc : pcmk_ok;
}
// \return Standard Pacemaker return code
int
lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash)
{
int rc = pcmk_rc_ok;
const char *value;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *data = create_xml_node(NULL, F_LRMD_OPERATION);
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
value = g_hash_table_lookup(hash, "stonith-watchdog-timeout");
if ((value) &&
(stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) {
crm_xml_add(data, F_LRMD_WATCHDOG, value);
}
rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
(native->type == pcmk__client_ipc));
free_xml(data);
return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
}
static int
lrmd_handshake(lrmd_t * lrmd, const char *name)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *reply = NULL;
xmlNode *hello = create_xml_node(NULL, "lrmd_command");
crm_xml_add(hello, F_TYPE, T_LRMD);
crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
crm_xml_add(hello, F_LRMD_CLIENTNAME, name);
crm_xml_add(hello, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
/* advertise that we are a proxy provider */
if (native->proxy_callback) {
pcmk__xe_set_bool_attr(hello, F_LRMD_IS_IPC_PROVIDER, true);
}
rc = lrmd_send_xml(lrmd, hello, -1, &reply);
if (rc < 0) {
crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
rc = -ECOMM;
} else if (reply == NULL) {
crm_err("Did not receive registration reply");
rc = -EPROTO;
} else {
const char *version = crm_element_value(reply, F_LRMD_PROTOCOL_VERSION);
const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);
const char *start_state = crm_element_value(reply, PCMK__XA_NODE_START_STATE);
long long uptime = -1;
crm_element_value_int(reply, F_LRMD_RC, &rc);
/* The remote executor may add its uptime to the XML reply, which is
* useful in handling transient attributes when the connection to the
* remote node unexpectedly drops. If no parameter is given, just
* default to -1.
*/
crm_element_value_ll(reply, PCMK__XA_UPTIME, &uptime);
native->remote->uptime = uptime;
if (start_state) {
native->remote->start_state = strdup(start_state);
}
if (rc == -EPROTO) {
crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
LRMD_PROTOCOL_VERSION, version);
crm_log_xml_err(reply, "Protocol Error");
} else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
crm_err("Invalid registration message: %s", msg_type);
crm_log_xml_err(reply, "Bad reply");
rc = -EPROTO;
} else if (tmp_ticket == NULL) {
crm_err("No registration token provided");
crm_log_xml_err(reply, "Bad reply");
rc = -EPROTO;
} else {
crm_trace("Obtained registration token: %s", tmp_ticket);
native->token = strdup(tmp_ticket);
native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
rc = pcmk_ok;
}
}
free_xml(reply);
free_xml(hello);
if (rc != pcmk_ok) {
lrmd_api_disconnect(lrmd);
}
return rc;
}
static int
lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
struct ipc_client_callbacks lrmd_callbacks = {
.dispatch = lrmd_ipc_dispatch,
.destroy = lrmd_ipc_connection_destroy
};
crm_info("Connecting to executor");
if (fd) {
/* No mainloop */
native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
if (native->ipc != NULL) {
rc = pcmk__connect_generic_ipc(native->ipc);
if (rc == pcmk_rc_ok) {
rc = pcmk__ipc_fd(native->ipc, fd);
}
if (rc != pcmk_rc_ok) {
crm_err("Connection to executor failed: %s", pcmk_rc_str(rc));
rc = -ENOTCONN;
}
}
} else {
native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
native->ipc = mainloop_get_ipc_client(native->source);
}
if (native->ipc == NULL) {
crm_debug("Could not connect to the executor API");
rc = -ENOTCONN;
}
return rc;
}
#ifdef HAVE_GNUTLS_GNUTLS_H
static void
copy_gnutls_datum(gnutls_datum_t *dest, gnutls_datum_t *source)
{
CRM_ASSERT((dest != NULL) && (source != NULL) && (source->data != NULL));
dest->data = gnutls_malloc(source->size);
CRM_ASSERT(dest->data);
memcpy(dest->data, source->data, source->size);
dest->size = source->size;
}
static void
clear_gnutls_datum(gnutls_datum_t *datum)
{
gnutls_free(datum->data);
datum->data = NULL;
datum->size = 0;
}
#define KEY_READ_LEN 256 // Chunk size for reading key from file
// \return Standard Pacemaker return code
static int
read_gnutls_key(const char *location, gnutls_datum_t *key)
{
FILE *stream = NULL;
size_t buf_len = KEY_READ_LEN;
if ((location == NULL) || (key == NULL)) {
return EINVAL;
}
stream = fopen(location, "r");
if (stream == NULL) {
return errno;
}
key->data = gnutls_malloc(buf_len);
key->size = 0;
while (!feof(stream)) {
int next = fgetc(stream);
if (next == EOF) {
if (!feof(stream)) {
crm_warn("Pacemaker Remote key read was partially successful "
"(copy in memory may be corrupted)");
}
break;
}
if (key->size == buf_len) {
buf_len = key->size + KEY_READ_LEN;
key->data = gnutls_realloc(key->data, buf_len);
CRM_ASSERT(key->data);
}
key->data[key->size++] = (unsigned char) next;
}
fclose(stream);
if (key->size == 0) {
clear_gnutls_datum(key);
return ENOKEY;
}
return pcmk_rc_ok;
}
// Cache the most recently used Pacemaker Remote authentication key
struct key_cache_s {
time_t updated; // When cached key was read (valid for 1 minute)
const char *location; // Where cached key was read from
gnutls_datum_t key; // Cached key
};
static bool
key_is_cached(struct key_cache_s *key_cache)
{
return key_cache->updated != 0;
}
static bool
key_cache_expired(struct key_cache_s *key_cache)
{
return (time(NULL) - key_cache->updated) >= 60;
}
static void
clear_key_cache(struct key_cache_s *key_cache)
{
clear_gnutls_datum(&(key_cache->key));
if ((key_cache->updated != 0) || (key_cache->location != NULL)) {
key_cache->updated = 0;
key_cache->location = NULL;
crm_debug("Cleared Pacemaker Remote key cache");
}
}
static void
get_cached_key(struct key_cache_s *key_cache, gnutls_datum_t *key)
{
copy_gnutls_datum(key, &(key_cache->key));
crm_debug("Using cached Pacemaker Remote key from %s",
pcmk__s(key_cache->location, "unknown location"));
}
static void
cache_key(struct key_cache_s *key_cache, gnutls_datum_t *key,
const char *location)
{
key_cache->updated = time(NULL);
key_cache->location = location;
copy_gnutls_datum(&(key_cache->key), key);
crm_debug("Using (and cacheing) Pacemaker Remote key from %s",
pcmk__s(location, "unknown location"));
}
/*!
* \internal
* \brief Get Pacemaker Remote authentication key from file or cache
*
* \param[in] location Path to key file to try (this memory must
* persist across all calls of this function)
* \param[out] key Key from location or cache
*
* \return Standard Pacemaker return code
*/
static int
get_remote_key(const char *location, gnutls_datum_t *key)
{
static struct key_cache_s key_cache = { 0, };
int rc = pcmk_rc_ok;
if ((location == NULL) || (key == NULL)) {
return EINVAL;
}
if (key_is_cached(&key_cache)) {
if (key_cache_expired(&key_cache)) {
clear_key_cache(&key_cache);
} else {
get_cached_key(&key_cache, key);
return pcmk_rc_ok;
}
}
rc = read_gnutls_key(location, key);
if (rc != pcmk_rc_ok) {
return rc;
}
cache_key(&key_cache, key, location);
return pcmk_rc_ok;
}
/*!
* \internal
* \brief Initialize the Pacemaker Remote authentication key
*
* Try loading the Pacemaker Remote authentication key from cache if available,
* otherwise from these locations, in order of preference: the value of the
* PCMK_authkey_location environment variable, if set; the Pacemaker default key
* file location; or (for historical reasons) /etc/corosync/authkey.
*
* \param[out] key Where to store key
*
* \return Standard Pacemaker return code
*/
int
lrmd__init_remote_key(gnutls_datum_t *key)
{
static const char *env_location = NULL;
static bool need_env = true;
int env_rc = pcmk_rc_ok;
int default_rc = pcmk_rc_ok;
int alt_rc = pcmk_rc_ok;
bool env_is_default = false;
bool env_is_fallback = false;
if (need_env) {
- env_location = getenv("PCMK_authkey_location");
+ env_location = pcmk__env_option(PCMK__ENV_AUTHKEY_LOCATION);
need_env = false;
}
// Try location in environment variable, if set
if (env_location != NULL) {
env_rc = get_remote_key(env_location, key);
if (env_rc == pcmk_rc_ok) {
return pcmk_rc_ok;
}
env_is_default = !strcmp(env_location, DEFAULT_REMOTE_KEY_LOCATION);
env_is_fallback = !strcmp(env_location, ALT_REMOTE_KEY_LOCATION);
/* @TODO It would be more secure to fail, rather than fall back to the
* default, if an explicitly set key location is not readable, and it
* would be better to never use the Corosync location as a fallback.
* However, that would break any deployments currently working with the
* fallbacks.
*/
}
// Try default location, if environment wasn't explicitly set to it
if (env_is_default) {
default_rc = env_rc;
} else {
default_rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key);
}
// Try fallback location, if environment wasn't set to it and default failed
if (env_is_fallback) {
alt_rc = env_rc;
} else if (default_rc != pcmk_rc_ok) {
alt_rc = get_remote_key(ALT_REMOTE_KEY_LOCATION, key);
}
// We have all results, so log and return
if ((env_rc != pcmk_rc_ok) && (default_rc != pcmk_rc_ok)
&& (alt_rc != pcmk_rc_ok)) { // Environment set, everything failed
crm_warn("Could not read Pacemaker Remote key from %s (%s%s%s%s%s): %s",
env_location,
env_is_default? "" : "or default location ",
env_is_default? "" : DEFAULT_REMOTE_KEY_LOCATION,
!env_is_default && !env_is_fallback? " " : "",
env_is_fallback? "" : "or fallback location ",
env_is_fallback? "" : ALT_REMOTE_KEY_LOCATION,
pcmk_rc_str(env_rc));
return ENOKEY;
}
if (env_rc != pcmk_rc_ok) { // Environment set but failed, using a default
crm_warn("Could not read Pacemaker Remote key from %s "
"(using %s location %s instead): %s",
env_location,
(default_rc == pcmk_rc_ok)? "default" : "fallback",
(default_rc == pcmk_rc_ok)? DEFAULT_REMOTE_KEY_LOCATION : ALT_REMOTE_KEY_LOCATION,
pcmk_rc_str(env_rc));
return pcmk_rc_ok;
}
if ((default_rc != pcmk_rc_ok) && (alt_rc != pcmk_rc_ok)) {
// Environment unset, defaults failed
crm_warn("Could not read Pacemaker Remote key from default location %s"
" (or fallback location %s): %s",
DEFAULT_REMOTE_KEY_LOCATION, ALT_REMOTE_KEY_LOCATION,
pcmk_rc_str(default_rc));
return ENOKEY;
}
return pcmk_rc_ok; // Environment variable unset, a default worked
}
static void
lrmd_gnutls_global_init(void)
{
static int gnutls_init = 0;
if (!gnutls_init) {
crm_gnutls_global_init();
}
gnutls_init = 1;
}
#endif
static void
report_async_connection_result(lrmd_t * lrmd, int rc)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->callback) {
lrmd_event_data_t event = { 0, };
event.type = lrmd_event_connect;
event.remote_nodename = native->remote_nodename;
event.connection_rc = rc;
native->callback(&event);
}
}
#ifdef HAVE_GNUTLS_GNUTLS_H
static inline int
lrmd__tls_client_handshake(pcmk__remote_t *remote)
{
return pcmk__tls_client_handshake(remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT);
}
/*!
* \internal
* \brief Add trigger and file descriptor mainloop sources for TLS
*
* \param[in,out] lrmd API connection with established TLS session
* \param[in] do_handshake Whether to perform executor handshake
*
* \return Standard Pacemaker return code
*/
static int
add_tls_to_mainloop(lrmd_t *lrmd, bool do_handshake)
{
lrmd_private_t *native = lrmd->lrmd_private;
int rc = pcmk_rc_ok;
char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
native->server, native->port);
struct mainloop_fd_callbacks tls_fd_callbacks = {
.dispatch = lrmd_tls_dispatch,
.destroy = lrmd_tls_connection_destroy,
};
native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
lrmd_tls_dispatch, lrmd);
native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
&tls_fd_callbacks);
/* Async connections lose the client name provided by the API caller, so we
* have to use our generated name here to perform the executor handshake.
*
* @TODO Keep track of the caller-provided name. Perhaps we should be using
* that name in this function instead of generating one anyway.
*/
if (do_handshake) {
rc = lrmd_handshake(lrmd, name);
rc = pcmk_legacy2rc(rc);
}
free(name);
return rc;
}
static void
lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
gnutls_datum_t psk_key = { NULL, 0 };
native->async_timer = 0;
if (rc != pcmk_rc_ok) {
lrmd_tls_connection_destroy(lrmd);
crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
CRM_XS " rc=%d",
native->server, native->port, pcmk_rc_str(rc), rc);
report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
return;
}
/* The TCP connection was successful, so establish the TLS connection.
* @TODO make this async to avoid blocking code in client
*/
native->sock = sock;
rc = lrmd__init_remote_key(&psk_key);
if (rc != pcmk_rc_ok) {
crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
CRM_XS " rc=%d",
native->server, native->port, pcmk_rc_str(rc), rc);
lrmd_tls_connection_destroy(lrmd);
report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
return;
}
gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
gnutls_free(psk_key.data);
native->remote->tls_session = pcmk__new_tls_session(sock, GNUTLS_CLIENT,
GNUTLS_CRD_PSK,
native->psk_cred_c);
if (native->remote->tls_session == NULL) {
lrmd_tls_connection_destroy(lrmd);
report_async_connection_result(lrmd, -EPROTO);
return;
}
if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
crm_warn("Disconnecting after TLS handshake with Pacemaker Remote server %s:%d failed",
native->server, native->port);
gnutls_deinit(*native->remote->tls_session);
gnutls_free(native->remote->tls_session);
native->remote->tls_session = NULL;
lrmd_tls_connection_destroy(lrmd);
report_async_connection_result(lrmd, -EKEYREJECTED);
return;
}
crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
native->server, native->port);
rc = add_tls_to_mainloop(lrmd, true);
report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
}
static int
lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
{
int rc;
int timer_id = 0;
lrmd_private_t *native = lrmd->lrmd_private;
lrmd_gnutls_global_init();
native->sock = -1;
rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
&(native->sock), lrmd, lrmd_tcp_connect_cb);
if (rc != pcmk_rc_ok) {
crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
CRM_XS " rc=%d",
native->server, native->port, pcmk_rc_str(rc), rc);
return pcmk_rc2legacy(rc);
}
native->async_timer = timer_id;
return pcmk_ok;
}
static int
lrmd_tls_connect(lrmd_t * lrmd, int *fd)
{
int rc;
lrmd_private_t *native = lrmd->lrmd_private;
gnutls_datum_t psk_key = { NULL, 0 };
lrmd_gnutls_global_init();
native->sock = -1;
rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
&(native->sock), NULL, NULL);
if (rc != pcmk_rc_ok) {
crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
CRM_XS " rc=%d",
native->server, native->port, pcmk_rc_str(rc), rc);
lrmd_tls_connection_destroy(lrmd);
return -ENOTCONN;
}
rc = lrmd__init_remote_key(&psk_key);
if (rc != pcmk_rc_ok) {
lrmd_tls_connection_destroy(lrmd);
return pcmk_rc2legacy(rc);
}
gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
gnutls_free(psk_key.data);
native->remote->tls_session = pcmk__new_tls_session(native->sock, GNUTLS_CLIENT,
GNUTLS_CRD_PSK,
native->psk_cred_c);
if (native->remote->tls_session == NULL) {
lrmd_tls_connection_destroy(lrmd);
return -EPROTO;
}
if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
crm_err("Session creation for %s:%d failed", native->server, native->port);
gnutls_deinit(*native->remote->tls_session);
gnutls_free(native->remote->tls_session);
native->remote->tls_session = NULL;
lrmd_tls_connection_destroy(lrmd);
return -EKEYREJECTED;
}
crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
native->port);
if (fd) {
*fd = native->sock;
} else {
add_tls_to_mainloop(lrmd, false);
}
return pcmk_ok;
}
#endif
static int
lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
{
int rc = -ENOTCONN;
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
rc = lrmd_ipc_connect(lrmd, fd);
break;
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
rc = lrmd_tls_connect(lrmd, fd);
break;
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
if (rc == pcmk_ok) {
rc = lrmd_handshake(lrmd, name);
}
return rc;
}
static int
lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
CRM_CHECK(native && native->callback, return -EINVAL);
switch (native->type) {
case pcmk__client_ipc:
/* fake async connection with ipc. it should be fast
* enough that we gain very little from async */
rc = lrmd_api_connect(lrmd, name, NULL);
if (!rc) {
report_async_connection_result(lrmd, rc);
}
break;
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
rc = lrmd_tls_connect_async(lrmd, timeout);
if (rc) {
/* connection failed, report rc now */
report_async_connection_result(lrmd, rc);
}
break;
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
return rc;
}
static void
lrmd_ipc_disconnect(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->source != NULL) {
/* Attached to mainloop */
mainloop_del_ipc_client(native->source);
native->source = NULL;
native->ipc = NULL;
} else if (native->ipc) {
/* Not attached to mainloop */
crm_ipc_t *ipc = native->ipc;
native->ipc = NULL;
crm_ipc_close(ipc);
crm_ipc_destroy(ipc);
}
}
#ifdef HAVE_GNUTLS_GNUTLS_H
static void
lrmd_tls_disconnect(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->remote->tls_session) {
gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
gnutls_deinit(*native->remote->tls_session);
gnutls_free(native->remote->tls_session);
native->remote->tls_session = 0;
}
if (native->async_timer) {
g_source_remove(native->async_timer);
native->async_timer = 0;
}
if (native->source != NULL) {
/* Attached to mainloop */
mainloop_del_ipc_client(native->source);
native->source = NULL;
} else if (native->sock) {
close(native->sock);
native->sock = 0;
}
if (native->pending_notify) {
g_list_free_full(native->pending_notify, lrmd_free_xml);
native->pending_notify = NULL;
}
}
#endif
static int
lrmd_api_disconnect(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
int rc = pcmk_ok;
switch (native->type) {
case pcmk__client_ipc:
crm_debug("Disconnecting from local executor");
lrmd_ipc_disconnect(lrmd);
break;
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
crm_debug("Disconnecting from remote executor on %s",
native->remote_nodename);
lrmd_tls_disconnect(lrmd);
break;
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
free(native->token);
native->token = NULL;
free(native->peer_version);
native->peer_version = NULL;
return rc;
}
static int
lrmd_api_register_rsc(lrmd_t * lrmd,
const char *rsc_id,
const char *class,
const char *provider, const char *type, enum lrmd_call_options options)
{
int rc = pcmk_ok;
xmlNode *data = NULL;
if (!class || !type || !rsc_id) {
return -EINVAL;
}
if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
&& (provider == NULL)) {
return -EINVAL;
}
data = create_xml_node(NULL, F_LRMD_RSC);
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
crm_xml_add(data, F_LRMD_CLASS, class);
crm_xml_add(data, F_LRMD_PROVIDER, provider);
crm_xml_add(data, F_LRMD_TYPE, type);
rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
free_xml(data);
return rc;
}
static int
lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
{
int rc = pcmk_ok;
xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
free_xml(data);
return rc;
}
lrmd_rsc_info_t *
lrmd_new_rsc_info(const char *rsc_id, const char *standard,
const char *provider, const char *type)
{
lrmd_rsc_info_t *rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
CRM_ASSERT(rsc_info);
pcmk__str_update(&rsc_info->id, rsc_id);
pcmk__str_update(&rsc_info->standard, standard);
pcmk__str_update(&rsc_info->provider, provider);
pcmk__str_update(&rsc_info->type, type);
return rsc_info;
}
lrmd_rsc_info_t *
lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
{
return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
rsc_info->provider, rsc_info->type);
}
void
lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
{
if (!rsc_info) {
return;
}
free(rsc_info->id);
free(rsc_info->type);
free(rsc_info->standard);
free(rsc_info->provider);
free(rsc_info);
}
static lrmd_rsc_info_t *
lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
{
lrmd_rsc_info_t *rsc_info = NULL;
xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
xmlNode *output = NULL;
const char *class = NULL;
const char *provider = NULL;
const char *type = NULL;
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
free_xml(data);
if (!output) {
return NULL;
}
class = crm_element_value(output, F_LRMD_CLASS);
provider = crm_element_value(output, F_LRMD_PROVIDER);
type = crm_element_value(output, F_LRMD_TYPE);
if (!class || !type) {
free_xml(output);
return NULL;
} else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
&& !provider) {
free_xml(output);
return NULL;
}
rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
free_xml(output);
return rsc_info;
}
void
lrmd_free_op_info(lrmd_op_info_t *op_info)
{
if (op_info) {
free(op_info->rsc_id);
free(op_info->action);
free(op_info->interval_ms_s);
free(op_info->timeout_ms_s);
free(op_info);
}
}
static int
lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms,
enum lrmd_call_options options, GList **output)
{
xmlNode *data = NULL;
xmlNode *output_xml = NULL;
int rc = pcmk_ok;
if (output == NULL) {
return -EINVAL;
}
*output = NULL;
// Send request
if (rsc_id) {
data = create_xml_node(NULL, F_LRMD_RSC);
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
}
rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
timeout_ms, options, TRUE);
if (data) {
free_xml(data);
}
// Process reply
if ((rc != pcmk_ok) || (output_xml == NULL)) {
return rc;
}
for (xmlNode *rsc_xml = first_named_child(output_xml, F_LRMD_RSC);
(rsc_xml != NULL) && (rc == pcmk_ok);
rsc_xml = crm_next_same_xml(rsc_xml)) {
rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
if (rsc_id == NULL) {
crm_err("Could not parse recurring operation information from executor");
continue;
}
for (xmlNode *op_xml = first_named_child(rsc_xml, T_LRMD_RSC_OP);
op_xml != NULL; op_xml = crm_next_same_xml(op_xml)) {
lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));
if (op_info == NULL) {
rc = -ENOMEM;
break;
}
op_info->rsc_id = strdup(rsc_id);
op_info->action = crm_element_value_copy(op_xml, F_LRMD_RSC_ACTION);
op_info->interval_ms_s = crm_element_value_copy(op_xml,
F_LRMD_RSC_INTERVAL);
op_info->timeout_ms_s = crm_element_value_copy(op_xml,
F_LRMD_TIMEOUT);
*output = g_list_prepend(*output, op_info);
}
}
free_xml(output_xml);
return rc;
}
static void
lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
{
lrmd_private_t *native = lrmd->lrmd_private;
native->callback = callback;
}
void
lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
{
lrmd_private_t *native = lrmd->lrmd_private;
native->proxy_callback = callback;
native->proxy_callback_userdata = userdata;
}
void
lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->proxy_callback) {
crm_log_xml_trace(msg, "PROXY_INBOUND");
native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
}
}
int
lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
{
if (lrmd == NULL) {
return -ENOTCONN;
}
crm_xml_add(msg, F_LRMD_OPERATION, CRM_OP_IPC_FWD);
crm_log_xml_trace(msg, "PROXY_OUTBOUND");
return lrmd_send_xml_no_reply(lrmd, msg);
}
static int
stonith_get_metadata(const char *provider, const char *type, char **output)
{
int rc = pcmk_ok;
stonith_t *stonith_api = stonith_api_new();
if (stonith_api == NULL) {
crm_err("Could not get fence agent meta-data: API memory allocation failed");
return -ENOMEM;
}
rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
provider, output, 0);
if ((rc == pcmk_ok) && (*output == NULL)) {
rc = -EIO;
}
stonith_api->cmds->free(stonith_api);
return rc;
}
static int
lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider,
const char *type, char **output,
enum lrmd_call_options options)
{
return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
output, options, NULL);
}
static int
lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard,
const char *provider, const char *type,
char **output, enum lrmd_call_options options,
lrmd_key_value_t *params)
{
svc_action_t *action = NULL;
GHashTable *params_table = NULL;
if (!standard || !type) {
lrmd_key_value_freeall(params);
return -EINVAL;
}
if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
lrmd_key_value_freeall(params);
return stonith_get_metadata(provider, type, output);
}
params_table = pcmk__strkey_table(free, free);
for (const lrmd_key_value_t *param = params; param; param = param->next) {
g_hash_table_insert(params_table, strdup(param->key), strdup(param->value));
}
action = services__create_resource_action(type, standard, provider, type,
PCMK_ACTION_META_DATA, 0,
PCMK_DEFAULT_METADATA_TIMEOUT_MS,
params_table, 0);
lrmd_key_value_freeall(params);
if (action == NULL) {
return -ENOMEM;
}
if (action->rc != PCMK_OCF_UNKNOWN) {
services_action_free(action);
return -EINVAL;
}
if (!services_action_sync(action)) {
crm_err("Failed to retrieve meta-data for %s:%s:%s",
standard, provider, type);
services_action_free(action);
return -EIO;
}
if (!action->stdout_data) {
crm_err("Failed to receive meta-data for %s:%s:%s",
standard, provider, type);
services_action_free(action);
return -EIO;
}
*output = strdup(action->stdout_data);
services_action_free(action);
return pcmk_ok;
}
static int
lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
const char *userdata, guint interval_ms,
int timeout, /* ms */
int start_delay, /* ms */
enum lrmd_call_options options, lrmd_key_value_t * params)
{
int rc = pcmk_ok;
xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
lrmd_key_value_t *tmp = NULL;
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
crm_xml_add(data, F_LRMD_RSC_ACTION, action);
crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);
for (tmp = params; tmp; tmp = tmp->next) {
hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
}
rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
free_xml(data);
lrmd_key_value_freeall(params);
return rc;
}
/* timeout is in ms */
static int
lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
int timeout, lrmd_key_value_t *params)
{
int rc = pcmk_ok;
xmlNode *data = create_xml_node(NULL, F_LRMD_ALERT);
xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
lrmd_key_value_t *tmp = NULL;
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
crm_xml_add(data, F_LRMD_ALERT_ID, alert_id);
crm_xml_add(data, F_LRMD_ALERT_PATH, alert_path);
crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
for (tmp = params; tmp; tmp = tmp->next) {
hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
}
rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
lrmd_opt_notify_orig_only, TRUE);
free_xml(data);
lrmd_key_value_freeall(params);
return rc;
}
static int
lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
guint interval_ms)
{
int rc = pcmk_ok;
xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
crm_xml_add(data, F_LRMD_RSC_ACTION, action);
crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
free_xml(data);
return rc;
}
static int
list_stonith_agents(lrmd_list_t ** resources)
{
int rc = 0;
stonith_t *stonith_api = stonith_api_new();
stonith_key_value_t *stonith_resources = NULL;
stonith_key_value_t *dIter = NULL;
if (stonith_api == NULL) {
crm_err("Could not list fence agents: API memory allocation failed");
return -ENOMEM;
}
stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
&stonith_resources, 0);
stonith_api->cmds->free(stonith_api);
for (dIter = stonith_resources; dIter; dIter = dIter->next) {
rc++;
if (resources) {
*resources = lrmd_list_add(*resources, dIter->value);
}
}
stonith_key_value_freeall(stonith_resources, 1, 0);
return rc;
}
static int
lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
const char *provider)
{
int rc = 0;
int stonith_count = 0; // Initially, whether to include stonith devices
if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
stonith_count = 1;
} else {
GList *gIter = NULL;
GList *agents = resources_list_agents(class, provider);
for (gIter = agents; gIter != NULL; gIter = gIter->next) {
*resources = lrmd_list_add(*resources, (const char *)gIter->data);
rc++;
}
g_list_free_full(agents, free);
if (!class) {
stonith_count = 1;
}
}
if (stonith_count) {
// Now, if stonith devices are included, how many there are
stonith_count = list_stonith_agents(resources);
if (stonith_count > 0) {
rc += stonith_count;
}
}
if (rc == 0) {
crm_notice("No agents found for class %s", class);
rc = -EPROTONOSUPPORT;
}
return rc;
}
static bool
does_provider_have_agent(const char *agent, const char *provider, const char *class)
{
bool found = false;
GList *agents = NULL;
GList *gIter2 = NULL;
agents = resources_list_agents(class, provider);
for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
found = true;
}
}
g_list_free_full(agents, free);
return found;
}
static int
lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
{
int rc = pcmk_ok;
char *provider = NULL;
GList *ocf_providers = NULL;
GList *gIter = NULL;
ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
provider = gIter->data;
if (!agent || does_provider_have_agent(agent, provider,
PCMK_RESOURCE_CLASS_OCF)) {
*providers = lrmd_list_add(*providers, (const char *)gIter->data);
rc++;
}
}
g_list_free_full(ocf_providers, free);
return rc;
}
static int
lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
{
int rc = 0;
GList *standards = NULL;
GList *gIter = NULL;
standards = resources_list_standards();
for (gIter = standards; gIter != NULL; gIter = gIter->next) {
*supported = lrmd_list_add(*supported, (const char *)gIter->data);
rc++;
}
if (list_stonith_agents(NULL) > 0) {
*supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
rc++;
}
g_list_free_full(standards, free);
return rc;
}
/*!
* \internal
* \brief Create an executor API object
*
* \param[out] api Will be set to newly created API object (it is the
* caller's responsibility to free this value with
* lrmd_api_delete() if this function succeeds)
* \param[in] nodename If the object will be used for a remote connection,
* the node name to use in cluster for remote executor
* \param[in] server If the object will be used for a remote connection,
* the resolvable host name to connect to
* \param[in] port If the object will be used for a remote connection,
* port number on \p server to connect to
*
* \return Standard Pacemaker return code
* \note If the caller leaves one of \p nodename or \p server NULL, the other's
* value will be used for both. If the caller leaves both NULL, an API
* object will be created for a local executor connection.
*/
int
lrmd__new(lrmd_t **api, const char *nodename, const char *server, int port)
{
lrmd_private_t *pvt = NULL;
if (api == NULL) {
return EINVAL;
}
*api = NULL;
// Allocate all memory needed
*api = calloc(1, sizeof(lrmd_t));
if (*api == NULL) {
return ENOMEM;
}
pvt = calloc(1, sizeof(lrmd_private_t));
if (pvt == NULL) {
lrmd_api_delete(*api);
*api = NULL;
return ENOMEM;
}
(*api)->lrmd_private = pvt;
// @TODO Do we need to do this for local connections?
pvt->remote = calloc(1, sizeof(pcmk__remote_t));
(*api)->cmds = calloc(1, sizeof(lrmd_api_operations_t));
if ((pvt->remote == NULL) || ((*api)->cmds == NULL)) {
lrmd_api_delete(*api);
*api = NULL;
return ENOMEM;
}
// Set methods
(*api)->cmds->connect = lrmd_api_connect;
(*api)->cmds->connect_async = lrmd_api_connect_async;
(*api)->cmds->is_connected = lrmd_api_is_connected;
(*api)->cmds->poke_connection = lrmd_api_poke_connection;
(*api)->cmds->disconnect = lrmd_api_disconnect;
(*api)->cmds->register_rsc = lrmd_api_register_rsc;
(*api)->cmds->unregister_rsc = lrmd_api_unregister_rsc;
(*api)->cmds->get_rsc_info = lrmd_api_get_rsc_info;
(*api)->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
(*api)->cmds->set_callback = lrmd_api_set_callback;
(*api)->cmds->get_metadata = lrmd_api_get_metadata;
(*api)->cmds->exec = lrmd_api_exec;
(*api)->cmds->cancel = lrmd_api_cancel;
(*api)->cmds->list_agents = lrmd_api_list_agents;
(*api)->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
(*api)->cmds->list_standards = lrmd_api_list_standards;
(*api)->cmds->exec_alert = lrmd_api_exec_alert;
(*api)->cmds->get_metadata_params = lrmd_api_get_metadata_params;
if ((nodename == NULL) && (server == NULL)) {
pvt->type = pcmk__client_ipc;
} else {
#ifdef HAVE_GNUTLS_GNUTLS_H
if (nodename == NULL) {
nodename = server;
} else if (server == NULL) {
server = nodename;
}
pvt->type = pcmk__client_tls;
pvt->remote_nodename = strdup(nodename);
pvt->server = strdup(server);
if ((pvt->remote_nodename == NULL) || (pvt->server == NULL)) {
lrmd_api_delete(*api);
*api = NULL;
return ENOMEM;
}
pvt->port = port;
if (pvt->port == 0) {
pvt->port = crm_default_remote_port();
}
#else
crm_err("Cannot communicate with Pacemaker Remote "
"because GnuTLS is not enabled for this build");
lrmd_api_delete(*api);
*api = NULL;
return EOPNOTSUPP;
#endif
}
return pcmk_rc_ok;
}
lrmd_t *
lrmd_api_new(void)
{
lrmd_t *api = NULL;
CRM_ASSERT(lrmd__new(&api, NULL, NULL, 0) == pcmk_rc_ok);
return api;
}
lrmd_t *
lrmd_remote_api_new(const char *nodename, const char *server, int port)
{
lrmd_t *api = NULL;
CRM_ASSERT(lrmd__new(&api, nodename, server, port) == pcmk_rc_ok);
return api;
}
void
lrmd_api_delete(lrmd_t * lrmd)
{
if (lrmd == NULL) {
return;
}
if (lrmd->cmds != NULL) { // Never NULL, but make static analysis happy
if (lrmd->cmds->disconnect != NULL) { // Also never really NULL
lrmd->cmds->disconnect(lrmd); // No-op if already disconnected
}
free(lrmd->cmds);
}
if (lrmd->lrmd_private != NULL) {
lrmd_private_t *native = lrmd->lrmd_private;
#ifdef HAVE_GNUTLS_GNUTLS_H
free(native->server);
#endif
free(native->remote_nodename);
free(native->remote);
free(native->token);
free(native->peer_version);
free(lrmd->lrmd_private);
}
free(lrmd);
}
struct metadata_cb {
void (*callback)(int pid, const pcmk__action_result_t *result,
void *user_data);
void *user_data;
};
/*!
* \internal
* \brief Process asynchronous metadata completion
*
* \param[in,out] action Metadata action that completed
*/
static void
metadata_complete(svc_action_t *action)
{
struct metadata_cb *metadata_cb = (struct metadata_cb *) action->cb_data;
pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
pcmk__set_result(&result, action->rc, action->status,
services__exit_reason(action));
pcmk__set_result_output(&result, action->stdout_data, action->stderr_data);
metadata_cb->callback(0, &result, metadata_cb->user_data);
result.action_stdout = NULL; // Prevent free, because action owns it
result.action_stderr = NULL; // Prevent free, because action owns it
pcmk__reset_result(&result);
free(metadata_cb);
}
/*!
* \internal
* \brief Retrieve agent metadata asynchronously
*
* \param[in] rsc Resource agent specification
* \param[in] callback Function to call with result (this will always be
* called, whether by this function directly or later
* via the main loop, and on success the metadata will
* be in its result argument's action_stdout)
* \param[in,out] user_data User data to pass to callback
*
* \return Standard Pacemaker return code
* \note This function is not a lrmd_api_operations_t method because it does not
* need an lrmd_t object and does not go through the executor, but
* executes the agent directly.
*/
int
lrmd__metadata_async(const lrmd_rsc_info_t *rsc,
void (*callback)(int pid,
const pcmk__action_result_t *result,
void *user_data),
void *user_data)
{
svc_action_t *action = NULL;
struct metadata_cb *metadata_cb = NULL;
pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
CRM_CHECK(callback != NULL, return EINVAL);
if ((rsc == NULL) || (rsc->standard == NULL) || (rsc->type == NULL)) {
pcmk__set_result(&result, PCMK_OCF_NOT_CONFIGURED,
PCMK_EXEC_ERROR_FATAL,
"Invalid resource specification");
callback(0, &result, user_data);
pcmk__reset_result(&result);
return EINVAL;
}
if (strcmp(rsc->standard, PCMK_RESOURCE_CLASS_STONITH) == 0) {
return stonith__metadata_async(rsc->type,
PCMK_DEFAULT_METADATA_TIMEOUT_MS / 1000,
callback, user_data);
}
action = services__create_resource_action(pcmk__s(rsc->id, rsc->type),
rsc->standard, rsc->provider,
rsc->type,
PCMK_ACTION_META_DATA, 0,
PCMK_DEFAULT_METADATA_TIMEOUT_MS,
NULL, 0);
if (action == NULL) {
pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
"Out of memory");
callback(0, &result, user_data);
pcmk__reset_result(&result);
return ENOMEM;
}
if (action->rc != PCMK_OCF_UNKNOWN) {
pcmk__set_result(&result, action->rc, action->status,
services__exit_reason(action));
callback(0, &result, user_data);
pcmk__reset_result(&result);
services_action_free(action);
return EINVAL;
}
action->cb_data = calloc(1, sizeof(struct metadata_cb));
if (action->cb_data == NULL) {
services_action_free(action);
pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
"Out of memory");
callback(0, &result, user_data);
pcmk__reset_result(&result);
return ENOMEM;
}
metadata_cb = (struct metadata_cb *) action->cb_data;
metadata_cb->callback = callback;
metadata_cb->user_data = user_data;
if (!services_action_async(action, metadata_complete)) {
services_action_free(action);
return pcmk_rc_error; // @TODO Derive from action->rc and ->status
}
// The services library has taken responsibility for action
return pcmk_rc_ok;
}
/*!
* \internal
* \brief Set the result of an executor event
*
* \param[in,out] event Executor event to set
* \param[in] rc OCF exit status of event
* \param[in] op_status Executor status of event
* \param[in] exit_reason Human-friendly description of event
*/
void
lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, int op_status,
const char *exit_reason)
{
if (event == NULL) {
return;
}
event->rc = rc;
event->op_status = op_status;
pcmk__str_update((char **) &event->exit_reason, exit_reason);
}
/*!
* \internal
* \brief Clear an executor event's exit reason, output, and error output
*
* \param[in,out] event Executor event to reset
*/
void
lrmd__reset_result(lrmd_event_data_t *event)
{
if (event == NULL) {
return;
}
free((void *) event->exit_reason);
event->exit_reason = NULL;
free((void *) event->output);
event->output = NULL;
}
/*!
* \internal
* \brief Get the uptime of a remote resource connection
*
* When the cluster connects to a remote resource, part of that resource's
* handshake includes the uptime of the remote resource's connection. This
* uptime is stored in the lrmd_t object.
*
* \return The connection's uptime, or -1 if unknown
*/
time_t
lrmd__uptime(lrmd_t *lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->remote == NULL) {
return -1;
} else {
return native->remote->uptime;
}
}
const char *
lrmd__node_start_state(lrmd_t *lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->remote == NULL) {
return NULL;
} else {
return native->remote->start_state;
}
}
diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
index 02260e8f84..89cbabd4d4 100644
--- a/lib/pengine/bundle.c
+++ b/lib/pengine/bundle.c
@@ -1,2219 +1,2219 @@
/*
* Copyright 2004-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <ctype.h>
#include <stdint.h>
#include <crm/pengine/rules.h>
#include <crm/pengine/status.h>
#include <crm/pengine/internal.h>
#include <crm/msg_xml.h>
#include <crm/common/output.h>
#include <crm/common/xml_internal.h>
#include <pe_status_private.h>
enum pe__bundle_mount_flags {
pe__bundle_mount_none = 0x00,
// mount instance-specific subdirectory rather than source directly
pe__bundle_mount_subdir = 0x01
};
typedef struct {
char *source;
char *target;
char *options;
uint32_t flags; // bitmask of pe__bundle_mount_flags
} pe__bundle_mount_t;
typedef struct {
char *source;
char *target;
} pe__bundle_port_t;
enum pe__container_agent {
PE__CONTAINER_AGENT_UNKNOWN,
PE__CONTAINER_AGENT_DOCKER,
PE__CONTAINER_AGENT_RKT,
PE__CONTAINER_AGENT_PODMAN,
};
#define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
#define PE__CONTAINER_AGENT_DOCKER_S "docker"
#define PE__CONTAINER_AGENT_RKT_S "rkt"
#define PE__CONTAINER_AGENT_PODMAN_S "podman"
typedef struct pe__bundle_variant_data_s {
int promoted_max;
int nreplicas;
int nreplicas_per_host;
char *prefix;
char *image;
const char *ip_last;
char *host_network;
char *host_netmask;
char *control_port;
char *container_network;
char *ip_range_start;
gboolean add_host;
gchar *container_host_options;
char *container_command;
char *launcher_options;
const char *attribute_target;
pcmk_resource_t *child;
GList *replicas; // pe__bundle_replica_t *
GList *ports; // pe__bundle_port_t *
GList *mounts; // pe__bundle_mount_t *
enum pe__container_agent agent_type;
} pe__bundle_variant_data_t;
#define get_bundle_variant_data(data, rsc) \
CRM_ASSERT(rsc != NULL); \
CRM_ASSERT(rsc->variant == pcmk_rsc_variant_bundle); \
CRM_ASSERT(rsc->variant_opaque != NULL); \
data = (pe__bundle_variant_data_t *) rsc->variant_opaque;
/*!
* \internal
* \brief Get maximum number of bundle replicas allowed to run
*
* \param[in] rsc Bundle or bundled resource to check
*
* \return Maximum replicas for bundle corresponding to \p rsc
*/
int
pe__bundle_max(const pcmk_resource_t *rsc)
{
const pe__bundle_variant_data_t *bundle_data = NULL;
get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
return bundle_data->nreplicas;
}
/*!
* \internal
* \brief Get the resource inside a bundle
*
* \param[in] bundle Bundle to check
*
* \return Resource inside \p bundle if any, otherwise NULL
*/
pcmk_resource_t *
pe__bundled_resource(const pcmk_resource_t *rsc)
{
const pe__bundle_variant_data_t *bundle_data = NULL;
get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
return bundle_data->child;
}
/*!
* \internal
* \brief Get containerized resource corresponding to a given bundle container
*
* \param[in] instance Collective instance that might be a bundle container
*
* \return Bundled resource instance inside \p instance if it is a bundle
* container instance, otherwise NULL
*/
const pcmk_resource_t *
pe__get_rsc_in_container(const pcmk_resource_t *instance)
{
const pe__bundle_variant_data_t *data = NULL;
const pcmk_resource_t *top = pe__const_top_resource(instance, true);
if ((top == NULL) || (top->variant != pcmk_rsc_variant_bundle)) {
return NULL;
}
get_bundle_variant_data(data, top);
for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
const pe__bundle_replica_t *replica = iter->data;
if (instance == replica->container) {
return replica->child;
}
}
return NULL;
}
/*!
* \internal
* \brief Check whether a given node is created by a bundle
*
* \param[in] bundle Bundle resource to check
* \param[in] node Node to check
*
* \return true if \p node is an instance of \p bundle, otherwise false
*/
bool
pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
const pcmk_node_t *node)
{
pe__bundle_variant_data_t *bundle_data = NULL;
get_bundle_variant_data(bundle_data, bundle);
for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
pe__bundle_replica_t *replica = iter->data;
if (pe__same_node(node, replica->node)) {
return true;
}
}
return false;
}
/*!
* \internal
* \brief Get the container of a bundle's first replica
*
* \param[in] bundle Bundle resource to get container for
*
* \return Container resource from first replica of \p bundle if any,
* otherwise NULL
*/
pcmk_resource_t *
pe__first_container(const pcmk_resource_t *bundle)
{
const pe__bundle_variant_data_t *bundle_data = NULL;
const pe__bundle_replica_t *replica = NULL;
get_bundle_variant_data(bundle_data, bundle);
if (bundle_data->replicas == NULL) {
return NULL;
}
replica = bundle_data->replicas->data;
return replica->container;
}
/*!
* \internal
* \brief Iterate over bundle replicas
*
* \param[in,out] bundle Bundle to iterate over
* \param[in] fn Function to call for each replica (its return value
* indicates whether to continue iterating)
* \param[in,out] user_data Pointer to pass to \p fn
*/
void
pe__foreach_bundle_replica(pcmk_resource_t *bundle,
bool (*fn)(pe__bundle_replica_t *, void *),
void *user_data)
{
const pe__bundle_variant_data_t *bundle_data = NULL;
get_bundle_variant_data(bundle_data, bundle);
for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
if (!fn((pe__bundle_replica_t *) iter->data, user_data)) {
break;
}
}
}
/*!
* \internal
* \brief Iterate over const bundle replicas
*
* \param[in] bundle Bundle to iterate over
* \param[in] fn Function to call for each replica (its return value
* indicates whether to continue iterating)
* \param[in,out] user_data Pointer to pass to \p fn
*/
void
pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
bool (*fn)(const pe__bundle_replica_t *,
void *),
void *user_data)
{
const pe__bundle_variant_data_t *bundle_data = NULL;
get_bundle_variant_data(bundle_data, bundle);
for (const GList *iter = bundle_data->replicas; iter != NULL;
iter = iter->next) {
if (!fn((const pe__bundle_replica_t *) iter->data, user_data)) {
break;
}
}
}
static char *
next_ip(const char *last_ip)
{
unsigned int oct1 = 0;
unsigned int oct2 = 0;
unsigned int oct3 = 0;
unsigned int oct4 = 0;
int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
if (rc != 4) {
/*@ TODO check for IPv6 */
return NULL;
} else if (oct3 > 253) {
return NULL;
} else if (oct4 > 253) {
++oct3;
oct4 = 1;
} else {
++oct4;
}
return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
}
static void
allocate_ip(pe__bundle_variant_data_t *data, pe__bundle_replica_t *replica,
GString *buffer)
{
if(data->ip_range_start == NULL) {
return;
} else if(data->ip_last) {
replica->ipaddr = next_ip(data->ip_last);
} else {
replica->ipaddr = strdup(data->ip_range_start);
}
data->ip_last = replica->ipaddr;
switch (data->agent_type) {
case PE__CONTAINER_AGENT_DOCKER:
case PE__CONTAINER_AGENT_PODMAN:
if (data->add_host) {
g_string_append_printf(buffer, " --add-host=%s-%d:%s",
data->prefix, replica->offset,
replica->ipaddr);
} else {
g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
replica->ipaddr, data->prefix,
replica->offset);
}
break;
case PE__CONTAINER_AGENT_RKT:
g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
replica->ipaddr, data->prefix,
replica->offset);
break;
default: // PE__CONTAINER_AGENT_UNKNOWN
break;
}
}
static xmlNode *
create_resource(const char *name, const char *provider, const char *kind)
{
xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
crm_xml_add(rsc, XML_ATTR_ID, name);
crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
crm_xml_add(rsc, XML_AGENT_ATTR_PROVIDER, provider);
crm_xml_add(rsc, XML_ATTR_TYPE, kind);
return rsc;
}
/*!
* \internal
* \brief Check whether cluster can manage resource inside container
*
* \param[in,out] data Container variant data
*
* \return TRUE if networking configuration is acceptable, FALSE otherwise
*
* \note The resource is manageable if an IP range or control port has been
* specified. If a control port is used without an IP range, replicas per
* host must be 1.
*/
static bool
valid_network(pe__bundle_variant_data_t *data)
{
if(data->ip_range_start) {
return TRUE;
}
if(data->control_port) {
if(data->nreplicas_per_host > 1) {
pe_err("Specifying the 'control-port' for %s requires 'replicas-per-host=1'", data->prefix);
data->nreplicas_per_host = 1;
// @TODO to be sure:
// pe__clear_resource_flags(rsc, pcmk_rsc_unique);
}
return TRUE;
}
return FALSE;
}
static int
create_ip_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
pe__bundle_replica_t *replica)
{
if(data->ip_range_start) {
char *id = NULL;
xmlNode *xml_ip = NULL;
xmlNode *xml_obj = NULL;
id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
crm_xml_sanitize_id(id);
xml_ip = create_resource(id, "heartbeat", "IPaddr2");
free(id);
xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
crm_xml_set_id(xml_obj, "%s-attributes-%d",
data->prefix, replica->offset);
crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
if(data->host_network) {
crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
}
if(data->host_netmask) {
crm_create_nvpair_xml(xml_obj, NULL,
"cidr_netmask", data->host_netmask);
} else {
crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
}
xml_obj = create_xml_node(xml_ip, "operations");
crm_create_op_xml(xml_obj, ID(xml_ip), PCMK_ACTION_MONITOR, "60s",
NULL);
// TODO: Other ops? Timeouts and intervals from underlying resource?
if (pe__unpack_resource(xml_ip, &replica->ip, parent,
parent->cluster) != pcmk_rc_ok) {
return pcmk_rc_unpack_error;
}
parent->children = g_list_append(parent->children, replica->ip);
}
return pcmk_rc_ok;
}
static const char*
container_agent_str(enum pe__container_agent t)
{
switch (t) {
case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
case PE__CONTAINER_AGENT_RKT: return PE__CONTAINER_AGENT_RKT_S;
case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
default: // PE__CONTAINER_AGENT_UNKNOWN
break;
}
return PE__CONTAINER_AGENT_UNKNOWN_S;
}
static int
create_container_resource(pcmk_resource_t *parent,
const pe__bundle_variant_data_t *data,
pe__bundle_replica_t *replica)
{
char *id = NULL;
xmlNode *xml_container = NULL;
xmlNode *xml_obj = NULL;
// Agent-specific
const char *hostname_opt = NULL;
const char *env_opt = NULL;
const char *agent_str = NULL;
int volid = 0; // rkt-only
GString *buffer = NULL;
GString *dbuffer = NULL;
// Where syntax differences are drop-in replacements, set them now
switch (data->agent_type) {
case PE__CONTAINER_AGENT_DOCKER:
case PE__CONTAINER_AGENT_PODMAN:
hostname_opt = "-h ";
env_opt = "-e ";
break;
case PE__CONTAINER_AGENT_RKT:
hostname_opt = "--hostname=";
env_opt = "--environment=";
break;
default: // PE__CONTAINER_AGENT_UNKNOWN
return pcmk_rc_unpack_error;
}
agent_str = container_agent_str(data->agent_type);
buffer = g_string_sized_new(4096);
id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
replica->offset);
crm_xml_sanitize_id(id);
xml_container = create_resource(id, "heartbeat", agent_str);
free(id);
xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);
crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
g_string_append(buffer, " --restart=no");
}
/* Set a container hostname only if we have an IP to map it to. The user can
* set -h or --uts=host themselves if they want a nicer name for logs, but
* this makes applications happy who need their hostname to match the IP
* they bind to.
*/
if (data->ip_range_start != NULL) {
g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
replica->offset);
}
pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);
if (data->container_network != NULL) {
pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
}
if (data->control_port != NULL) {
- pcmk__g_strcat(buffer, " ", env_opt, "PCMK_remote_port=",
- data->control_port, NULL);
+ pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
+ data->control_port, NULL);
} else {
- g_string_append_printf(buffer, " %sPCMK_remote_port=%d", env_opt,
- DEFAULT_REMOTE_PORT);
+ g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
+ env_opt, DEFAULT_REMOTE_PORT);
}
for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
pe__bundle_mount_t *mount = (pe__bundle_mount_t *) iter->data;
char *source = NULL;
if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
replica->offset);
pcmk__add_separated_word(&dbuffer, 1024, source, ",");
}
switch (data->agent_type) {
case PE__CONTAINER_AGENT_DOCKER:
case PE__CONTAINER_AGENT_PODMAN:
pcmk__g_strcat(buffer,
" -v ", pcmk__s(source, mount->source),
":", mount->target, NULL);
if (mount->options != NULL) {
pcmk__g_strcat(buffer, ":", mount->options, NULL);
}
break;
case PE__CONTAINER_AGENT_RKT:
g_string_append_printf(buffer,
" --volume vol%d,kind=host,"
"source=%s%s%s "
"--mount volume=vol%d,target=%s",
volid, pcmk__s(source, mount->source),
(mount->options != NULL)? "," : "",
pcmk__s(mount->options, ""),
volid, mount->target);
volid++;
break;
default:
break;
}
free(source);
}
for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
pe__bundle_port_t *port = (pe__bundle_port_t *) iter->data;
switch (data->agent_type) {
case PE__CONTAINER_AGENT_DOCKER:
case PE__CONTAINER_AGENT_PODMAN:
if (replica->ipaddr != NULL) {
pcmk__g_strcat(buffer,
" -p ", replica->ipaddr, ":", port->source,
":", port->target, NULL);
} else if (!pcmk__str_eq(data->container_network, "host",
pcmk__str_none)) {
// No need to do port mapping if net == host
pcmk__g_strcat(buffer,
" -p ", port->source, ":", port->target,
NULL);
}
break;
case PE__CONTAINER_AGENT_RKT:
if (replica->ipaddr != NULL) {
pcmk__g_strcat(buffer,
" --port=", port->target,
":", replica->ipaddr, ":", port->source,
NULL);
} else {
pcmk__g_strcat(buffer,
" --port=", port->target, ":", port->source,
NULL);
}
break;
default:
break;
}
}
/* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
* it would cause restarts during rolling upgrades.
*
* In a previous version of the container resource creation logic, if
* data->launcher_options is not NULL, we append
* (" %s", data->launcher_options) even if data->launcher_options is an
* empty string. Likewise for data->container_host_options. Using
*
* pcmk__add_word(buffer, 0, data->launcher_options)
*
* removes that extra trailing space, causing a resource definition change.
*/
if (data->launcher_options != NULL) {
pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
}
if (data->container_host_options != NULL) {
pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
}
crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
(const char *) buffer->str);
g_string_free(buffer, TRUE);
crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
(dbuffer != NULL)? (const char *) dbuffer->str : "");
if (dbuffer != NULL) {
g_string_free(dbuffer, TRUE);
}
if (replica->child != NULL) {
if (data->container_command != NULL) {
crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
data->container_command);
} else {
crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
SBIN_DIR "/pacemaker-remoted");
}
/* TODO: Allow users to specify their own?
*
* We just want to know if the container is alive; we'll monitor the
* child independently.
*/
crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
#if 0
/* @TODO Consider supporting the use case where we can start and stop
* resources, but not proxy local commands (such as setting node
* attributes), by running the local executor in stand-alone mode.
* However, this would probably be better done via ACLs as with other
* Pacemaker Remote nodes.
*/
} else if ((child != NULL) && data->untrusted) {
crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
CRM_DAEMON_DIR "/pacemaker-execd");
crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
#endif
} else {
if (data->container_command != NULL) {
crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
data->container_command);
}
/* TODO: Allow users to specify their own?
*
* We don't know what's in the container, so we just want to know if it
* is alive.
*/
crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
}
xml_obj = create_xml_node(xml_container, "operations");
crm_create_op_xml(xml_obj, ID(xml_container), PCMK_ACTION_MONITOR, "60s",
NULL);
// TODO: Other ops? Timeouts and intervals from underlying resource?
if (pe__unpack_resource(xml_container, &replica->container, parent,
parent->cluster) != pcmk_rc_ok) {
return pcmk_rc_unpack_error;
}
pe__set_resource_flags(replica->container, pcmk_rsc_replica_container);
parent->children = g_list_append(parent->children, replica->container);
return pcmk_rc_ok;
}
/*!
* \brief Ban a node from a resource's (and its children's) allowed nodes list
*
* \param[in,out] rsc Resource to modify
* \param[in] uname Name of node to ban
*/
static void
disallow_node(pcmk_resource_t *rsc, const char *uname)
{
gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);
if (match) {
((pcmk_node_t *) match)->weight = -INFINITY;
((pcmk_node_t *) match)->rsc_discover_mode = pcmk_probe_never;
}
if (rsc->children) {
g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
}
}
static int
create_remote_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
pe__bundle_replica_t *replica)
{
if (replica->child && valid_network(data)) {
GHashTableIter gIter;
pcmk_node_t *node = NULL;
xmlNode *xml_remote = NULL;
char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
char *port_s = NULL;
const char *uname = NULL;
const char *connect_name = NULL;
if (pe_find_resource(parent->cluster->resources, id) != NULL) {
free(id);
// The biggest hammer we have
id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
replica->child->id, replica->offset);
//@TODO return error instead of asserting?
CRM_ASSERT(pe_find_resource(parent->cluster->resources,
id) == NULL);
}
/* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
* connection does not have its own IP is a magic string that we use to
* support nested remotes (i.e. a bundle running on a remote node).
*/
connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
if (data->control_port == NULL) {
port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
}
/* This sets replica->container as replica->remote's container, which is
* similar to what happens with guest nodes. This is how the scheduler
* knows that the bundle node is fenced by recovering the container, and
* that remote should be ordered relative to the container.
*/
xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
NULL, NULL, NULL,
connect_name, (data->control_port?
data->control_port : port_s));
free(port_s);
/* Abandon our created ID, and pull the copy from the XML, because we
* need something that will get freed during scheduler data cleanup to
* use as the node ID and uname.
*/
free(id);
id = NULL;
uname = ID(xml_remote);
/* Ensure a node has been created for the guest (it may have already
* been, if it has a permanent node attribute), and ensure its weight is
* -INFINITY so no other resources can run on it.
*/
node = pe_find_node(parent->cluster->nodes, uname);
if (node == NULL) {
node = pe_create_node(uname, uname, "remote", "-INFINITY",
parent->cluster);
} else {
node->weight = -INFINITY;
}
node->rsc_discover_mode = pcmk_probe_never;
/* unpack_remote_nodes() ensures that each remote node and guest node
* has a pcmk_node_t entry. Ideally, it would do the same for bundle
* nodes. Unfortunately, a bundle has to be mostly unpacked before it's
* obvious what nodes will be needed, so we do it just above.
*
* Worse, that means that the node may have been utilized while
* unpacking other resources, without our weight correction. The most
* likely place for this to happen is when pe__unpack_resource() calls
* resource_location() to set a default score in symmetric clusters.
* This adds a node *copy* to each resource's allowed nodes, and these
* copies will have the wrong weight.
*
* As a hacky workaround, fix those copies here.
*
* @TODO Possible alternative: ensure bundles are unpacked before other
* resources, so the weight is correct before any copies are made.
*/
g_list_foreach(parent->cluster->resources, (GFunc) disallow_node,
(gpointer) uname);
replica->node = pe__copy_node(node);
replica->node->weight = 500;
replica->node->rsc_discover_mode = pcmk_probe_exclusive;
/* Ensure the node shows up as allowed and with the correct discovery set */
if (replica->child->allowed_nodes != NULL) {
g_hash_table_destroy(replica->child->allowed_nodes);
}
replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
g_hash_table_insert(replica->child->allowed_nodes,
(gpointer) replica->node->details->id,
pe__copy_node(replica->node));
{
pcmk_node_t *copy = pe__copy_node(replica->node);
copy->weight = -INFINITY;
g_hash_table_insert(replica->child->parent->allowed_nodes,
(gpointer) replica->node->details->id, copy);
}
if (pe__unpack_resource(xml_remote, &replica->remote, parent,
parent->cluster) != pcmk_rc_ok) {
return pcmk_rc_unpack_error;
}
g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
if (pe__is_guest_or_remote_node(node)) {
/* Remote resources can only run on 'normal' cluster node */
node->weight = -INFINITY;
}
}
replica->node->details->remote_rsc = replica->remote;
// Ensure pe__is_guest_node() functions correctly immediately
replica->remote->container = replica->container;
/* A bundle's #kind is closer to "container" (guest node) than the
* "remote" set by pe_create_node().
*/
g_hash_table_insert(replica->node->details->attrs,
strdup(CRM_ATTR_KIND), strdup("container"));
/* One effect of this is that setup_container() will add
* replica->remote to replica->container's fillers, which will make
* pe__resource_contains_guest_node() true for replica->container.
*
* replica->child does NOT get added to replica->container's fillers.
* The only noticeable effect if it did would be for its fail count to
* be taken into account when checking replica->container's migration
* threshold.
*/
parent->children = g_list_append(parent->children, replica->remote);
}
return pcmk_rc_ok;
}
static int
create_replica_resources(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
pe__bundle_replica_t *replica)
{
int rc = pcmk_rc_ok;
rc = create_container_resource(parent, data, replica);
if (rc != pcmk_rc_ok) {
return rc;
}
rc = create_ip_resource(parent, data, replica);
if (rc != pcmk_rc_ok) {
return rc;
}
rc = create_remote_resource(parent, data, replica);
if (rc != pcmk_rc_ok) {
return rc;
}
if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
}
if (replica->remote != NULL) {
/*
* Allow the remote connection resource to be allocated to a
* different node than the one on which the container is active.
*
* This makes it possible to have Pacemaker Remote nodes running
* containers with pacemaker-remoted inside in order to start
* services inside those containers.
*/
pe__set_resource_flags(replica->remote,
pcmk_rsc_remote_nesting_allowed);
}
return rc;
}
static void
mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
const char *target, const char *options, uint32_t flags)
{
pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));
CRM_ASSERT(mount != NULL);
mount->source = strdup(source);
mount->target = strdup(target);
pcmk__str_update(&mount->options, options);
mount->flags = flags;
bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
}
static void
mount_free(pe__bundle_mount_t *mount)
{
free(mount->source);
free(mount->target);
free(mount->options);
free(mount);
}
static void
port_free(pe__bundle_port_t *port)
{
free(port->source);
free(port->target);
free(port);
}
static pe__bundle_replica_t *
replica_for_remote(pcmk_resource_t *remote)
{
pcmk_resource_t *top = remote;
pe__bundle_variant_data_t *bundle_data = NULL;
if (top == NULL) {
return NULL;
}
while (top->parent != NULL) {
top = top->parent;
}
get_bundle_variant_data(bundle_data, top);
for (GList *gIter = bundle_data->replicas; gIter != NULL;
gIter = gIter->next) {
pe__bundle_replica_t *replica = gIter->data;
if (replica->remote == remote) {
return replica;
}
}
CRM_LOG_ASSERT(FALSE);
return NULL;
}
bool
pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
{
const char *value;
GHashTable *params = NULL;
if (rsc == NULL) {
return false;
}
// Use NULL node since pcmk__bundle_expand() uses that to set value
params = pe_rsc_params(rsc, NULL, rsc->cluster);
value = g_hash_table_lookup(params, XML_RSC_ATTR_REMOTE_RA_ADDR);
return pcmk__str_eq(value, "#uname", pcmk__str_casei)
&& xml_contains_remote_node(rsc->xml);
}
const char *
pe__add_bundle_remote_name(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler,
xmlNode *xml, const char *field)
{
// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
pcmk_node_t *node = NULL;
pe__bundle_replica_t *replica = NULL;
if (!pe__bundle_needs_remote_name(rsc)) {
return NULL;
}
replica = replica_for_remote(rsc);
if (replica == NULL) {
return NULL;
}
node = replica->container->allocated_to;
if (node == NULL) {
/* If it won't be running anywhere after the
* transition, go with where it's running now.
*/
node = pe__current_node(replica->container);
}
if(node == NULL) {
crm_trace("Cannot determine address for bundle connection %s", rsc->id);
return NULL;
}
crm_trace("Setting address for bundle connection %s to bundle host %s",
rsc->id, pe__node_name(node));
if(xml != NULL && field != NULL) {
crm_xml_add(xml, field, node->details->uname);
}
return node->details->uname;
}
#define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do { \
flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
"Bundle mount", ID(mount_xml), flags, \
(flags_to_set), #flags_to_set); \
} while (0)
gboolean
pe__unpack_bundle(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler)
{
const char *value = NULL;
xmlNode *xml_obj = NULL;
xmlNode *xml_resource = NULL;
pe__bundle_variant_data_t *bundle_data = NULL;
bool need_log_mount = TRUE;
CRM_ASSERT(rsc != NULL);
pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);
bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
rsc->variant_opaque = bundle_data;
bundle_data->prefix = strdup(rsc->id);
xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
if (xml_obj != NULL) {
bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
} else {
xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
if (xml_obj != NULL) {
bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
} else {
xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
if (xml_obj != NULL) {
bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
} else {
return FALSE;
}
}
}
// Use 0 for default, minimum, and invalid promoted-max
value = crm_element_value(xml_obj, PCMK_META_PROMOTED_MAX);
if (value == NULL) {
// @COMPAT deprecated since 2.0.0
value = crm_element_value(xml_obj, "masters");
}
pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);
// Default replicas to promoted-max if it was specified and 1 otherwise
value = crm_element_value(xml_obj, "replicas");
if ((value == NULL) && (bundle_data->promoted_max > 0)) {
bundle_data->nreplicas = bundle_data->promoted_max;
} else {
pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
}
/*
* Communication between containers on the same host via the
* floating IPs only works if the container is started with:
* --userland-proxy=false --ip-masq=false
*/
value = crm_element_value(xml_obj, "replicas-per-host");
pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
if (bundle_data->nreplicas_per_host == 1) {
pe__clear_resource_flags(rsc, pcmk_rsc_unique);
}
bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
bundle_data->image = crm_element_value_copy(xml_obj, "image");
bundle_data->container_network = crm_element_value_copy(xml_obj, "network");
xml_obj = first_named_child(rsc->xml, "network");
if(xml_obj) {
bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
value = crm_element_value(xml_obj, "add-host");
if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
bundle_data->add_host = TRUE;
}
for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
xml_child = pcmk__xe_next(xml_child)) {
pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
port->source = crm_element_value_copy(xml_child, "port");
if(port->source == NULL) {
port->source = crm_element_value_copy(xml_child, "range");
} else {
port->target = crm_element_value_copy(xml_child, "internal-port");
}
if(port->source != NULL && strlen(port->source) > 0) {
if(port->target == NULL) {
port->target = strdup(port->source);
}
bundle_data->ports = g_list_append(bundle_data->ports, port);
} else {
pe_err("Invalid port directive %s", ID(xml_child));
port_free(port);
}
}
}
xml_obj = first_named_child(rsc->xml, "storage");
for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
xml_child = pcmk__xe_next(xml_child)) {
const char *source = crm_element_value(xml_child, "source-dir");
const char *target = crm_element_value(xml_child, "target-dir");
const char *options = crm_element_value(xml_child, "options");
int flags = pe__bundle_mount_none;
if (source == NULL) {
source = crm_element_value(xml_child, "source-dir-root");
pe__set_bundle_mount_flags(xml_child, flags,
pe__bundle_mount_subdir);
}
if (source && target) {
mount_add(bundle_data, source, target, options, flags);
if (strcmp(target, "/var/log") == 0) {
need_log_mount = FALSE;
}
} else {
pe_err("Invalid mount directive %s", ID(xml_child));
}
}
xml_obj = first_named_child(rsc->xml, "primitive");
if (xml_obj && valid_network(bundle_data)) {
char *value = NULL;
xmlNode *xml_set = NULL;
xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);
/* @COMPAT We no longer use the <master> tag, but we need to keep it as
* part of the resource name, so that bundles don't restart in a rolling
* upgrade. (It also avoids needing to change regression tests.)
*/
crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
(bundle_data->promoted_max? "master"
: (const char *)xml_resource->name));
xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);
crm_create_nvpair_xml(xml_set, NULL,
XML_RSC_ATTR_ORDERED, XML_BOOLEAN_TRUE);
value = pcmk__itoa(bundle_data->nreplicas);
crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
free(value);
value = pcmk__itoa(bundle_data->nreplicas_per_host);
crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
free(value);
crm_create_nvpair_xml(xml_set, NULL, XML_RSC_ATTR_UNIQUE,
pcmk__btoa(bundle_data->nreplicas_per_host > 1));
if (bundle_data->promoted_max) {
crm_create_nvpair_xml(xml_set, NULL,
XML_RSC_ATTR_PROMOTABLE, XML_BOOLEAN_TRUE);
value = pcmk__itoa(bundle_data->promoted_max);
crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
free(value);
}
//crm_xml_add(xml_obj, XML_ATTR_ID, bundle_data->prefix);
add_node_copy(xml_resource, xml_obj);
} else if(xml_obj) {
pe_err("Cannot control %s inside %s without either ip-range-start or control-port",
rsc->id, ID(xml_obj));
return FALSE;
}
if(xml_resource) {
int lpc = 0;
GList *childIter = NULL;
pe__bundle_port_t *port = NULL;
GString *buffer = NULL;
if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
scheduler) != pcmk_rc_ok) {
return FALSE;
}
/* Currently, we always map the default authentication key location
* into the same location inside the container.
*
* Ideally, we would respect the host's PCMK_authkey_location, but:
* - it may be different on different nodes;
* - the actual connection will do extra checking to make sure the key
* file exists and is readable, that we can't do here on the DC
* - tools such as crm_resource and crm_simulate may not have the same
* environment variables as the cluster, causing operation digests to
* differ
*
* Always using the default location inside the container is fine,
* because we control the pacemaker_remote environment, and it avoids
* having to pass another environment variable to the container.
*
* @TODO A better solution may be to have only pacemaker_remote use the
* environment variable, and have the cluster nodes use a new
* cluster option for key location. This would introduce the limitation
* of the location being the same on all cluster nodes, but that's
* reasonable.
*/
mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
if (need_log_mount) {
mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
pe__bundle_mount_subdir);
}
port = calloc(1, sizeof(pe__bundle_port_t));
if(bundle_data->control_port) {
port->source = strdup(bundle_data->control_port);
} else {
/* If we wanted to respect PCMK_remote_port, we could use
* crm_default_remote_port() here and elsewhere in this file instead
* of DEFAULT_REMOTE_PORT.
*
* However, it gains nothing, since we control both the container
* environment and the connection resource parameters, and the user
* can use a different port if desired by setting control-port.
*/
port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
}
port->target = strdup(port->source);
bundle_data->ports = g_list_append(bundle_data->ports, port);
buffer = g_string_sized_new(1024);
for (childIter = bundle_data->child->children; childIter != NULL;
childIter = childIter->next) {
pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
replica->child = childIter->data;
replica->child->exclusive_discover = TRUE;
replica->offset = lpc++;
// Ensure the child's notify gets set based on the underlying primitive's value
if (pcmk_is_set(replica->child->flags, pcmk_rsc_notify)) {
pe__set_resource_flags(bundle_data->child, pcmk_rsc_notify);
}
allocate_ip(bundle_data, replica, buffer);
bundle_data->replicas = g_list_append(bundle_data->replicas,
replica);
bundle_data->attribute_target = g_hash_table_lookup(replica->child->meta,
XML_RSC_ATTR_TARGET);
}
bundle_data->container_host_options = g_string_free(buffer, FALSE);
if (bundle_data->attribute_target) {
g_hash_table_replace(rsc->meta, strdup(XML_RSC_ATTR_TARGET),
strdup(bundle_data->attribute_target));
g_hash_table_replace(bundle_data->child->meta,
strdup(XML_RSC_ATTR_TARGET),
strdup(bundle_data->attribute_target));
}
} else {
// Just a naked container, no pacemaker-remote
GString *buffer = g_string_sized_new(1024);
for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
replica->offset = lpc;
allocate_ip(bundle_data, replica, buffer);
bundle_data->replicas = g_list_append(bundle_data->replicas,
replica);
}
bundle_data->container_host_options = g_string_free(buffer, FALSE);
}
for (GList *gIter = bundle_data->replicas; gIter != NULL;
gIter = gIter->next) {
pe__bundle_replica_t *replica = gIter->data;
if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
pe_err("Failed unpacking resource %s", rsc->id);
rsc->fns->free(rsc);
return FALSE;
}
/* Utilization needs special handling for bundles. It makes no sense for
* the inner primitive to have utilization, because it is tied
* one-to-one to the guest node created by the container resource -- and
* there's no way to set capacities for that guest node anyway.
*
* What the user really wants is to configure utilization for the
* container. However, the schema only allows utilization for
* primitives, and the container resource is implicit anyway, so the
* user can *only* configure utilization for the inner primitive. If
* they do, move the primitive's utilization values to the container.
*
* @TODO This means that bundles without an inner primitive can't have
* utilization. An alternative might be to allow utilization values in
* the top-level bundle XML in the schema, and copy those to each
* container.
*/
if (replica->child != NULL) {
GHashTable *empty = replica->container->utilization;
replica->container->utilization = replica->child->utilization;
replica->child->utilization = empty;
}
}
if (bundle_data->child) {
rsc->children = g_list_append(rsc->children, bundle_data->child);
}
return TRUE;
}
static int
replica_resource_active(pcmk_resource_t *rsc, gboolean all)
{
if (rsc) {
gboolean child_active = rsc->fns->active(rsc, all);
if (child_active && !all) {
return TRUE;
} else if (!child_active && all) {
return FALSE;
}
}
return -1;
}
gboolean
pe__bundle_active(pcmk_resource_t *rsc, gboolean all)
{
pe__bundle_variant_data_t *bundle_data = NULL;
GList *iter = NULL;
get_bundle_variant_data(bundle_data, rsc);
for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
pe__bundle_replica_t *replica = iter->data;
int rsc_active;
rsc_active = replica_resource_active(replica->ip, all);
if (rsc_active >= 0) {
return (gboolean) rsc_active;
}
rsc_active = replica_resource_active(replica->child, all);
if (rsc_active >= 0) {
return (gboolean) rsc_active;
}
rsc_active = replica_resource_active(replica->container, all);
if (rsc_active >= 0) {
return (gboolean) rsc_active;
}
rsc_active = replica_resource_active(replica->remote, all);
if (rsc_active >= 0) {
return (gboolean) rsc_active;
}
}
/* If "all" is TRUE, we've already checked that no resources were inactive,
* so return TRUE; if "all" is FALSE, we didn't find any active resources,
* so return FALSE.
*/
return all;
}
/*!
* \internal
* \brief Find the bundle replica corresponding to a given node
*
* \param[in] bundle Top-level bundle resource
* \param[in] node Node to search for
*
* \return Bundle replica if found, NULL otherwise
*/
pcmk_resource_t *
pe__find_bundle_replica(const pcmk_resource_t *bundle, const pcmk_node_t *node)
{
pe__bundle_variant_data_t *bundle_data = NULL;
CRM_ASSERT(bundle && node);
get_bundle_variant_data(bundle_data, bundle);
for (GList *gIter = bundle_data->replicas; gIter != NULL;
gIter = gIter->next) {
pe__bundle_replica_t *replica = gIter->data;
CRM_ASSERT(replica && replica->node);
if (replica->node->details == node->details) {
return replica->child;
}
}
return NULL;
}
/*!
* \internal
* \deprecated This function will be removed in a future release
*/
static void
print_rsc_in_list(pcmk_resource_t *rsc, const char *pre_text, long options,
void *print_data)
{
if (rsc != NULL) {
if (options & pe_print_html) {
status_print("<li>");
}
rsc->fns->print(rsc, pre_text, options, print_data);
if (options & pe_print_html) {
status_print("</li>\n");
}
}
}
/*!
* \internal
* \deprecated This function will be removed in a future release
*/
static void
bundle_print_xml(pcmk_resource_t *rsc, const char *pre_text, long options,
void *print_data)
{
pe__bundle_variant_data_t *bundle_data = NULL;
char *child_text = NULL;
CRM_CHECK(rsc != NULL, return);
if (pre_text == NULL) {
pre_text = "";
}
child_text = crm_strdup_printf("%s ", pre_text);
get_bundle_variant_data(bundle_data, rsc);
status_print("%s<bundle ", pre_text);
status_print(XML_ATTR_ID "=\"%s\" ", rsc->id);
status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
status_print("image=\"%s\" ", bundle_data->image);
status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_unique));
status_print("managed=\"%s\" ",
pe__rsc_bool_str(rsc, pcmk_rsc_managed));
status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_failed));
status_print(">\n");
for (GList *gIter = bundle_data->replicas; gIter != NULL;
gIter = gIter->next) {
pe__bundle_replica_t *replica = gIter->data;
CRM_ASSERT(replica);
status_print("%s <replica " XML_ATTR_ID "=\"%d\">\n",
pre_text, replica->offset);
print_rsc_in_list(replica->ip, child_text, options, print_data);
print_rsc_in_list(replica->child, child_text, options, print_data);
print_rsc_in_list(replica->container, child_text, options, print_data);
print_rsc_in_list(replica->remote, child_text, options, print_data);
status_print("%s </replica>\n", pre_text);
}
status_print("%s</bundle>\n", pre_text);
free(child_text);
}
PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
"GList *")
int
pe__bundle_xml(pcmk__output_t *out, va_list args)
{
uint32_t show_opts = va_arg(args, uint32_t);
pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
GList *only_node = va_arg(args, GList *);
GList *only_rsc = va_arg(args, GList *);
pe__bundle_variant_data_t *bundle_data = NULL;
int rc = pcmk_rc_no_output;
gboolean printed_header = FALSE;
gboolean print_everything = TRUE;
const char *desc = NULL;
CRM_ASSERT(rsc != NULL);
get_bundle_variant_data(bundle_data, rsc);
if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
return rc;
}
print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
for (GList *gIter = bundle_data->replicas; gIter != NULL;
gIter = gIter->next) {
pe__bundle_replica_t *replica = gIter->data;
char *id = NULL;
gboolean print_ip, print_child, print_ctnr, print_remote;
CRM_ASSERT(replica);
if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
continue;
}
print_ip = replica->ip != NULL &&
!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
print_child = replica->child != NULL &&
!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
print_remote = replica->remote != NULL &&
!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
continue;
}
if (!printed_header) {
printed_header = TRUE;
desc = pe__resource_description(rsc, show_opts);
rc = pe__name_and_nvpairs_xml(out, true, "bundle", 8,
"id", rsc->id,
"type", container_agent_str(bundle_data->agent_type),
"image", bundle_data->image,
"unique", pe__rsc_bool_str(rsc, pcmk_rsc_unique),
"maintenance",
pe__rsc_bool_str(rsc, pcmk_rsc_maintenance),
"managed", pe__rsc_bool_str(rsc, pcmk_rsc_managed),
"failed", pe__rsc_bool_str(rsc, pcmk_rsc_failed),
"description", desc);
CRM_ASSERT(rc == pcmk_rc_ok);
}
id = pcmk__itoa(replica->offset);
rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, "id", id);
free(id);
CRM_ASSERT(rc == pcmk_rc_ok);
if (print_ip) {
out->message(out, crm_map_element_name(replica->ip->xml), show_opts,
replica->ip, only_node, only_rsc);
}
if (print_child) {
out->message(out, crm_map_element_name(replica->child->xml), show_opts,
replica->child, only_node, only_rsc);
}
if (print_ctnr) {
out->message(out, crm_map_element_name(replica->container->xml), show_opts,
replica->container, only_node, only_rsc);
}
if (print_remote) {
out->message(out, crm_map_element_name(replica->remote->xml), show_opts,
replica->remote, only_node, only_rsc);
}
pcmk__output_xml_pop_parent(out); // replica
}
if (printed_header) {
pcmk__output_xml_pop_parent(out); // bundle
}
return rc;
}
static void
pe__bundle_replica_output_html(pcmk__output_t *out, pe__bundle_replica_t *replica,
pcmk_node_t *node, uint32_t show_opts)
{
pcmk_resource_t *rsc = replica->child;
int offset = 0;
char buffer[LINE_MAX];
if(rsc == NULL) {
rsc = replica->container;
}
if (replica->remote) {
offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
rsc_printable_id(replica->remote));
} else {
offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
rsc_printable_id(replica->container));
}
if (replica->ipaddr) {
offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
replica->ipaddr);
}
pe__common_output_html(out, rsc, buffer, node, show_opts);
}
/*!
* \internal
* \brief Get a string describing a resource's unmanaged state or lack thereof
*
* \param[in] rsc Resource to describe
*
* \return A string indicating that a resource is in maintenance mode or
* otherwise unmanaged, or an empty string otherwise
*/
static const char *
get_unmanaged_str(const pcmk_resource_t *rsc)
{
if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
return " (maintenance)";
}
if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
return " (unmanaged)";
}
return "";
}
PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
"GList *")
int
pe__bundle_html(pcmk__output_t *out, va_list args)
{
uint32_t show_opts = va_arg(args, uint32_t);
pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
GList *only_node = va_arg(args, GList *);
GList *only_rsc = va_arg(args, GList *);
const char *desc = NULL;
pe__bundle_variant_data_t *bundle_data = NULL;
int rc = pcmk_rc_no_output;
gboolean print_everything = TRUE;
CRM_ASSERT(rsc != NULL);
get_bundle_variant_data(bundle_data, rsc);
desc = pe__resource_description(rsc, show_opts);
if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
return rc;
}
print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
for (GList *gIter = bundle_data->replicas; gIter != NULL;
gIter = gIter->next) {
pe__bundle_replica_t *replica = gIter->data;
gboolean print_ip, print_child, print_ctnr, print_remote;
CRM_ASSERT(replica);
if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
continue;
}
print_ip = replica->ip != NULL &&
!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
print_child = replica->child != NULL &&
!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
print_remote = replica->remote != NULL &&
!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
(print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
/* The text output messages used below require pe_print_implicit to
* be set to do anything.
*/
uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
(bundle_data->nreplicas > 1)? " set" : "",
rsc->id, bundle_data->image,
pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
get_unmanaged_str(rsc));
if (pcmk__list_of_multiple(bundle_data->replicas)) {
out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
}
if (print_ip) {
out->message(out, crm_map_element_name(replica->ip->xml),
new_show_opts, replica->ip, only_node, only_rsc);
}
if (print_child) {
out->message(out, crm_map_element_name(replica->child->xml),
new_show_opts, replica->child, only_node, only_rsc);
}
if (print_ctnr) {
out->message(out, crm_map_element_name(replica->container->xml),
new_show_opts, replica->container, only_node, only_rsc);
}
if (print_remote) {
out->message(out, crm_map_element_name(replica->remote->xml),
new_show_opts, replica->remote, only_node, only_rsc);
}
if (pcmk__list_of_multiple(bundle_data->replicas)) {
out->end_list(out);
}
} else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
continue;
} else {
PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
(bundle_data->nreplicas > 1)? " set" : "",
rsc->id, bundle_data->image,
pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
get_unmanaged_str(rsc));
pe__bundle_replica_output_html(out, replica, pe__current_node(replica->container),
show_opts);
}
}
PCMK__OUTPUT_LIST_FOOTER(out, rc);
return rc;
}
static void
pe__bundle_replica_output_text(pcmk__output_t *out, pe__bundle_replica_t *replica,
pcmk_node_t *node, uint32_t show_opts)
{
const pcmk_resource_t *rsc = replica->child;
int offset = 0;
char buffer[LINE_MAX];
if(rsc == NULL) {
rsc = replica->container;
}
if (replica->remote) {
offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
rsc_printable_id(replica->remote));
} else {
offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
rsc_printable_id(replica->container));
}
if (replica->ipaddr) {
offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
replica->ipaddr);
}
pe__common_output_text(out, rsc, buffer, node, show_opts);
}
PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
"GList *")
int
pe__bundle_text(pcmk__output_t *out, va_list args)
{
uint32_t show_opts = va_arg(args, uint32_t);
pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
GList *only_node = va_arg(args, GList *);
GList *only_rsc = va_arg(args, GList *);
const char *desc = NULL;
pe__bundle_variant_data_t *bundle_data = NULL;
int rc = pcmk_rc_no_output;
gboolean print_everything = TRUE;
desc = pe__resource_description(rsc, show_opts);
get_bundle_variant_data(bundle_data, rsc);
CRM_ASSERT(rsc != NULL);
if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
return rc;
}
print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
for (GList *gIter = bundle_data->replicas; gIter != NULL;
gIter = gIter->next) {
pe__bundle_replica_t *replica = gIter->data;
gboolean print_ip, print_child, print_ctnr, print_remote;
CRM_ASSERT(replica);
if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
continue;
}
print_ip = replica->ip != NULL &&
!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
print_child = replica->child != NULL &&
!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
print_remote = replica->remote != NULL &&
!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
(print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
/* The text output messages used below require pe_print_implicit to
* be set to do anything.
*/
uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
(bundle_data->nreplicas > 1)? " set" : "",
rsc->id, bundle_data->image,
pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
get_unmanaged_str(rsc));
if (pcmk__list_of_multiple(bundle_data->replicas)) {
out->list_item(out, NULL, "Replica[%d]", replica->offset);
}
out->begin_list(out, NULL, NULL, NULL);
if (print_ip) {
out->message(out, crm_map_element_name(replica->ip->xml),
new_show_opts, replica->ip, only_node, only_rsc);
}
if (print_child) {
out->message(out, crm_map_element_name(replica->child->xml),
new_show_opts, replica->child, only_node, only_rsc);
}
if (print_ctnr) {
out->message(out, crm_map_element_name(replica->container->xml),
new_show_opts, replica->container, only_node, only_rsc);
}
if (print_remote) {
out->message(out, crm_map_element_name(replica->remote->xml),
new_show_opts, replica->remote, only_node, only_rsc);
}
out->end_list(out);
} else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
continue;
} else {
PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
(bundle_data->nreplicas > 1)? " set" : "",
rsc->id, bundle_data->image,
pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
get_unmanaged_str(rsc));
pe__bundle_replica_output_text(out, replica, pe__current_node(replica->container),
show_opts);
}
}
PCMK__OUTPUT_LIST_FOOTER(out, rc);
return rc;
}
/*!
* \internal
* \deprecated This function will be removed in a future release
*/
static void
print_bundle_replica(pe__bundle_replica_t *replica, const char *pre_text,
long options, void *print_data)
{
pcmk_node_t *node = NULL;
pcmk_resource_t *rsc = replica->child;
int offset = 0;
char buffer[LINE_MAX];
if(rsc == NULL) {
rsc = replica->container;
}
if (replica->remote) {
offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
rsc_printable_id(replica->remote));
} else {
offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
rsc_printable_id(replica->container));
}
if (replica->ipaddr) {
offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
replica->ipaddr);
}
node = pe__current_node(replica->container);
common_print(rsc, pre_text, buffer, node, options, print_data);
}
/*!
* \internal
* \deprecated This function will be removed in a future release
*/
void
pe__print_bundle(pcmk_resource_t *rsc, const char *pre_text, long options,
void *print_data)
{
pe__bundle_variant_data_t *bundle_data = NULL;
char *child_text = NULL;
CRM_CHECK(rsc != NULL, return);
if (options & pe_print_xml) {
bundle_print_xml(rsc, pre_text, options, print_data);
return;
}
get_bundle_variant_data(bundle_data, rsc);
if (pre_text == NULL) {
pre_text = " ";
}
status_print("%sContainer bundle%s: %s [%s]%s%s\n",
pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
rsc->id, bundle_data->image,
pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " (unmanaged)");
if (options & pe_print_html) {
status_print("<br />\n<ul>\n");
}
for (GList *gIter = bundle_data->replicas; gIter != NULL;
gIter = gIter->next) {
pe__bundle_replica_t *replica = gIter->data;
CRM_ASSERT(replica);
if (options & pe_print_html) {
status_print("<li>");
}
if (pcmk_is_set(options, pe_print_implicit)) {
child_text = crm_strdup_printf(" %s", pre_text);
if (pcmk__list_of_multiple(bundle_data->replicas)) {
status_print(" %sReplica[%d]\n", pre_text, replica->offset);
}
if (options & pe_print_html) {
status_print("<br />\n<ul>\n");
}
print_rsc_in_list(replica->ip, child_text, options, print_data);
print_rsc_in_list(replica->container, child_text, options, print_data);
print_rsc_in_list(replica->remote, child_text, options, print_data);
print_rsc_in_list(replica->child, child_text, options, print_data);
if (options & pe_print_html) {
status_print("</ul>\n");
}
} else {
child_text = crm_strdup_printf("%s ", pre_text);
print_bundle_replica(replica, child_text, options, print_data);
}
free(child_text);
if (options & pe_print_html) {
status_print("</li>\n");
}
}
if (options & pe_print_html) {
status_print("</ul>\n");
}
}
static void
free_bundle_replica(pe__bundle_replica_t *replica)
{
if (replica == NULL) {
return;
}
if (replica->node) {
free(replica->node);
replica->node = NULL;
}
if (replica->ip) {
free_xml(replica->ip->xml);
replica->ip->xml = NULL;
replica->ip->fns->free(replica->ip);
replica->ip = NULL;
}
if (replica->container) {
free_xml(replica->container->xml);
replica->container->xml = NULL;
replica->container->fns->free(replica->container);
replica->container = NULL;
}
if (replica->remote) {
free_xml(replica->remote->xml);
replica->remote->xml = NULL;
replica->remote->fns->free(replica->remote);
replica->remote = NULL;
}
free(replica->ipaddr);
free(replica);
}
void
pe__free_bundle(pcmk_resource_t *rsc)
{
pe__bundle_variant_data_t *bundle_data = NULL;
CRM_CHECK(rsc != NULL, return);
get_bundle_variant_data(bundle_data, rsc);
pe_rsc_trace(rsc, "Freeing %s", rsc->id);
free(bundle_data->prefix);
free(bundle_data->image);
free(bundle_data->control_port);
free(bundle_data->host_network);
free(bundle_data->host_netmask);
free(bundle_data->ip_range_start);
free(bundle_data->container_network);
free(bundle_data->launcher_options);
free(bundle_data->container_command);
g_free(bundle_data->container_host_options);
g_list_free_full(bundle_data->replicas,
(GDestroyNotify) free_bundle_replica);
g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
g_list_free(rsc->children);
if(bundle_data->child) {
free_xml(bundle_data->child->xml);
bundle_data->child->xml = NULL;
bundle_data->child->fns->free(bundle_data->child);
}
common_free(rsc);
}
enum rsc_role_e
pe__bundle_resource_state(const pcmk_resource_t *rsc, gboolean current)
{
enum rsc_role_e container_role = pcmk_role_unknown;
return container_role;
}
/*!
* \brief Get the number of configured replicas in a bundle
*
* \param[in] rsc Bundle resource
*
* \return Number of configured replicas, or 0 on error
*/
int
pe_bundle_replicas(const pcmk_resource_t *rsc)
{
if ((rsc == NULL) || (rsc->variant != pcmk_rsc_variant_bundle)) {
return 0;
} else {
pe__bundle_variant_data_t *bundle_data = NULL;
get_bundle_variant_data(bundle_data, rsc);
return bundle_data->nreplicas;
}
}
void
pe__count_bundle(pcmk_resource_t *rsc)
{
pe__bundle_variant_data_t *bundle_data = NULL;
get_bundle_variant_data(bundle_data, rsc);
for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
pe__bundle_replica_t *replica = item->data;
if (replica->ip) {
replica->ip->fns->count(replica->ip);
}
if (replica->child) {
replica->child->fns->count(replica->child);
}
if (replica->container) {
replica->container->fns->count(replica->container);
}
if (replica->remote) {
replica->remote->fns->count(replica->remote);
}
}
}
gboolean
pe__bundle_is_filtered(const pcmk_resource_t *rsc, GList *only_rsc,
gboolean check_parent)
{
gboolean passes = FALSE;
pe__bundle_variant_data_t *bundle_data = NULL;
if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
passes = TRUE;
} else {
get_bundle_variant_data(bundle_data, rsc);
for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
pe__bundle_replica_t *replica = gIter->data;
if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
passes = TRUE;
break;
} else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
passes = TRUE;
break;
} else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
passes = TRUE;
break;
} else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
passes = TRUE;
break;
}
}
}
return !passes;
}
/*!
* \internal
* \brief Get a list of a bundle's containers
*
* \param[in] bundle Bundle resource
*
* \return Newly created list of \p bundle's containers
* \note It is the caller's responsibility to free the result with
* g_list_free().
*/
GList *
pe__bundle_containers(const pcmk_resource_t *bundle)
{
GList *containers = NULL;
const pe__bundle_variant_data_t *data = NULL;
get_bundle_variant_data(data, bundle);
for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
pe__bundle_replica_t *replica = iter->data;
containers = g_list_append(containers, replica->container);
}
return containers;
}
// Bundle implementation of resource_object_functions_t:active_node()
pcmk_node_t *
pe__bundle_active_node(const pcmk_resource_t *rsc, unsigned int *count_all,
unsigned int *count_clean)
{
pcmk_node_t *active = NULL;
pcmk_node_t *node = NULL;
pcmk_resource_t *container = NULL;
GList *containers = NULL;
GList *iter = NULL;
GHashTable *nodes = NULL;
const pe__bundle_variant_data_t *data = NULL;
if (count_all != NULL) {
*count_all = 0;
}
if (count_clean != NULL) {
*count_clean = 0;
}
if (rsc == NULL) {
return NULL;
}
/* For the purposes of this method, we only care about where the bundle's
* containers are active, so build a list of active containers.
*/
get_bundle_variant_data(data, rsc);
for (iter = data->replicas; iter != NULL; iter = iter->next) {
pe__bundle_replica_t *replica = iter->data;
if (replica->container->running_on != NULL) {
containers = g_list_append(containers, replica->container);
}
}
if (containers == NULL) {
return NULL;
}
/* If the bundle has only a single active container, just use that
* container's method. If live migration is ever supported for bundle
* containers, this will allow us to prefer the migration source when there
* is only one container and it is migrating. For now, this just lets us
* avoid creating the nodes table.
*/
if (pcmk__list_of_1(containers)) {
container = containers->data;
node = container->fns->active_node(container, count_all, count_clean);
g_list_free(containers);
return node;
}
// Add all containers' active nodes to a hash table (for uniqueness)
nodes = g_hash_table_new(NULL, NULL);
for (iter = containers; iter != NULL; iter = iter->next) {
container = iter->data;
for (GList *node_iter = container->running_on; node_iter != NULL;
node_iter = node_iter->next) {
node = node_iter->data;
// If insert returns true, we haven't counted this node yet
if (g_hash_table_insert(nodes, (gpointer) node->details,
(gpointer) node)
&& !pe__count_active_node(rsc, node, &active, count_all,
count_clean)) {
goto done;
}
}
}
done:
g_list_free(containers);
g_hash_table_destroy(nodes);
return active;
}
/*!
* \internal
* \brief Get maximum bundle resource instances per node
*
* \param[in] rsc Bundle resource to check
*
* \return Maximum number of \p rsc instances that can be active on one node
*/
unsigned int
pe__bundle_max_per_node(const pcmk_resource_t *rsc)
{
pe__bundle_variant_data_t *bundle_data = NULL;
get_bundle_variant_data(bundle_data, rsc);
CRM_ASSERT(bundle_data->nreplicas_per_host >= 0);
return (unsigned int) bundle_data->nreplicas_per_host;
}
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index 0045aeedaa..6f6ed69946 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -1,2236 +1,2236 @@
/*
* Copyright 2004-2023 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <crm_resource.h>
#include <crm/common/ipc_attrd_internal.h>
#include <crm/common/ipc_controld.h>
#include <crm/common/lists_internal.h>
#include <crm/services_internal.h>
static GList *
build_node_info_list(const pcmk_resource_t *rsc)
{
GList *retval = NULL;
for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
const pcmk_resource_t *child = (const pcmk_resource_t *) iter->data;
for (const GList *iter2 = child->running_on;
iter2 != NULL; iter2 = iter2->next) {
const pcmk_node_t *node = (const pcmk_node_t *) iter2->data;
node_info_t *ni = calloc(1, sizeof(node_info_t));
ni->node_name = node->details->uname;
ni->promoted = pcmk_is_set(rsc->flags, pcmk_rsc_promotable) &&
child->fns->state(child, TRUE) == pcmk_role_promoted;
retval = g_list_prepend(retval, ni);
}
}
return retval;
}
GList *
cli_resource_search(pcmk_resource_t *rsc, const char *requested_name,
pcmk_scheduler_t *scheduler)
{
GList *retval = NULL;
const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);
if (pe_rsc_is_clone(rsc)) {
retval = build_node_info_list(rsc);
/* The anonymous clone children's common ID is supplied */
} else if (pe_rsc_is_clone(parent)
&& !pcmk_is_set(rsc->flags, pcmk_rsc_unique)
&& rsc->clone_name
&& pcmk__str_eq(requested_name, rsc->clone_name, pcmk__str_casei)
&& !pcmk__str_eq(requested_name, rsc->id, pcmk__str_casei)) {
retval = build_node_info_list(parent);
} else if (rsc->running_on != NULL) {
for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
pcmk_node_t *node = (pcmk_node_t *) iter->data;
node_info_t *ni = calloc(1, sizeof(node_info_t));
ni->node_name = node->details->uname;
ni->promoted = (rsc->fns->state(rsc, TRUE) == pcmk_role_promoted);
retval = g_list_prepend(retval, ni);
}
}
return retval;
}
// \return Standard Pacemaker return code
static int
find_resource_attr(pcmk__output_t *out, cib_t * the_cib, const char *attr,
const char *rsc, const char *attr_set_type, const char *set_name,
const char *attr_id, const char *attr_name, char **value)
{
int rc = pcmk_rc_ok;
xmlNode *xml_search = NULL;
GString *xpath = NULL;
const char *xpath_base = NULL;
if(value) {
*value = NULL;
}
if(the_cib == NULL) {
return ENOTCONN;
}
xpath_base = pcmk_cib_xpath_for(XML_CIB_TAG_RESOURCES);
if (xpath_base == NULL) {
crm_err(XML_CIB_TAG_RESOURCES " CIB element not known (bug?)");
return ENOMSG;
}
xpath = g_string_sized_new(1024);
pcmk__g_strcat(xpath,
xpath_base, "//*[@" XML_ATTR_ID "=\"", rsc, "\"]", NULL);
if (attr_set_type != NULL) {
pcmk__g_strcat(xpath, "/", attr_set_type, NULL);
if (set_name != NULL) {
pcmk__g_strcat(xpath, "[@" XML_ATTR_ID "=\"", set_name, "\"]",
NULL);
}
}
g_string_append(xpath, "//" XML_CIB_TAG_NVPAIR "[");
if (attr_id != NULL) {
pcmk__g_strcat(xpath, "@" XML_ATTR_ID "=\"", attr_id, "\"", NULL);
}
if (attr_name != NULL) {
if (attr_id != NULL) {
g_string_append(xpath, " and ");
}
pcmk__g_strcat(xpath, "@" XML_NVPAIR_ATTR_NAME "=\"", attr_name, "\"",
NULL);
}
g_string_append_c(xpath, ']');
rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
cib_sync_call | cib_scope_local | cib_xpath);
rc = pcmk_legacy2rc(rc);
if (rc != pcmk_rc_ok) {
goto done;
}
crm_log_xml_debug(xml_search, "Match");
if (xml_search->children != NULL) {
xmlNode *child = NULL;
rc = ENOTUNIQ;
out->info(out, "Multiple attributes match name=%s", attr_name);
for (child = pcmk__xml_first_child(xml_search); child != NULL;
child = pcmk__xml_next(child)) {
out->info(out, " Value: %s \t(id=%s)",
crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
}
out->spacer(out);
} else if(value) {
pcmk__str_update(value, crm_element_value(xml_search, attr));
}
done:
g_string_free(xpath, TRUE);
free_xml(xml_search);
return rc;
}
/* PRIVATE. Use the find_matching_attr_resources instead. */
static void
find_matching_attr_resources_recursive(pcmk__output_t *out,
GList /* <pcmk_resource_t*> */ **result,
pcmk_resource_t *rsc, const char *rsc_id,
const char * attr_set, const char * attr_set_type,
const char * attr_id, const char * attr_name,
cib_t * cib, const char * cmd, int depth)
{
int rc = pcmk_rc_ok;
char *lookup_id = clone_strip(rsc->id);
char *local_attr_id = NULL;
/* visit the children */
for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
find_matching_attr_resources_recursive(out, result,
(pcmk_resource_t *) gIter->data,
rsc_id, attr_set, attr_set_type,
attr_id, attr_name, cib, cmd, depth+1);
/* do it only once for clones */
if (rsc->variant == pcmk_rsc_variant_clone) {
break;
}
}
rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
attr_set, attr_id, attr_name, &local_attr_id);
/* Post-order traversal.
* The root is always on the list and it is the last item. */
if((0 == depth) || (pcmk_rc_ok == rc)) {
/* push the head */
*result = g_list_append(*result, rsc);
}
free(local_attr_id);
free(lookup_id);
}
/* The result is a linearized pre-ordered tree of resources. */
static GList/*<pcmk_resource_t*>*/ *
find_matching_attr_resources(pcmk__output_t *out, pcmk_resource_t *rsc,
const char * rsc_id, const char * attr_set,
const char * attr_set_type, const char * attr_id,
const char * attr_name, cib_t * cib, const char * cmd,
gboolean force)
{
int rc = pcmk_rc_ok;
char *lookup_id = NULL;
char *local_attr_id = NULL;
GList * result = NULL;
/* If --force is used, update only the requested resource (clone or primitive).
* Otherwise, if the primitive has the attribute, use that.
* Otherwise use the clone. */
if(force == TRUE) {
return g_list_append(result, rsc);
}
if ((rsc->parent != NULL)
&& (rsc->parent->variant == pcmk_rsc_variant_clone)) {
int rc = pcmk_rc_ok;
char *local_attr_id = NULL;
rc = find_resource_attr(out, cib, XML_ATTR_ID, rsc_id, attr_set_type,
attr_set, attr_id, attr_name, &local_attr_id);
free(local_attr_id);
if(rc != pcmk_rc_ok) {
rsc = rsc->parent;
out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
cmd, attr_name, rsc->id, rsc_id);
}
return g_list_append(result, rsc);
} else if ((rsc->parent == NULL) && (rsc->children != NULL)
&& (rsc->variant == pcmk_rsc_variant_clone)) {
pcmk_resource_t *child = rsc->children->data;
if (child->variant == pcmk_rsc_variant_primitive) {
lookup_id = clone_strip(child->id); /* Could be a cloned group! */
rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
attr_set, attr_id, attr_name, &local_attr_id);
if(rc == pcmk_rc_ok) {
rsc = child;
out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
attr_name, lookup_id, cmd, rsc_id);
}
free(local_attr_id);
free(lookup_id);
}
return g_list_append(result, rsc);
}
/* If the resource is a group ==> children inherit the attribute if defined. */
find_matching_attr_resources_recursive(out, &result, rsc, rsc_id, attr_set,
attr_set_type, attr_id, attr_name,
cib, cmd, 0);
return result;
}
// \return Standard Pacemaker return code
int
cli_resource_update_attribute(pcmk_resource_t *rsc, const char *requested_name,
const char *attr_set, const char *attr_set_type,
const char *attr_id, const char *attr_name,
const char *attr_value, gboolean recursive,
cib_t *cib, int cib_options, gboolean force)
{
pcmk__output_t *out = rsc->cluster->priv;
int rc = pcmk_rc_ok;
char *found_attr_id = NULL;
GList/*<pcmk_resource_t*>*/ *resources = NULL;
const char *top_id = pe__const_top_resource(rsc, false)->id;
if ((attr_id == NULL) && !force) {
find_resource_attr(out, cib, XML_ATTR_ID, top_id, NULL, NULL, NULL,
attr_name, NULL);
}
if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
if (!force) {
rc = find_resource_attr(out, cib, XML_ATTR_ID, top_id,
XML_TAG_META_SETS, attr_set, attr_id,
attr_name, &found_attr_id);
if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
out->err(out,
"WARNING: There is already a meta attribute "
"for '%s' called '%s' (id=%s)",
top_id, attr_name, found_attr_id);
out->err(out,
" Delete '%s' first or use the force option "
"to override", found_attr_id);
}
free(found_attr_id);
if (rc == pcmk_rc_ok) {
return ENOTUNIQ;
}
}
resources = g_list_append(resources, rsc);
} else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
crm_xml_add(rsc->xml, attr_name, attr_value);
CRM_ASSERT(cib != NULL);
rc = cib->cmds->replace(cib, XML_CIB_TAG_RESOURCES, rsc->xml,
cib_options);
rc = pcmk_legacy2rc(rc);
if (rc == pcmk_rc_ok) {
out->info(out, "Set attribute: name=%s value=%s",
attr_name, attr_value);
}
return rc;
} else {
resources = find_matching_attr_resources(out, rsc, requested_name,
attr_set, attr_set_type,
attr_id, attr_name, cib,
"update", force);
}
/* If the user specified attr_set or attr_id, the intent is to modify a
* single resource, which will be the last item in the list.
*/
if ((attr_set != NULL) || (attr_id != NULL)) {
GList *last = g_list_last(resources);
resources = g_list_remove_link(resources, last);
g_list_free(resources);
resources = last;
}
for (GList *iter = resources; iter != NULL; iter = iter->next) {
char *lookup_id = NULL;
char *local_attr_set = NULL;
const char *rsc_attr_id = attr_id;
const char *rsc_attr_set = attr_set;
xmlNode *xml_top = NULL;
xmlNode *xml_obj = NULL;
found_attr_id = NULL;
rsc = (pcmk_resource_t *) iter->data;
lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
attr_set, attr_id, attr_name, &found_attr_id);
switch (rc) {
case pcmk_rc_ok:
crm_debug("Found a match for name=%s: id=%s",
attr_name, found_attr_id);
rsc_attr_id = found_attr_id;
break;
case ENXIO:
if (rsc_attr_set == NULL) {
local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
attr_set_type);
rsc_attr_set = local_attr_set;
}
if (rsc_attr_id == NULL) {
found_attr_id = crm_strdup_printf("%s-%s",
rsc_attr_set, attr_name);
rsc_attr_id = found_attr_id;
}
xml_top = create_xml_node(NULL, (const char *) rsc->xml->name);
crm_xml_add(xml_top, XML_ATTR_ID, lookup_id);
xml_obj = create_xml_node(xml_top, attr_set_type);
crm_xml_add(xml_obj, XML_ATTR_ID, rsc_attr_set);
break;
default:
free(lookup_id);
free(found_attr_id);
g_list_free(resources);
return rc;
}
xml_obj = crm_create_nvpair_xml(xml_obj, rsc_attr_id, attr_name,
attr_value);
if (xml_top == NULL) {
xml_top = xml_obj;
}
crm_log_xml_debug(xml_top, "Update");
rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top,
cib_options);
rc = pcmk_legacy2rc(rc);
if (rc == pcmk_rc_ok) {
out->info(out, "Set '%s' option: id=%s%s%s%s%s value=%s",
lookup_id, found_attr_id,
((rsc_attr_set == NULL)? "" : " set="),
pcmk__s(rsc_attr_set, ""),
((attr_name == NULL)? "" : " name="),
pcmk__s(attr_name, ""), attr_value);
}
free_xml(xml_top);
free(lookup_id);
free(found_attr_id);
free(local_attr_set);
if (recursive
&& pcmk__str_eq(attr_set_type, XML_TAG_META_SETS,
pcmk__str_casei)) {
GList *lpc = NULL;
static bool need_init = true;
if (need_init) {
need_init = false;
pcmk__unpack_constraints(rsc->cluster);
pe__clear_resource_flags_on_all(rsc->cluster,
pcmk_rsc_detect_loop);
}
/* We want to set the attribute only on resources explicitly
* colocated with this one, so we use rsc->rsc_cons_lhs directly
* rather than the with_this_colocations() method.
*/
pe__set_resource_flags(rsc, pcmk_rsc_detect_loop);
for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
pcmk__colocation_t *cons = (pcmk__colocation_t *) lpc->data;
crm_debug("Checking %s %d", cons->id, cons->score);
if (!pcmk_is_set(cons->dependent->flags, pcmk_rsc_detect_loop)
&& (cons->score > 0)) {
crm_debug("Setting %s=%s for dependent resource %s",
attr_name, attr_value, cons->dependent->id);
cli_resource_update_attribute(cons->dependent,
cons->dependent->id, NULL,
attr_set_type, NULL,
attr_name, attr_value,
recursive, cib, cib_options,
force);
}
}
}
}
g_list_free(resources);
return rc;
}
// \return Standard Pacemaker return code
int
cli_resource_delete_attribute(pcmk_resource_t *rsc, const char *requested_name,
const char *attr_set, const char *attr_set_type,
const char *attr_id, const char *attr_name,
cib_t *cib, int cib_options, gboolean force)
{
pcmk__output_t *out = rsc->cluster->priv;
int rc = pcmk_rc_ok;
GList/*<pcmk_resource_t*>*/ *resources = NULL;
if ((attr_id == NULL) && !force) {
find_resource_attr(out, cib, XML_ATTR_ID,
pe__const_top_resource(rsc, false)->id, NULL,
NULL, NULL, attr_name, NULL);
}
if (pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
resources = find_matching_attr_resources(out, rsc, requested_name,
attr_set, attr_set_type,
attr_id, attr_name, cib,
"delete", force);
} else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
xml_remove_prop(rsc->xml, attr_name);
CRM_ASSERT(cib != NULL);
rc = cib->cmds->replace(cib, XML_CIB_TAG_RESOURCES, rsc->xml,
cib_options);
rc = pcmk_legacy2rc(rc);
if (rc == pcmk_rc_ok) {
out->info(out, "Deleted attribute: %s", attr_name);
}
return rc;
} else {
resources = g_list_append(resources, rsc);
}
for (GList *iter = resources; iter != NULL; iter = iter->next) {
char *lookup_id = NULL;
xmlNode *xml_obj = NULL;
char *found_attr_id = NULL;
const char *rsc_attr_id = attr_id;
rsc = (pcmk_resource_t *) iter->data;
lookup_id = clone_strip(rsc->id);
rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
attr_set, attr_id, attr_name, &found_attr_id);
switch (rc) {
case pcmk_rc_ok:
break;
case ENXIO:
free(lookup_id);
rc = pcmk_rc_ok;
continue;
default:
free(lookup_id);
g_list_free(resources);
return rc;
}
if (rsc_attr_id == NULL) {
rsc_attr_id = found_attr_id;
}
xml_obj = crm_create_nvpair_xml(NULL, rsc_attr_id, attr_name, NULL);
crm_log_xml_debug(xml_obj, "Delete");
CRM_ASSERT(cib);
rc = cib->cmds->remove(cib, XML_CIB_TAG_RESOURCES, xml_obj,
cib_options);
rc = pcmk_legacy2rc(rc);
if (rc == pcmk_rc_ok) {
out->info(out, "Deleted '%s' option: id=%s%s%s%s%s",
lookup_id, found_attr_id,
((attr_set == NULL)? "" : " set="),
pcmk__s(attr_set, ""),
((attr_name == NULL)? "" : " name="),
pcmk__s(attr_name, ""));
}
free(lookup_id);
free_xml(xml_obj);
free(found_attr_id);
}
g_list_free(resources);
return rc;
}
// \return Standard Pacemaker return code
static int
send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
const char *host_uname, const char *rsc_id,
pcmk_scheduler_t *scheduler)
{
pcmk__output_t *out = scheduler->priv;
const char *router_node = host_uname;
const char *rsc_api_id = NULL;
const char *rsc_long_id = NULL;
const char *rsc_class = NULL;
const char *rsc_provider = NULL;
const char *rsc_type = NULL;
bool cib_only = false;
pcmk_resource_t *rsc = pe_find_resource(scheduler->resources, rsc_id);
if (rsc == NULL) {
out->err(out, "Resource %s not found", rsc_id);
return ENXIO;
} else if (rsc->variant != pcmk_rsc_variant_primitive) {
out->err(out, "We can only process primitive resources, not %s", rsc_id);
return EINVAL;
}
rsc_class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
rsc_provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
rsc_type = crm_element_value(rsc->xml, XML_ATTR_TYPE);
if ((rsc_class == NULL) || (rsc_type == NULL)) {
out->err(out, "Resource %s does not have a class and type", rsc_id);
return EINVAL;
}
{
pcmk_node_t *node = pe_find_node(scheduler->nodes, host_uname);
if (node == NULL) {
out->err(out, "Node %s not found", host_uname);
return pcmk_rc_node_unknown;
}
if (!(node->details->online)) {
if (do_fail_resource) {
out->err(out, "Node %s is not online", host_uname);
return ENOTCONN;
} else {
cib_only = true;
}
}
if (!cib_only && pe__is_guest_or_remote_node(node)) {
node = pe__current_node(node->details->remote_rsc);
if (node == NULL) {
out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
host_uname);
return ENOTCONN;
}
router_node = node->details->uname;
}
}
if (rsc->clone_name) {
rsc_api_id = rsc->clone_name;
rsc_long_id = rsc->id;
} else {
rsc_api_id = rsc->id;
}
if (do_fail_resource) {
return pcmk_controld_api_fail(controld_api, host_uname, router_node,
rsc_api_id, rsc_long_id,
rsc_class, rsc_provider, rsc_type);
} else {
return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
rsc_api_id, rsc_long_id, rsc_class,
rsc_provider, rsc_type, cib_only);
}
}
/*!
* \internal
* \brief Get resource name as used in failure-related node attributes
*
* \param[in] rsc Resource to check
*
* \return Newly allocated string containing resource's fail name
* \note The caller is responsible for freeing the result.
*/
static inline char *
rsc_fail_name(const pcmk_resource_t *rsc)
{
const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);
if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
return strdup(name);
}
return clone_strip(name);
}
// \return Standard Pacemaker return code
static int
clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname,
const char *rsc_id, pcmk_scheduler_t *scheduler)
{
int rc = pcmk_rc_ok;
/* Erase the resource's entire LRM history in the CIB, even if we're only
* clearing a single operation's fail count. If we erased only entries for a
* single operation, we might wind up with a wrong idea of the current
* resource state, and we might not re-probe the resource.
*/
rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, scheduler);
if (rc != pcmk_rc_ok) {
return rc;
}
crm_trace("Processing %d mainloop inputs",
pcmk_controld_api_replies_expected(controld_api));
while (g_main_context_iteration(NULL, FALSE)) {
crm_trace("Processed mainloop input, %d still remaining",
pcmk_controld_api_replies_expected(controld_api));
}
return rc;
}
// \return Standard Pacemaker return code
static int
clear_rsc_failures(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
const char *node_name, const char *rsc_id, const char *operation,
const char *interval_spec, pcmk_scheduler_t *scheduler)
{
int rc = pcmk_rc_ok;
const char *failed_value = NULL;
const char *failed_id = NULL;
const char *interval_ms_s = NULL;
GHashTable *rscs = NULL;
GHashTableIter iter;
/* Create a hash table to use as a set of resources to clean. This lets us
* clean each resource only once (per node) regardless of how many failed
* operations it has.
*/
rscs = pcmk__strkey_table(NULL, NULL);
// Normalize interval to milliseconds for comparison to history entry
if (operation) {
interval_ms_s = crm_strdup_printf("%u",
crm_parse_interval_spec(interval_spec));
}
for (xmlNode *xml_op = pcmk__xml_first_child(scheduler->failed);
xml_op != NULL;
xml_op = pcmk__xml_next(xml_op)) {
failed_id = crm_element_value(xml_op, XML_LRM_ATTR_RSCID);
if (failed_id == NULL) {
// Malformed history entry, should never happen
continue;
}
// No resource specified means all resources match
if (rsc_id) {
pcmk_resource_t *fail_rsc = NULL;
fail_rsc = pe_find_resource_with_flags(scheduler->resources,
failed_id,
pcmk_rsc_match_history
|pcmk_rsc_match_anon_basename);
if (!fail_rsc || !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
continue;
}
}
// Host name should always have been provided by this point
failed_value = crm_element_value(xml_op, XML_ATTR_UNAME);
if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
continue;
}
// No operation specified means all operations match
if (operation) {
failed_value = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
continue;
}
// Interval (if operation was specified) defaults to 0 (not all)
failed_value = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
continue;
}
}
g_hash_table_add(rscs, (gpointer) failed_id);
}
g_hash_table_iter_init(&iter, rscs);
while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
crm_debug("Erasing failures of %s on %s", failed_id, node_name);
rc = clear_rsc_history(controld_api, node_name, failed_id, scheduler);
if (rc != pcmk_rc_ok) {
return rc;
}
}
g_hash_table_destroy(rscs);
return rc;
}
// \return Standard Pacemaker return code
static int
clear_rsc_fail_attrs(const pcmk_resource_t *rsc, const char *operation,
const char *interval_spec, const pcmk_node_t *node)
{
int rc = pcmk_rc_ok;
int attr_options = pcmk__node_attr_none;
char *rsc_name = rsc_fail_name(rsc);
if (pe__is_guest_or_remote_node(node)) {
attr_options |= pcmk__node_attr_remote;
}
rc = pcmk__attrd_api_clear_failures(NULL, node->details->uname, rsc_name,
operation, interval_spec, NULL,
attr_options);
free(rsc_name);
return rc;
}
// \return Standard Pacemaker return code
int
cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
const pcmk_resource_t *rsc, const char *operation,
const char *interval_spec, bool just_failures,
pcmk_scheduler_t *scheduler, gboolean force)
{
pcmk__output_t *out = scheduler->priv;
int rc = pcmk_rc_ok;
pcmk_node_t *node = NULL;
if (rsc == NULL) {
return ENXIO;
} else if (rsc->children) {
for (const GList *lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
const pcmk_resource_t *child = (const pcmk_resource_t *) lpc->data;
rc = cli_resource_delete(controld_api, host_uname, child, operation,
interval_spec, just_failures, scheduler,
force);
if (rc != pcmk_rc_ok) {
return rc;
}
}
return pcmk_rc_ok;
} else if (host_uname == NULL) {
GList *lpc = NULL;
GList *nodes = g_hash_table_get_values(rsc->known_on);
if(nodes == NULL && force) {
nodes = pcmk__copy_node_list(scheduler->nodes, false);
} else if(nodes == NULL && rsc->exclusive_discover) {
GHashTableIter iter;
pcmk_node_t *node = NULL;
g_hash_table_iter_init(&iter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
if(node->weight >= 0) {
nodes = g_list_prepend(nodes, node);
}
}
} else if(nodes == NULL) {
nodes = g_hash_table_get_values(rsc->allowed_nodes);
}
for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
node = (pcmk_node_t *) lpc->data;
if (node->details->online) {
rc = cli_resource_delete(controld_api, node->details->uname, rsc,
operation, interval_spec, just_failures,
scheduler, force);
}
if (rc != pcmk_rc_ok) {
g_list_free(nodes);
return rc;
}
}
g_list_free(nodes);
return pcmk_rc_ok;
}
node = pe_find_node(scheduler->nodes, host_uname);
if (node == NULL) {
out->err(out, "Unable to clean up %s because node %s not found",
rsc->id, host_uname);
return ENODEV;
}
if (!node->details->rsc_discovery_enabled) {
out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
rsc->id, host_uname);
return EOPNOTSUPP;
}
if (controld_api == NULL) {
out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
rsc->id, host_uname);
return pcmk_rc_ok;
}
rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
if (rc != pcmk_rc_ok) {
out->err(out, "Unable to clean up %s failures on %s: %s",
rsc->id, host_uname, pcmk_rc_str(rc));
return rc;
}
if (just_failures) {
rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
interval_spec, scheduler);
} else {
rc = clear_rsc_history(controld_api, host_uname, rsc->id, scheduler);
}
if (rc != pcmk_rc_ok) {
out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
rsc->id, host_uname, pcmk_rc_str(rc));
} else {
out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
}
return rc;
}
// \return Standard Pacemaker return code
int
cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
const char *operation, const char *interval_spec,
pcmk_scheduler_t *scheduler)
{
pcmk__output_t *out = scheduler->priv;
int rc = pcmk_rc_ok;
int attr_options = pcmk__node_attr_none;
const char *display_name = node_name? node_name : "all nodes";
if (controld_api == NULL) {
out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
display_name);
return rc;
}
if (node_name) {
pcmk_node_t *node = pe_find_node(scheduler->nodes, node_name);
if (node == NULL) {
out->err(out, "Unknown node: %s", node_name);
return ENXIO;
}
if (pe__is_guest_or_remote_node(node)) {
attr_options |= pcmk__node_attr_remote;
}
}
rc = pcmk__attrd_api_clear_failures(NULL, node_name, NULL, operation,
interval_spec, NULL, attr_options);
if (rc != pcmk_rc_ok) {
out->err(out, "Unable to clean up all failures on %s: %s",
display_name, pcmk_rc_str(rc));
return rc;
}
if (node_name) {
rc = clear_rsc_failures(out, controld_api, node_name, NULL,
operation, interval_spec, scheduler);
if (rc != pcmk_rc_ok) {
out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
node_name, pcmk_rc_str(rc));
return rc;
}
} else {
for (GList *iter = scheduler->nodes; iter; iter = iter->next) {
pcmk_node_t *node = (pcmk_node_t *) iter->data;
rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
operation, interval_spec, scheduler);
if (rc != pcmk_rc_ok) {
out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
pcmk_rc_str(rc));
return rc;
}
}
}
out->info(out, "Cleaned up all resources on %s", display_name);
return rc;
}
static void
check_role(resource_checks_t *checks)
{
const char *role_s = g_hash_table_lookup(checks->rsc->meta,
XML_RSC_ATTR_TARGET_ROLE);
if (role_s == NULL) {
return;
}
switch (text2role(role_s)) {
case pcmk_role_stopped:
checks->flags |= rsc_remain_stopped;
break;
case pcmk_role_unpromoted:
if (pcmk_is_set(pe__const_top_resource(checks->rsc, false)->flags,
pcmk_rsc_promotable)) {
checks->flags |= rsc_unpromotable;
}
break;
default:
break;
}
}
static void
check_managed(resource_checks_t *checks)
{
const char *managed_s = g_hash_table_lookup(checks->rsc->meta,
XML_RSC_ATTR_MANAGED);
if ((managed_s != NULL) && !crm_is_true(managed_s)) {
checks->flags |= rsc_unmanaged;
}
}
static void
check_locked(resource_checks_t *checks)
{
if (checks->rsc->lock_node != NULL) {
checks->flags |= rsc_locked;
checks->lock_node = checks->rsc->lock_node->details->uname;
}
}
static bool
node_is_unhealthy(pcmk_node_t *node)
{
switch (pe__health_strategy(node->details->data_set)) {
case pcmk__health_strategy_none:
break;
case pcmk__health_strategy_no_red:
if (pe__node_health(node) < 0) {
return true;
}
break;
case pcmk__health_strategy_only_green:
if (pe__node_health(node) <= 0) {
return true;
}
break;
case pcmk__health_strategy_progressive:
case pcmk__health_strategy_custom:
/* @TODO These are finite scores, possibly with rules, and possibly
* combining with other scores, so attributing these as a cause is
* nontrivial.
*/
break;
}
return false;
}
static void
check_node_health(resource_checks_t *checks, pcmk_node_t *node)
{
if (node == NULL) {
GHashTableIter iter;
bool allowed = false;
bool all_nodes_unhealthy = true;
g_hash_table_iter_init(&iter, checks->rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
allowed = true;
if (!node_is_unhealthy(node)) {
all_nodes_unhealthy = false;
break;
}
}
if (allowed && all_nodes_unhealthy) {
checks->flags |= rsc_node_health;
}
} else if (node_is_unhealthy(node)) {
checks->flags |= rsc_node_health;
}
}
int
cli_resource_check(pcmk__output_t *out, pcmk_resource_t *rsc, pcmk_node_t *node)
{
resource_checks_t checks = { .rsc = rsc };
check_role(&checks);
check_managed(&checks);
check_locked(&checks);
check_node_health(&checks, node);
return out->message(out, "resource-check-list", &checks);
}
// \return Standard Pacemaker return code
int
cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname,
const char *rsc_id, pcmk_scheduler_t *scheduler)
{
crm_notice("Failing %s on %s", rsc_id, host_uname);
return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, scheduler);
}
static GHashTable *
generate_resource_params(pcmk_resource_t *rsc, pcmk_node_t *node,
pcmk_scheduler_t *scheduler)
{
GHashTable *params = NULL;
GHashTable *meta = NULL;
GHashTable *combined = NULL;
GHashTableIter iter;
char *key = NULL;
char *value = NULL;
combined = pcmk__strkey_table(free, free);
params = pe_rsc_params(rsc, node, scheduler);
if (params != NULL) {
g_hash_table_iter_init(&iter, params);
while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
g_hash_table_insert(combined, strdup(key), strdup(value));
}
}
meta = pcmk__strkey_table(free, free);
get_meta_attributes(meta, rsc, node, scheduler);
if (meta != NULL) {
g_hash_table_iter_init(&iter, meta);
while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
char *crm_name = crm_meta_name(key);
g_hash_table_insert(combined, crm_name, strdup(value));
}
g_hash_table_destroy(meta);
}
return combined;
}
bool resource_is_running_on(pcmk_resource_t *rsc, const char *host)
{
bool found = true;
GList *hIter = NULL;
GList *hosts = NULL;
if (rsc == NULL) {
return false;
}
rsc->fns->location(rsc, &hosts, TRUE);
for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
pcmk_node_t *node = (pcmk_node_t *) hIter->data;
if (pcmk__strcase_any_of(host, node->details->uname, node->details->id, NULL)) {
crm_trace("Resource %s is running on %s\n", rsc->id, host);
goto done;
}
}
if (host != NULL) {
crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
found = false;
} else if(host == NULL && hosts == NULL) {
crm_trace("Resource %s is not running\n", rsc->id);
found = false;
}
done:
g_list_free(hosts);
return found;
}
/*!
* \internal
* \brief Create a list of all resources active on host from a given list
*
* \param[in] host Name of host to check whether resources are active
* \param[in] rsc_list List of resources to check
*
* \return New list of resources from list that are active on host
*/
static GList *
get_active_resources(const char *host, GList *rsc_list)
{
GList *rIter = NULL;
GList *active = NULL;
for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
pcmk_resource_t *rsc = (pcmk_resource_t *) rIter->data;
/* Expand groups to their members, because if we're restarting a member
* other than the first, we can't otherwise tell which resources are
* stopping and starting.
*/
if (rsc->variant == pcmk_rsc_variant_group) {
active = g_list_concat(active,
get_active_resources(host, rsc->children));
} else if (resource_is_running_on(rsc, host)) {
active = g_list_append(active, strdup(rsc->id));
}
}
return active;
}
static void dump_list(GList *items, const char *tag)
{
int lpc = 0;
GList *item = NULL;
for (item = items; item != NULL; item = item->next) {
crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
lpc++;
}
}
static void display_list(pcmk__output_t *out, GList *items, const char *tag)
{
GList *item = NULL;
for (item = items; item != NULL; item = item->next) {
out->info(out, "%s%s", tag, (const char *)item->data);
}
}
/*!
* \internal
* \brief Upgrade XML to latest schema version and use it as scheduler input
*
* This also updates the scheduler timestamp to the current time.
*
* \param[in,out] scheduler Scheduler data to update
* \param[in,out] xml XML to use as input
*
* \return Standard Pacemaker return code
* \note On success, caller is responsible for freeing memory allocated for
* scheduler->now.
* \todo This follows the example of other callers of cli_config_update()
* and returns ENOKEY ("Required key not available") if that fails,
* but perhaps pcmk_rc_schema_validation would be better in that case.
*/
int
update_scheduler_input(pcmk_scheduler_t *scheduler, xmlNode **xml)
{
if (cli_config_update(xml, NULL, FALSE) == FALSE) {
return ENOKEY;
}
scheduler->input = *xml;
scheduler->now = crm_time_new(NULL);
return pcmk_rc_ok;
}
/*!
* \internal
* \brief Update scheduler XML input based on a CIB query
*
* \param[in] scheduler Scheduler data to initialize
* \param[in] cib Connection to the CIB manager
*
* \return Standard Pacemaker return code
* \note On success, caller is responsible for freeing memory allocated for
* scheduler->input and scheduler->now.
*/
static int
update_scheduler_input_to_cib(pcmk__output_t *out, pcmk_scheduler_t *scheduler,
cib_t *cib)
{
xmlNode *cib_xml_copy = NULL;
int rc = pcmk_rc_ok;
rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
rc = pcmk_legacy2rc(rc);
if (rc != pcmk_rc_ok) {
out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_rc_str(rc), rc);
return rc;
}
rc = update_scheduler_input(scheduler, &cib_xml_copy);
if (rc != pcmk_rc_ok) {
out->err(out, "Could not upgrade the current CIB XML");
free_xml(cib_xml_copy);
return rc;
}
return rc;
}
// \return Standard Pacemaker return code
static int
update_dataset(cib_t *cib, pcmk_scheduler_t *scheduler, bool simulate)
{
char *pid = NULL;
char *shadow_file = NULL;
cib_t *shadow_cib = NULL;
int rc = pcmk_rc_ok;
pcmk__output_t *out = scheduler->priv;
pe_reset_working_set(scheduler);
pe__set_working_set_flags(scheduler,
pcmk_sched_no_counts|pcmk_sched_no_compat);
rc = update_scheduler_input_to_cib(out, scheduler, cib);
if (rc != pcmk_rc_ok) {
return rc;
}
if(simulate) {
bool prev_quiet = false;
pid = pcmk__getpid_s();
shadow_cib = cib_shadow_new(pid);
shadow_file = get_shadow_file(pid);
if (shadow_cib == NULL) {
out->err(out, "Could not create shadow cib: '%s'", pid);
rc = ENXIO;
goto done;
}
rc = write_xml_file(scheduler->input, shadow_file, FALSE);
if (rc < 0) {
out->err(out, "Could not populate shadow cib: %s (%d)", pcmk_strerror(rc), rc);
goto done;
}
rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
rc = pcmk_legacy2rc(rc);
if (rc != pcmk_rc_ok) {
out->err(out, "Could not connect to shadow cib: %s (%d)", pcmk_rc_str(rc), rc);
goto done;
}
pcmk__schedule_actions(scheduler->input,
pcmk_sched_no_counts|pcmk_sched_no_compat,
scheduler);
prev_quiet = out->is_quiet(out);
out->quiet = true;
pcmk__simulate_transition(scheduler, shadow_cib, NULL);
out->quiet = prev_quiet;
rc = update_dataset(shadow_cib, scheduler, false);
} else {
cluster_status(scheduler);
}
done:
// Do not free scheduler->input here, we need rsc->xml to be valid later on
cib_delete(shadow_cib);
free(pid);
if(shadow_file) {
unlink(shadow_file);
free(shadow_file);
}
return rc;
}
/*!
* \internal
* \brief Find the maximum stop timeout of a resource and its children (if any)
*
* \param[in,out] rsc Resource to get timeout for
*
* \return Maximum stop timeout for \p rsc (in milliseconds)
*/
static int
max_rsc_stop_timeout(pcmk_resource_t *rsc)
{
pcmk_action_t *stop = NULL;
long long result_ll;
int max_delay = 0;
if (rsc == NULL) {
return 0;
}
// If resource is collective, use maximum of its children's stop timeouts
if (rsc->children != NULL) {
for (GList *iter = rsc->children; iter; iter = iter->next) {
pcmk_resource_t *child = iter->data;
int delay = max_rsc_stop_timeout(child);
if (delay > max_delay) {
pe_rsc_trace(rsc,
"Maximum stop timeout for %s is now %s due to %s",
rsc->id, pcmk__readable_interval(delay), child->id);
max_delay = delay;
}
}
return max_delay;
}
/* Create a (transient) instance of the resource's stop action, to fully
* evaluate its timeout for rules, defaults, etc.
*
* @TODO This currently ignores node (which might matter for rules)
*/
stop = custom_action(rsc, stop_key(rsc), PCMK_ACTION_STOP, NULL, TRUE,
FALSE, rsc->cluster);
if ((pcmk__scan_ll(g_hash_table_lookup(stop->meta, XML_ATTR_TIMEOUT),
&result_ll, -1LL) == pcmk_rc_ok)
&& (result_ll >= 0) && (result_ll <= INT_MAX)) {
max_delay = (int) result_ll;
}
pe_free_action(stop);
return max_delay;
}
/*!
* \internal
* \brief Find a reasonable waiting time for stopping any one resource in a list
*
* \param[in,out] scheduler Scheduler data
* \param[in] resources List of names of resources that will be stopped
*
* \return Rough estimate of a reasonable time to wait (in seconds) to stop any
* one resource in \p resources
* \note This estimate is very rough, simply the maximum stop timeout of all
* given resources and their children, plus a small fudge factor. It does
* not account for children that must be stopped in sequence, action
* throttling, or any demotions needed. It checks the stop timeout, even
* if the resources in question are actually being started.
*/
static int
wait_time_estimate(pcmk_scheduler_t *scheduler, const GList *resources)
{
int max_delay = 0;
// Find maximum stop timeout in milliseconds
for (const GList *item = resources; item != NULL; item = item->next) {
pcmk_resource_t *rsc = pe_find_resource(scheduler->resources,
(const char *) item->data);
int delay = max_rsc_stop_timeout(rsc);
if (delay > max_delay) {
pe_rsc_trace(rsc,
"Wait time is now %s due to %s",
pcmk__readable_interval(delay), rsc->id);
max_delay = delay;
}
}
return (max_delay / 1000) + 5;
}
#define waiting_for_starts(d, r, h) ((d != NULL) || \
(!resource_is_running_on((r), (h))))
/*!
* \internal
* \brief Restart a resource (on a particular host if requested).
*
* \param[in,out] out Output object
* \param[in,out] rsc The resource to restart
* \param[in] node Node to restart resource on (NULL for all)
* \param[in] move_lifetime If not NULL, how long constraint should
* remain in effect (as ISO 8601 string)
* \param[in] timeout_ms Consider failed if actions do not complete
* in this time (specified in milliseconds,
* but a two-second granularity is actually
* used; if 0, it will be calculated based on
* the resource timeout)
* \param[in,out] cib Connection to the CIB manager
* \param[in] cib_options Group of enum cib_call_options flags to
* use with CIB calls
* \param[in] promoted_role_only If true, limit to promoted instances
* \param[in] force If true, apply only to requested instance
* if part of a collective resource
*
* \return Standard Pacemaker return code (exits on certain failures)
*/
int
cli_resource_restart(pcmk__output_t *out, pcmk_resource_t *rsc,
const pcmk_node_t *node, const char *move_lifetime,
int timeout_ms, cib_t *cib, int cib_options,
gboolean promoted_role_only, gboolean force)
{
int rc = pcmk_rc_ok;
int lpc = 0;
int before = 0;
int step_timeout_s = 0;
int sleep_interval = 2;
int timeout = timeout_ms / 1000;
bool stop_via_ban = false;
char *rsc_id = NULL;
char *lookup_id = NULL;
char *orig_target_role = NULL;
GList *list_delta = NULL;
GList *target_active = NULL;
GList *current_active = NULL;
GList *restart_target_active = NULL;
pcmk_scheduler_t *scheduler = NULL;
pcmk_resource_t *parent = uber_parent(rsc);
bool running = false;
const char *id = rsc->clone_name ? rsc->clone_name : rsc->id;
const char *host = node ? node->details->uname : NULL;
/* If the implicit resource or primitive resource of a bundle is given, operate on the
* bundle itself instead.
*/
if (pe_rsc_is_bundled(rsc)) {
rsc = parent->parent;
}
running = resource_is_running_on(rsc, host);
if (pe_rsc_is_clone(parent) && !running) {
if (pe_rsc_is_unique_clone(parent)) {
lookup_id = strdup(rsc->id);
} else {
lookup_id = clone_strip(rsc->id);
}
rsc = parent->fns->find_rsc(parent, lookup_id, node,
pcmk_rsc_match_basename
|pcmk_rsc_match_current_node);
free(lookup_id);
running = resource_is_running_on(rsc, host);
}
if (!running) {
if (host) {
out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
} else {
out->err(out, "%s is not running anywhere and so cannot be restarted", id);
}
return ENXIO;
}
if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
out->err(out, "Unmanaged resources cannot be restarted.");
return EAGAIN;
}
rsc_id = strdup(rsc->id);
if (pe_rsc_is_unique_clone(parent)) {
lookup_id = strdup(rsc->id);
} else {
lookup_id = clone_strip(rsc->id);
}
if (host) {
if (pe_rsc_is_clone(rsc) || pe_bundle_replicas(rsc)) {
stop_via_ban = true;
} else if (pe_rsc_is_clone(parent)) {
stop_via_ban = true;
free(lookup_id);
lookup_id = strdup(parent->id);
}
}
/*
grab full cib
determine originally active resources
disable or ban
poll cib and watch for affected resources to get stopped
without --timeout, calculate the stop timeout for each step and wait for that
if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
if everything stopped, re-enable or un-ban
poll cib and watch for affected resources to get started
without --timeout, calculate the start timeout for each step and wait for that
if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
report success
Optimizations:
- use constraints to determine ordered list of affected resources
- Allow a --no-deps option (aka. --force-restart)
*/
scheduler = pe_new_working_set();
if (scheduler == NULL) {
rc = errno;
out->err(out, "Could not allocate scheduler data: %s", pcmk_rc_str(rc));
goto done;
}
scheduler->priv = out;
rc = update_dataset(cib, scheduler, false);
if(rc != pcmk_rc_ok) {
out->err(out, "Could not get new resource list: %s (%d)", pcmk_rc_str(rc), rc);
goto done;
}
restart_target_active = get_active_resources(host, scheduler->resources);
current_active = get_active_resources(host, scheduler->resources);
dump_list(current_active, "Origin");
if (stop_via_ban) {
/* Stop the clone or bundle instance by banning it from the host */
out->quiet = true;
rc = cli_resource_ban(out, lookup_id, host, move_lifetime, cib,
cib_options, promoted_role_only,
PCMK__ROLE_PROMOTED);
} else {
/* Stop the resource by setting target-role to Stopped.
* Remember any existing target-role so we can restore it later
* (though it only makes any difference if it's Unpromoted).
*/
find_resource_attr(out, cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
NULL, XML_RSC_ATTR_TARGET_ROLE, &orig_target_role);
rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
NULL, XML_RSC_ATTR_TARGET_ROLE,
PCMK_ACTION_STOPPED, FALSE, cib,
cib_options, force);
}
if(rc != pcmk_rc_ok) {
out->err(out, "Could not set target-role for %s: %s (%d)", rsc_id, pcmk_rc_str(rc), rc);
if (current_active != NULL) {
g_list_free_full(current_active, free);
current_active = NULL;
}
if (restart_target_active != NULL) {
g_list_free_full(restart_target_active, free);
restart_target_active = NULL;
}
goto done;
}
rc = update_dataset(cib, scheduler, true);
if(rc != pcmk_rc_ok) {
out->err(out, "Could not determine which resources would be stopped");
goto failure;
}
target_active = get_active_resources(host, scheduler->resources);
dump_list(target_active, "Target");
list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
display_list(out, list_delta, " * ");
step_timeout_s = timeout / sleep_interval;
while (list_delta != NULL) {
before = g_list_length(list_delta);
if(timeout_ms == 0) {
step_timeout_s = wait_time_estimate(scheduler, list_delta)
/ sleep_interval;
}
/* We probably don't need the entire step timeout */
for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
sleep(sleep_interval);
if(timeout) {
timeout -= sleep_interval;
crm_trace("%ds remaining", timeout);
}
rc = update_dataset(cib, scheduler, FALSE);
if(rc != pcmk_rc_ok) {
out->err(out, "Could not determine which resources were stopped");
goto failure;
}
if (current_active != NULL) {
g_list_free_full(current_active, free);
current_active = NULL;
}
current_active = get_active_resources(host, scheduler->resources);
g_list_free(list_delta);
list_delta = NULL;
list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
dump_list(current_active, "Current");
dump_list(list_delta, "Delta");
}
crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
if(before == g_list_length(list_delta)) {
/* aborted during stop phase, print the contents of list_delta */
out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
display_list(out, list_delta, " * ");
rc = ETIME;
goto failure;
}
}
if (stop_via_ban) {
rc = cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
} else if (orig_target_role) {
rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
NULL, XML_RSC_ATTR_TARGET_ROLE,
orig_target_role, FALSE, cib,
cib_options, force);
free(orig_target_role);
orig_target_role = NULL;
} else {
rc = cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
NULL, XML_RSC_ATTR_TARGET_ROLE, cib,
cib_options, force);
}
if(rc != pcmk_rc_ok) {
out->err(out, "Could not unset target-role for %s: %s (%d)", rsc_id, pcmk_rc_str(rc), rc);
goto done;
}
if (target_active != NULL) {
g_list_free_full(target_active, free);
target_active = NULL;
}
target_active = restart_target_active;
list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
display_list(out, list_delta, " * ");
step_timeout_s = timeout / sleep_interval;
while (waiting_for_starts(list_delta, rsc, host)) {
before = g_list_length(list_delta);
if(timeout_ms == 0) {
step_timeout_s = wait_time_estimate(scheduler, list_delta)
/ sleep_interval;
}
/* We probably don't need the entire step timeout */
for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {
sleep(sleep_interval);
if(timeout) {
timeout -= sleep_interval;
crm_trace("%ds remaining", timeout);
}
rc = update_dataset(cib, scheduler, false);
if(rc != pcmk_rc_ok) {
out->err(out, "Could not determine which resources were started");
goto failure;
}
if (current_active != NULL) {
g_list_free_full(current_active, free);
current_active = NULL;
}
/* It's OK if dependent resources moved to a different node,
* so we check active resources on all nodes.
*/
current_active = get_active_resources(NULL, scheduler->resources);
g_list_free(list_delta);
list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
dump_list(current_active, "Current");
dump_list(list_delta, "Delta");
}
if(before == g_list_length(list_delta)) {
/* aborted during start phase, print the contents of list_delta */
out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
display_list(out, list_delta, " * ");
rc = ETIME;
goto failure;
}
}
rc = pcmk_rc_ok;
goto done;
failure:
if (stop_via_ban) {
cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
} else if (orig_target_role) {
cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
XML_RSC_ATTR_TARGET_ROLE, orig_target_role,
FALSE, cib, cib_options, force);
free(orig_target_role);
} else {
cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
NULL, XML_RSC_ATTR_TARGET_ROLE, cib,
cib_options, force);
}
done:
if (list_delta != NULL) {
g_list_free(list_delta);
}
if (current_active != NULL) {
g_list_free_full(current_active, free);
}
if (target_active != NULL && (target_active != restart_target_active)) {
g_list_free_full(target_active, free);
}
if (restart_target_active != NULL) {
g_list_free_full(restart_target_active, free);
}
free(rsc_id);
free(lookup_id);
pe_free_working_set(scheduler);
return rc;
}
static inline bool
action_is_pending(const pcmk_action_t *action)
{
if (pcmk_any_flags_set(action->flags,
pcmk_action_optional|pcmk_action_pseudo)
|| !pcmk_is_set(action->flags, pcmk_action_runnable)
|| pcmk__str_eq(PCMK_ACTION_NOTIFY, action->task, pcmk__str_casei)) {
return false;
}
return true;
}
/*!
* \internal
* \brief Check whether any actions in a list are pending
*
* \param[in] actions List of actions to check
*
* \return true if any actions in the list are pending, otherwise false
*/
static bool
actions_are_pending(const GList *actions)
{
for (const GList *action = actions; action != NULL; action = action->next) {
const pcmk_action_t *a = (const pcmk_action_t *) action->data;
if (action_is_pending(a)) {
crm_notice("Waiting for %s (flags=%#.8x)", a->uuid, a->flags);
return true;
}
}
return false;
}
static void
print_pending_actions(pcmk__output_t *out, GList *actions)
{
GList *action;
out->info(out, "Pending actions:");
for (action = actions; action != NULL; action = action->next) {
pcmk_action_t *a = (pcmk_action_t *) action->data;
if (!action_is_pending(a)) {
continue;
}
if (a->node) {
out->info(out, "\tAction %d: %s\ton %s",
a->id, a->uuid, pe__node_name(a->node));
} else {
out->info(out, "\tAction %d: %s", a->id, a->uuid);
}
}
}
/* For --wait, timeout (in seconds) to use if caller doesn't specify one */
#define WAIT_DEFAULT_TIMEOUT_S (60 * 60)
/* For --wait, how long to sleep between cluster state checks */
#define WAIT_SLEEP_S (2)
/*!
* \internal
* \brief Wait until all pending cluster actions are complete
*
* This waits until either the CIB's transition graph is idle or a timeout is
* reached.
*
* \param[in,out] out Output object
* \param[in] timeout_ms Consider failed if actions do not complete in
* this time (specified in milliseconds, but
* one-second granularity is actually used; if 0, a
* default will be used)
* \param[in,out] cib Connection to the CIB manager
*
* \return Standard Pacemaker return code
*/
int
wait_till_stable(pcmk__output_t *out, int timeout_ms, cib_t * cib)
{
pcmk_scheduler_t *scheduler = NULL;
int rc = pcmk_rc_ok;
int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
time_t expire_time = time(NULL) + timeout_s;
time_t time_diff;
bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet
scheduler = pe_new_working_set();
if (scheduler == NULL) {
return ENOMEM;
}
do {
/* Abort if timeout is reached */
time_diff = expire_time - time(NULL);
if (time_diff > 0) {
crm_info("Waiting up to %lld seconds for cluster actions to complete", (long long) time_diff);
} else {
print_pending_actions(out, scheduler->actions);
pe_free_working_set(scheduler);
return ETIME;
}
if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
sleep(WAIT_SLEEP_S);
}
/* Get latest transition graph */
pe_reset_working_set(scheduler);
rc = update_scheduler_input_to_cib(out, scheduler, cib);
if (rc != pcmk_rc_ok) {
pe_free_working_set(scheduler);
return rc;
}
pcmk__schedule_actions(scheduler->input,
pcmk_sched_no_counts|pcmk_sched_no_compat,
scheduler);
if (!printed_version_warning) {
/* If the DC has a different version than the local node, the two
* could come to different conclusions about what actions need to be
* done. Warn the user in this case.
*
* @TODO A possible long-term solution would be to reimplement the
* wait as a new controller operation that would be forwarded to the
* DC. However, that would have potential problems of its own.
*/
const char *dc_version = g_hash_table_lookup(scheduler->config_hash,
"dc-version");
if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
out->info(out, "warning: wait option may not work properly in "
"mixed-version cluster");
printed_version_warning = true;
}
}
} while (actions_are_pending(scheduler->actions));
pe_free_working_set(scheduler);
return rc;
}
static const char *
get_action(const char *rsc_action) {
const char *action = NULL;
if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
action = PCMK_ACTION_VALIDATE_ALL;
} else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
action = PCMK_ACTION_MONITOR;
} else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
"force-demote", "force-promote", NULL)) {
action = rsc_action+6;
} else {
action = rsc_action;
}
return action;
}
/*!
* \brief Set up environment variables as expected by resource agents
*
* When the cluster executes resource agents, it adds certain environment
* variables (directly or via resource meta-attributes) expected by some
* resource agents. Add the essential ones that many resource agents expect, so
* the behavior is the same for command-line execution.
*
* \param[in,out] params Resource parameters that will be passed to agent
* \param[in] timeout_ms Action timeout (in milliseconds)
* \param[in] check_level OCF check level
* \param[in] verbosity Verbosity level
*/
static void
set_agent_environment(GHashTable *params, int timeout_ms, int check_level,
int verbosity)
{
g_hash_table_insert(params, strdup("CRM_meta_timeout"),
crm_strdup_printf("%d", timeout_ms));
g_hash_table_insert(params, strdup(XML_ATTR_CRM_VERSION),
strdup(CRM_FEATURE_SET));
if (check_level >= 0) {
char *level = crm_strdup_printf("%d", check_level);
setenv("OCF_CHECK_LEVEL", level, 1);
free(level);
}
- setenv("HA_debug", (verbosity > 0)? "1" : "0", 1);
+ pcmk__set_env_option(PCMK__ENV_DEBUG, ((verbosity > 0)? "1" : "0"), true);
if (verbosity > 1) {
setenv("OCF_TRACE_RA", "1", 1);
}
/* A resource agent using the standard ocf-shellfuncs library will not print
* messages to stderr if it doesn't have a controlling terminal (e.g. if
* crm_resource is called via script or ssh). This forces it to do so.
*/
setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
}
/*!
* \internal
* \brief Apply command-line overrides to resource parameters
*
* \param[in,out] params Parameters to be passed to agent
* \param[in] overrides Parameters to override (or NULL if none)
*/
static void
apply_overrides(GHashTable *params, GHashTable *overrides)
{
if (overrides != NULL) {
GHashTableIter iter;
char *name = NULL;
char *value = NULL;
g_hash_table_iter_init(&iter, overrides);
while (g_hash_table_iter_next(&iter, (gpointer *) &name,
(gpointer *) &value)) {
g_hash_table_replace(params, strdup(name), strdup(value));
}
}
}
crm_exit_t
cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
const char *rsc_class, const char *rsc_prov,
const char *rsc_type, const char *rsc_action,
GHashTable *params, GHashTable *override_hash,
int timeout_ms, int resource_verbose, gboolean force,
int check_level)
{
const char *class = rsc_class;
const char *action = get_action(rsc_action);
crm_exit_t exit_code = CRM_EX_OK;
svc_action_t *op = NULL;
// If no timeout was provided, use the same default as the cluster
if (timeout_ms == 0) {
timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
}
set_agent_environment(params, timeout_ms, check_level, resource_verbose);
apply_overrides(params, override_hash);
op = services__create_resource_action(rsc_name? rsc_name : "test",
rsc_class, rsc_prov, rsc_type, action,
0, timeout_ms, params, 0);
if (op == NULL) {
out->err(out, "Could not execute %s using %s%s%s:%s: %s",
action, rsc_class, (rsc_prov? ":" : ""),
(rsc_prov? rsc_prov : ""), rsc_type, strerror(ENOMEM));
g_hash_table_destroy(params);
return CRM_EX_OSERR;
}
if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
class = resources_find_service_class(rsc_type);
}
if (!pcmk__strcase_any_of(class, PCMK_RESOURCE_CLASS_OCF,
PCMK_RESOURCE_CLASS_LSB, NULL)) {
services__format_result(op, CRM_EX_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
"Manual execution of the %s standard is "
"unsupported", pcmk__s(class, "unspecified"));
}
if (op->rc != PCMK_OCF_UNKNOWN) {
exit_code = op->rc;
goto done;
}
services_action_sync(op);
// Map results to OCF codes for consistent reporting to user
{
enum ocf_exitcode ocf_code = services_result2ocf(class, action, op->rc);
// Cast variable instead of function return to keep compilers happy
exit_code = (crm_exit_t) ocf_code;
}
done:
out->message(out, "resource-agent-action", resource_verbose, rsc_class,
rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
exit_code, op->status, services__exit_reason(op),
op->stdout_data, op->stderr_data);
services_action_free(op);
return exit_code;
}
crm_exit_t
cli_resource_execute(pcmk_resource_t *rsc, const char *requested_name,
const char *rsc_action, GHashTable *override_hash,
int timeout_ms, cib_t *cib, pcmk_scheduler_t *scheduler,
int resource_verbose, gboolean force, int check_level)
{
pcmk__output_t *out = scheduler->priv;
crm_exit_t exit_code = CRM_EX_OK;
const char *rid = NULL;
const char *rtype = NULL;
const char *rprov = NULL;
const char *rclass = NULL;
GHashTable *params = NULL;
if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
"force-promote", NULL)) {
if(pe_rsc_is_clone(rsc)) {
GList *nodes = cli_resource_search(rsc, requested_name, scheduler);
if(nodes != NULL && force == FALSE) {
out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
rsc_action, rsc->id);
out->err(out, "Try setting target-role=Stopped first or specifying "
"the force option");
return CRM_EX_UNSAFE;
}
g_list_free_full(nodes, free);
}
}
if(pe_rsc_is_clone(rsc)) {
/* Grab the first child resource in the hope it's not a group */
rsc = rsc->children->data;
}
if (rsc->variant == pcmk_rsc_variant_group) {
out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
return CRM_EX_UNIMPLEMENT_FEATURE;
} else if (pe_rsc_is_bundled(rsc)) {
out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
return CRM_EX_UNIMPLEMENT_FEATURE;
}
rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
params = generate_resource_params(rsc, NULL /* @TODO use local node */,
scheduler);
if (timeout_ms == 0) {
timeout_ms = pe_get_configured_timeout(rsc, get_action(rsc_action),
scheduler);
}
rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;
exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
params, override_hash, timeout_ms,
resource_verbose, force, check_level);
return exit_code;
}
// \return Standard Pacemaker return code
int
cli_resource_move(const pcmk_resource_t *rsc, const char *rsc_id,
const char *host_name, const char *move_lifetime, cib_t *cib,
int cib_options, pcmk_scheduler_t *scheduler,
gboolean promoted_role_only, gboolean force)
{
pcmk__output_t *out = scheduler->priv;
int rc = pcmk_rc_ok;
unsigned int count = 0;
pcmk_node_t *current = NULL;
pcmk_node_t *dest = pe_find_node(scheduler->nodes, host_name);
bool cur_is_dest = false;
if (dest == NULL) {
return pcmk_rc_node_unknown;
}
if (promoted_role_only
&& !pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
const pcmk_resource_t *p = pe__const_top_resource(rsc, false);
if (pcmk_is_set(p->flags, pcmk_rsc_promotable)) {
out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
rsc_id = p->id;
rsc = p;
} else {
out->info(out, "Ignoring --promoted option: %s is not promotable",
rsc_id);
promoted_role_only = FALSE;
}
}
current = pe__find_active_requires(rsc, &count);
if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
unsigned int promoted_count = 0;
pcmk_node_t *promoted_node = NULL;
for (const GList *iter = rsc->children; iter; iter = iter->next) {
const pcmk_resource_t *child = (const pcmk_resource_t *) iter->data;
enum rsc_role_e child_role = child->fns->state(child, TRUE);
if (child_role == pcmk_role_promoted) {
rsc = child;
promoted_node = pe__current_node(child);
promoted_count++;
}
}
if (promoted_role_only || (promoted_count != 0)) {
count = promoted_count;
current = promoted_node;
}
}
if (count > 1) {
if (pe_rsc_is_clone(rsc)) {
current = NULL;
} else {
return pcmk_rc_multiple;
}
}
if (current && (current->details == dest->details)) {
cur_is_dest = true;
if (force) {
crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
rsc_id, promoted_role_only?"promoted":"active",
pe__node_name(dest));
} else {
return pcmk_rc_already;
}
}
/* Clear any previous prefer constraints across all nodes. */
cli_resource_clear(rsc_id, NULL, scheduler->nodes, cib, cib_options, false,
force);
/* Clear any previous ban constraints on 'dest'. */
cli_resource_clear(rsc_id, dest->details->uname, scheduler->nodes, cib,
cib_options, TRUE, force);
/* Record an explicit preference for 'dest' */
rc = cli_resource_prefer(out, rsc_id, dest->details->uname, move_lifetime,
cib, cib_options, promoted_role_only,
PCMK__ROLE_PROMOTED);
crm_trace("%s%s now prefers %s%s",
rsc->id, (promoted_role_only? " (promoted)" : ""),
pe__node_name(dest), force?"(forced)":"");
/* only ban the previous location if current location != destination location.
* it is possible to use -M to enforce a location without regard of where the
* resource is currently located */
if (force && !cur_is_dest) {
/* Ban the original location if possible */
if(current) {
(void)cli_resource_ban(out, rsc_id, current->details->uname, move_lifetime,
cib, cib_options, promoted_role_only,
PCMK__ROLE_PROMOTED);
} else if(count > 1) {
out->info(out, "Resource '%s' is currently %s in %d locations. "
"One may now move to %s",
rsc_id, (promoted_role_only? "promoted" : "active"),
count, pe__node_name(dest));
out->info(out, "To prevent '%s' from being %s at a specific location, "
"specify a node.",
rsc_id, (promoted_role_only? "promoted" : "active"));
} else {
crm_trace("Not banning %s from its current location: not active", rsc_id);
}
}
return rc;
}

File Metadata

Mime Type
text/x-diff
Expires
Mon, Apr 21, 6:08 PM (1 d, 4 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1665091
Default Alt Text
(661 KB)

Event Timeline