No OneTemporary
Actions

Size

118 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/daemons/execd/execd_commands.c b/daemons/execd/execd_commands.c
	index 3ece88859b..b4fc1d8042 100644
	--- a/daemons/execd/execd_commands.c
	+++ b/daemons/execd/execd_commands.c
	@@ -1,1994 +1,1996 @@
	/*
	* Copyright 2012-2025 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/fencing/internal.h>

	#include <glib.h>

	// Check whether we have a high-resolution monotonic clock
	#undef PCMK__TIME_USE_CGT
	#if HAVE_DECL_CLOCK_MONOTONIC && defined(CLOCK_MONOTONIC)
	# define PCMK__TIME_USE_CGT
	# include <time.h> /* clock_gettime */
	#endif

	#include <unistd.h>

	#include <crm/crm.h>
	#include <crm/fencing/internal.h>
	#include <crm/services.h>
	#include <crm/services_internal.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/xml.h>

	#include "pacemaker-execd.h"

	GHashTable *rsc_list = NULL;

	typedef struct lrmd_cmd_s {
	int timeout;
	guint interval_ms;
	int start_delay;
	int timeout_orig;

	int call_id;

	int call_opts;
	/* Timer ids, must be removed on cmd destruction. */
	int delay_id;
	int stonith_recurring_id;

	int rsc_deleted;

	int service_flags;

	char *client_id;
	char *origin;
	char *rsc_id;
	char *action;
	char *real_action;
	char *userdata_str;

	pcmk__action_result_t result;

	/* We can track operation queue time and run time, to be saved with the CIB
	* resource history (and displayed in cluster status). We need
	* high-resolution monotonic time for this purpose, so we use
	* clock_gettime(CLOCK_MONOTONIC, ...) (if available, otherwise this feature
	* is disabled).
	*
	* However, we also need epoch timestamps for recording the time the command
	* last ran and the time its return value last changed, for use in time
	* displays (as opposed to interval calculations). We keep time_t values for
	* this purpose.
	*
	* The last run time is used for both purposes, so we keep redundant
	* monotonic and epoch values for this. Technically the two could represent
	* different times, but since time_t has only second resolution and the
	* values are used for distinct purposes, that is not significant.
	*/
	#ifdef PCMK__TIME_USE_CGT
	/* Recurring and systemd operations may involve more than one executor
	* command per operation, so they need info about the original and the most
	* recent.
	*/
	struct timespec t_first_run; // When op first ran
	struct timespec t_run; // When op most recently ran
	struct timespec t_first_queue; // When op was first queued
	struct timespec t_queue; // When op was most recently queued
	#endif
	time_t epoch_last_run; // Epoch timestamp of when op last ran
	time_t epoch_rcchange; // Epoch timestamp of when rc last changed

	bool first_notify_sent;
	int last_notify_rc;
	int last_notify_op_status;
	int last_pid;

	GHashTable *params;
	} lrmd_cmd_t;

	static void cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc);
	static gboolean execute_resource_action(gpointer user_data);
	static void cancel_all_recurring(lrmd_rsc_t * rsc, const char *client_id);

	#ifdef PCMK__TIME_USE_CGT

	/*!
	* \internal
	* \brief Check whether a struct timespec has been set
	*
	* \param[in] timespec Time to check
	*
	* \return true if timespec has been set (i.e. is nonzero), false otherwise
	*/
	static inline bool
	time_is_set(const struct timespec *timespec)
	{
	return (timespec != NULL) &&
	((timespec->tv_sec != 0) \|\| (timespec->tv_nsec != 0));
	}

	/*
	* \internal
	* \brief Set a timespec (and its original if unset) to the current time
	*
	* \param[out] t_current Where to store current time
	* \param[out] t_orig Where to copy t_current if unset
	*/
	static void
	get_current_time(struct timespec t_current, struct timespec t_orig)
	{
	clock_gettime(CLOCK_MONOTONIC, t_current);
	if ((t_orig != NULL) && !time_is_set(t_orig)) {
	t_orig = t_current;
	}
	}

	/*!
	* \internal
	* \brief Return difference between two times in milliseconds
	*
	* \param[in] now More recent time (or NULL to use current time)
	* \param[in] old Earlier time
	*
	* \return milliseconds difference (or 0 if old is NULL or unset)
	*
	* \note Can overflow on 32bit machines when the differences is around
	* 24 days or more.
	*/
	static int
	time_diff_ms(const struct timespec now, const struct timespec old)
	{
	int diff_ms = 0;

	if (time_is_set(old)) {
	struct timespec local_now = { 0, };

	if (now == NULL) {
	clock_gettime(CLOCK_MONOTONIC, &local_now);
	now = &local_now;
	}
	diff_ms = (now->tv_sec - old->tv_sec) * 1000
	+ (now->tv_nsec - old->tv_nsec) / 1000000;
	}
	return diff_ms;
	}

	/*!
	* \internal
	* \brief Reset a command's operation times to their original values.
	*
	* Reset a command's run and queued timestamps to the timestamps of the original
	* command, so we report the entire time since then and not just the time since
	* the most recent command (for recurring and systemd operations).
	*
	* \param[in,out] cmd Executor command object to reset
	*
	* \note It's not obvious what the queued time should be for a systemd
	* start/stop operation, which might go like this:
	* initial command queued 5ms, runs 3s
	* monitor command queued 10ms, runs 10s
	* monitor command queued 10ms, runs 10s
	* Is the queued time for that operation 5ms, 10ms or 25ms? The current
	* implementation will report 5ms. If it's 25ms, then we need to
	* subtract 20ms from the total exec time so as not to count it twice.
	* We can implement that later if it matters to anyone ...
	*/
	static void
	cmd_original_times(lrmd_cmd_t * cmd)
	{
	cmd->t_run = cmd->t_first_run;
	cmd->t_queue = cmd->t_first_queue;
	}
	#endif

	static inline bool
	action_matches(const lrmd_cmd_t cmd, const char action, guint interval_ms)
	{
	return (cmd->interval_ms == interval_ms)
	&& pcmk__str_eq(cmd->action, action, pcmk__str_casei);
	}

	/*!
	* \internal
	* \brief Log the result of an asynchronous command
	*
	* \param[in] cmd Command to log result for
	* \param[in] exec_time_ms Execution time in milliseconds, if known
	* \param[in] queue_time_ms Queue time in milliseconds, if known
	*/
	static void
	log_finished(const lrmd_cmd_t *cmd, int exec_time_ms, int queue_time_ms)
	{
	int log_level = LOG_INFO;
	GString *str = g_string_sized_new(100); // reasonable starting size

	if (pcmk__str_eq(cmd->action, PCMK_ACTION_MONITOR, pcmk__str_casei)) {
	log_level = LOG_DEBUG;
	}

	g_string_append_printf(str, "%s %s (call %d",
	cmd->rsc_id, cmd->action, cmd->call_id);
	if (cmd->last_pid != 0) {
	g_string_append_printf(str, ", PID %d", cmd->last_pid);
	}
	switch (cmd->result.execution_status) {
	case PCMK_EXEC_DONE:
	g_string_append_printf(str, ") exited with status %d",
	cmd->result.exit_status);
	break;
	case PCMK_EXEC_CANCELLED:
	g_string_append_printf(str, ") cancelled");
	break;
	default:
	pcmk__g_strcat(str, ") could not be executed: ",
	pcmk_exec_status_str(cmd->result.execution_status),
	NULL);
	break;
	}
	if (cmd->result.exit_reason != NULL) {
	pcmk__g_strcat(str, " (", cmd->result.exit_reason, ")", NULL);
	}

	#ifdef PCMK__TIME_USE_CGT
	pcmk__g_strcat(str, " (execution time ",
	pcmk__readable_interval(exec_time_ms), NULL);
	if (queue_time_ms > 0) {
	pcmk__g_strcat(str, " after being queued ",
	pcmk__readable_interval(queue_time_ms), NULL);
	}
	g_string_append_c(str, ')');
	#endif

	do_crm_log(log_level, "%s", str->str);
	g_string_free(str, TRUE);
	}

	static void
	log_execute(lrmd_cmd_t * cmd)
	{
	int log_level = LOG_INFO;

	if (pcmk__str_eq(cmd->action, PCMK_ACTION_MONITOR, pcmk__str_casei)) {
	log_level = LOG_DEBUG;
	}

	do_crm_log(log_level, "executing - rsc:%s action:%s call_id:%d",
	cmd->rsc_id, cmd->action, cmd->call_id);
	}

	static const char *
	normalize_action_name(lrmd_rsc_t * rsc, const char *action)
	{
	if (pcmk__str_eq(action, PCMK_ACTION_MONITOR, pcmk__str_casei) &&
	pcmk_is_set(pcmk_get_ra_caps(rsc->class), pcmk_ra_cap_status)) {
	return PCMK_ACTION_STATUS;
	}
	return action;
	}

	static lrmd_rsc_t *
	build_rsc_from_xml(xmlNode * msg)
	{
	xmlNode *rsc_xml = get_xpath_object("//" PCMK__XE_LRMD_RSC, msg, LOG_ERR);
	lrmd_rsc_t *rsc = NULL;

	rsc = pcmk__assert_alloc(1, sizeof(lrmd_rsc_t));

	crm_element_value_int(msg, PCMK__XA_LRMD_CALLOPT, &rsc->call_opts);

	rsc->rsc_id = crm_element_value_copy(rsc_xml, PCMK__XA_LRMD_RSC_ID);
	rsc->class = crm_element_value_copy(rsc_xml, PCMK__XA_LRMD_CLASS);
	rsc->provider = crm_element_value_copy(rsc_xml, PCMK__XA_LRMD_PROVIDER);
	rsc->type = crm_element_value_copy(rsc_xml, PCMK__XA_LRMD_TYPE);
	rsc->work = mainloop_add_trigger(G_PRIORITY_HIGH, execute_resource_action,
	rsc);

	// Initialize fence device probes (to return "not running")
	pcmk__set_result(&rsc->fence_probe_result, CRM_EX_ERROR,
	PCMK_EXEC_NO_FENCE_DEVICE, NULL);
	return rsc;
	}

	static lrmd_cmd_t *
	create_lrmd_cmd(xmlNode msg, pcmk__client_t client)
	{
	int call_options = 0;
	xmlNode *rsc_xml = get_xpath_object("//" PCMK__XE_LRMD_RSC, msg, LOG_ERR);
	lrmd_cmd_t *cmd = NULL;

	cmd = pcmk__assert_alloc(1, sizeof(lrmd_cmd_t));

	crm_element_value_int(msg, PCMK__XA_LRMD_CALLOPT, &call_options);
	cmd->call_opts = call_options;
	cmd->client_id = pcmk__str_copy(client->id);

	crm_element_value_int(msg, PCMK__XA_LRMD_CALLID, &cmd->call_id);
	crm_element_value_ms(rsc_xml, PCMK__XA_LRMD_RSC_INTERVAL,
	&cmd->interval_ms);
	crm_element_value_int(rsc_xml, PCMK__XA_LRMD_TIMEOUT, &cmd->timeout);
	crm_element_value_int(rsc_xml, PCMK__XA_LRMD_RSC_START_DELAY,
	&cmd->start_delay);
	cmd->timeout_orig = cmd->timeout;

	cmd->origin = crm_element_value_copy(rsc_xml, PCMK__XA_LRMD_ORIGIN);
	cmd->action = crm_element_value_copy(rsc_xml, PCMK__XA_LRMD_RSC_ACTION);
	cmd->userdata_str = crm_element_value_copy(rsc_xml,
	PCMK__XA_LRMD_RSC_USERDATA_STR);
	cmd->rsc_id = crm_element_value_copy(rsc_xml, PCMK__XA_LRMD_RSC_ID);

	cmd->params = xml2list(rsc_xml);

	if (pcmk__str_eq(g_hash_table_lookup(cmd->params, "CRM_meta_on_fail"),
	PCMK_VALUE_BLOCK, pcmk__str_casei)) {
	crm_debug("Setting flag to leave pid group on timeout and "
	"only kill action pid for " PCMK__OP_FMT,
	cmd->rsc_id, cmd->action, cmd->interval_ms);
	cmd->service_flags = pcmk__set_flags_as(__func__, __LINE__,
	LOG_TRACE, "Action",
	cmd->action, 0,
	SVC_ACTION_LEAVE_GROUP,
	"SVC_ACTION_LEAVE_GROUP");
	}
	return cmd;
	}

	static void
	stop_recurring_timer(lrmd_cmd_t *cmd)
	{
	if (cmd) {
	if (cmd->stonith_recurring_id) {
	g_source_remove(cmd->stonith_recurring_id);
	}
	cmd->stonith_recurring_id = 0;
	}
	}

	static void
	free_lrmd_cmd(lrmd_cmd_t * cmd)
	{
	stop_recurring_timer(cmd);
	if (cmd->delay_id) {
	g_source_remove(cmd->delay_id);
	}
	if (cmd->params) {
	g_hash_table_destroy(cmd->params);
	}
	pcmk__reset_result(&(cmd->result));
	free(cmd->origin);
	free(cmd->action);
	free(cmd->real_action);
	free(cmd->userdata_str);
	free(cmd->rsc_id);
	free(cmd->client_id);
	free(cmd);
	}

	static gboolean
	stonith_recurring_op_helper(gpointer data)
	{
	lrmd_cmd_t *cmd = data;
	lrmd_rsc_t *rsc;

	cmd->stonith_recurring_id = 0;

	if (!cmd->rsc_id) {
	return FALSE;
	}

	rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id);

	pcmk__assert(rsc != NULL);
	/* take it out of recurring_ops list, and put it in the pending ops
	* to be executed */
	rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd);
	rsc->pending_ops = g_list_append(rsc->pending_ops, cmd);
	#ifdef PCMK__TIME_USE_CGT
	get_current_time(&(cmd->t_queue), &(cmd->t_first_queue));
	#endif
	mainloop_set_trigger(rsc->work);

	return FALSE;
	}

	static inline void
	start_recurring_timer(lrmd_cmd_t *cmd)
	{
	if (!cmd \|\| (cmd->interval_ms <= 0)) {
	return;
	}

	cmd->stonith_recurring_id = pcmk__create_timer(cmd->interval_ms,
	stonith_recurring_op_helper,
	cmd);
	}

	static gboolean
	start_delay_helper(gpointer data)
	{
	lrmd_cmd_t *cmd = data;
	lrmd_rsc_t *rsc = NULL;

	cmd->delay_id = 0;
	rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;

	if (rsc) {
	mainloop_set_trigger(rsc->work);
	}

	return FALSE;
	}

	/*!
	* \internal
	* \brief Check whether a list already contains the equivalent of a given action
	*
	* \param[in] action_list List to search
	* \param[in] cmd Action to search for
	*/
	static lrmd_cmd_t *
	find_duplicate_action(const GList action_list, const lrmd_cmd_t cmd)
	{
	for (const GList *item = action_list; item != NULL; item = item->next) {
	lrmd_cmd_t *dup = item->data;

	if (action_matches(cmd, dup->action, dup->interval_ms)) {
	return dup;
	}
	}
	return NULL;
	}

	static bool
	merge_recurring_duplicate(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
	{
	lrmd_cmd_t * dup = NULL;
	bool dup_pending = true;

	if (cmd->interval_ms == 0) {
	return false;
	}

	// Search for a duplicate of this action (in-flight or not)
	dup = find_duplicate_action(rsc->pending_ops, cmd);
	if (dup == NULL) {
	dup_pending = false;
	dup = find_duplicate_action(rsc->recurring_ops, cmd);
	if (dup == NULL) {
	return false;
	}
	}

	/* Do not merge fencing monitors marked for cancellation, so we can reply to
	* the cancellation separately.
	*/
	if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH,
	pcmk__str_casei)
	&& (dup->result.execution_status == PCMK_EXEC_CANCELLED)) {
	return false;
	}

	/* This should not occur. If it does, we need to investigate how something
	* like this is possible in the controller.
	*/
	crm_warn("Duplicate recurring op entry detected (" PCMK__OP_FMT
	"), merging with previous op entry",
	rsc->rsc_id, normalize_action_name(rsc, dup->action),
	dup->interval_ms);

	// Merge new action's call ID and user data into existing action
	dup->first_notify_sent = false;
	free(dup->userdata_str);
	dup->userdata_str = cmd->userdata_str;
	cmd->userdata_str = NULL;
	dup->call_id = cmd->call_id;
	free_lrmd_cmd(cmd);
	cmd = NULL;

	/* If dup is not pending, that means it has already executed at least once
	* and is waiting in the interval. In that case, stop waiting and initiate
	* a new instance now.
	*/
	if (!dup_pending) {
	if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH,
	pcmk__str_casei)) {
	stop_recurring_timer(dup);
	stonith_recurring_op_helper(dup);
	} else {
	services_action_kick(rsc->rsc_id,
	normalize_action_name(rsc, dup->action),
	dup->interval_ms);
	}
	}
	return true;
	}

	static void
	schedule_lrmd_cmd(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
	{
	CRM_CHECK(cmd != NULL, return);
	CRM_CHECK(rsc != NULL, return);

	crm_trace("Scheduling %s on %s", cmd->action, rsc->rsc_id);

	if (merge_recurring_duplicate(rsc, cmd)) {
	// Equivalent of cmd has already been scheduled
	return;
	}

	/* The controller expects the executor to automatically cancel
	* recurring operations before a resource stops.
	*/
	if (pcmk__str_eq(cmd->action, PCMK_ACTION_STOP, pcmk__str_casei)) {
	cancel_all_recurring(rsc, NULL);
	}

	rsc->pending_ops = g_list_append(rsc->pending_ops, cmd);
	#ifdef PCMK__TIME_USE_CGT
	get_current_time(&(cmd->t_queue), &(cmd->t_first_queue));
	#endif
	mainloop_set_trigger(rsc->work);

	if (cmd->start_delay) {
	cmd->delay_id = pcmk__create_timer(cmd->start_delay, start_delay_helper, cmd);
	}
	}

	static xmlNode *
	create_lrmd_reply(const char *origin, int rc, int call_id)
	{
	xmlNode *reply = pcmk__xe_create(NULL, PCMK__XE_LRMD_REPLY);

	crm_xml_add(reply, PCMK__XA_LRMD_ORIGIN, origin);
	crm_xml_add_int(reply, PCMK__XA_LRMD_RC, rc);
	crm_xml_add_int(reply, PCMK__XA_LRMD_CALLID, call_id);
	return reply;
	}

	static void
	send_client_notify(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNode *update_msg = user_data;
	pcmk__client_t *client = value;
	int rc;
	int log_level = LOG_WARNING;
	const char *msg = NULL;

	CRM_CHECK(client != NULL, return);
	if (client->name == NULL) {
	crm_trace("Skipping notification to client without name");
	return;
	}
	if (pcmk_is_set(client->flags, pcmk__client_to_proxy)) {
	/* We only want to notify clients of the executor IPC API. If we are
	* running as Pacemaker Remote, we may have clients proxied to other
	* IPC services in the cluster, so skip those.
	*/
	crm_trace("Skipping executor API notification to client %s",
	pcmk__client_name(client));
	return;
	}

	rc = lrmd_server_send_notify(client, update_msg);
	if (rc == pcmk_rc_ok) {
	return;
	}

	switch (rc) {
	case ENOTCONN:
	case EPIPE: // Client exited without waiting for notification
	log_level = LOG_INFO;
	msg = "Disconnected";
	break;

	default:
	msg = pcmk_rc_str(rc);
	break;
	}
	do_crm_log(log_level, "Could not notify client %s: %s " QB_XS " rc=%d",
	pcmk__client_name(client), msg, rc);
	}

	static void
	send_cmd_complete_notify(lrmd_cmd_t * cmd)
	{
	xmlNode *notify = NULL;
	int exec_time = 0;
	int queue_time = 0;

	#ifdef PCMK__TIME_USE_CGT
	exec_time = time_diff_ms(NULL, &(cmd->t_run));
	queue_time = time_diff_ms(&cmd->t_run, &(cmd->t_queue));
	#endif
	log_finished(cmd, exec_time, queue_time);

	/* If the originator requested to be notified only for changes in recurring
	* operation results, skip the notification if the result hasn't changed.
	*/
	if (cmd->first_notify_sent
	&& pcmk_is_set(cmd->call_opts, lrmd_opt_notify_changes_only)
	&& (cmd->last_notify_rc == cmd->result.exit_status)
	&& (cmd->last_notify_op_status == cmd->result.execution_status)) {
	return;
	}

	cmd->first_notify_sent = true;
	cmd->last_notify_rc = cmd->result.exit_status;
	cmd->last_notify_op_status = cmd->result.execution_status;

	notify = pcmk__xe_create(NULL, PCMK__XE_LRMD_NOTIFY);

	crm_xml_add(notify, PCMK__XA_LRMD_ORIGIN, __func__);
	crm_xml_add_int(notify, PCMK__XA_LRMD_TIMEOUT, cmd->timeout);
	crm_xml_add_ms(notify, PCMK__XA_LRMD_RSC_INTERVAL, cmd->interval_ms);
	crm_xml_add_int(notify, PCMK__XA_LRMD_RSC_START_DELAY, cmd->start_delay);
	crm_xml_add_int(notify, PCMK__XA_LRMD_EXEC_RC, cmd->result.exit_status);
	crm_xml_add_int(notify, PCMK__XA_LRMD_EXEC_OP_STATUS,
	cmd->result.execution_status);
	crm_xml_add_int(notify, PCMK__XA_LRMD_CALLID, cmd->call_id);
	crm_xml_add_int(notify, PCMK__XA_LRMD_RSC_DELETED, cmd->rsc_deleted);

	crm_xml_add_ll(notify, PCMK__XA_LRMD_RUN_TIME,
	(long long) cmd->epoch_last_run);
	crm_xml_add_ll(notify, PCMK__XA_LRMD_RCCHANGE_TIME,
	(long long) cmd->epoch_rcchange);
	#ifdef PCMK__TIME_USE_CGT
	crm_xml_add_int(notify, PCMK__XA_LRMD_EXEC_TIME, exec_time);
	crm_xml_add_int(notify, PCMK__XA_LRMD_QUEUE_TIME, queue_time);
	#endif

	crm_xml_add(notify, PCMK__XA_LRMD_OP, LRMD_OP_RSC_EXEC);
	crm_xml_add(notify, PCMK__XA_LRMD_RSC_ID, cmd->rsc_id);
	if(cmd->real_action) {
	crm_xml_add(notify, PCMK__XA_LRMD_RSC_ACTION, cmd->real_action);
	} else {
	crm_xml_add(notify, PCMK__XA_LRMD_RSC_ACTION, cmd->action);
	}
	crm_xml_add(notify, PCMK__XA_LRMD_RSC_USERDATA_STR, cmd->userdata_str);
	crm_xml_add(notify, PCMK__XA_LRMD_RSC_EXIT_REASON, cmd->result.exit_reason);

	if (cmd->result.action_stderr != NULL) {
	crm_xml_add(notify, PCMK__XA_LRMD_RSC_OUTPUT,
	cmd->result.action_stderr);

	} else if (cmd->result.action_stdout != NULL) {
	crm_xml_add(notify, PCMK__XA_LRMD_RSC_OUTPUT,
	cmd->result.action_stdout);
	}

	if (cmd->params) {
	char *key = NULL;
	char *value = NULL;
	GHashTableIter iter;

	xmlNode *args = pcmk__xe_create(notify, PCMK__XE_ATTRIBUTES);

	g_hash_table_iter_init(&iter, cmd->params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	hash2smartfield((gpointer) key, (gpointer) value, args);
	}
	}
	if ((cmd->client_id != NULL)
	&& pcmk_is_set(cmd->call_opts, lrmd_opt_notify_orig_only)) {

	pcmk__client_t *client = pcmk__find_client_by_id(cmd->client_id);

	if (client != NULL) {
	send_client_notify(client->id, client, notify);
	}
	} else {
	pcmk__foreach_ipc_client(send_client_notify, notify);
	}

	pcmk__xml_free(notify);
	}

	static void
	send_generic_notify(int rc, xmlNode * request)
	{
	if (pcmk__ipc_client_count() != 0) {
	int call_id = 0;
	xmlNode *notify = NULL;
	xmlNode *rsc_xml = get_xpath_object("//" PCMK__XE_LRMD_RSC, request,
	LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, PCMK__XA_LRMD_RSC_ID);
	const char *op = crm_element_value(request, PCMK__XA_LRMD_OP);

	crm_element_value_int(request, PCMK__XA_LRMD_CALLID, &call_id);

	notify = pcmk__xe_create(NULL, PCMK__XE_LRMD_NOTIFY);
	crm_xml_add(notify, PCMK__XA_LRMD_ORIGIN, __func__);
	crm_xml_add_int(notify, PCMK__XA_LRMD_RC, rc);
	crm_xml_add_int(notify, PCMK__XA_LRMD_CALLID, call_id);
	crm_xml_add(notify, PCMK__XA_LRMD_OP, op);
	crm_xml_add(notify, PCMK__XA_LRMD_RSC_ID, rsc_id);

	pcmk__foreach_ipc_client(send_client_notify, notify);

	pcmk__xml_free(notify);
	}
	}

	static void
	cmd_reset(lrmd_cmd_t * cmd)
	{
	cmd->last_pid = 0;
	#ifdef PCMK__TIME_USE_CGT
	memset(&cmd->t_run, 0, sizeof(cmd->t_run));
	memset(&cmd->t_queue, 0, sizeof(cmd->t_queue));
	#endif
	cmd->epoch_last_run = 0;

	pcmk__reset_result(&(cmd->result));
	cmd->result.execution_status = PCMK_EXEC_DONE;
	}

	static void
	cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc)
	{
	crm_trace("Resource operation rsc:%s action:%s completed (%p %p)", cmd->rsc_id, cmd->action,
	rsc ? rsc->active : NULL, cmd);

	if (rsc && (rsc->active == cmd)) {
	rsc->active = NULL;
	mainloop_set_trigger(rsc->work);
	}

	if (!rsc) {
	cmd->rsc_deleted = 1;
	}

	/* reset original timeout so client notification has correct information */
	cmd->timeout = cmd->timeout_orig;

	send_cmd_complete_notify(cmd);

	if ((cmd->interval_ms != 0)
	&& (cmd->result.execution_status == PCMK_EXEC_CANCELLED)) {

	if (rsc) {
	rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd);
	rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd);
	}
	free_lrmd_cmd(cmd);
	} else if (cmd->interval_ms == 0) {
	if (rsc) {
	rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd);
	}
	free_lrmd_cmd(cmd);
	} else {
	/* Clear all the values pertaining just to the last iteration of a recurring op. */
	cmd_reset(cmd);
	}
	}

	struct notify_new_client_data {
	xmlNode *notify;
	pcmk__client_t *new_client;
	};

	static void
	notify_one_client(gpointer key, gpointer value, gpointer user_data)
	{
	pcmk__client_t *client = value;
	struct notify_new_client_data *data = user_data;

	if (!pcmk__str_eq(client->id, data->new_client->id, pcmk__str_casei)) {
	send_client_notify(key, (gpointer) client, (gpointer) data->notify);
	}
	}

	void
	notify_of_new_client(pcmk__client_t *new_client)
	{
	struct notify_new_client_data data;

	data.new_client = new_client;
	data.notify = pcmk__xe_create(NULL, PCMK__XE_LRMD_NOTIFY);
	crm_xml_add(data.notify, PCMK__XA_LRMD_ORIGIN, __func__);
	crm_xml_add(data.notify, PCMK__XA_LRMD_OP, LRMD_OP_NEW_CLIENT);
	pcmk__foreach_ipc_client(notify_one_client, &data);
	pcmk__xml_free(data.notify);
	}

	void
	client_disconnect_cleanup(const char *client_id)
	{
	GHashTableIter iter;
	lrmd_rsc_t *rsc = NULL;
	char *key = NULL;

	g_hash_table_iter_init(&iter, rsc_list);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & rsc)) {
	if (pcmk_all_flags_set(rsc->call_opts, lrmd_opt_drop_recurring)) {
	/* This client is disconnecting, drop any recurring operations
	* it may have initiated on the resource */
	cancel_all_recurring(rsc, client_id);
	}
	}
	}

	static void
	action_complete(svc_action_t * action)
	{
	lrmd_rsc_t *rsc;
	lrmd_cmd_t *cmd = action->cb_data;
	enum ocf_exitcode code;

	#ifdef PCMK__TIME_USE_CGT
	const char *rclass = NULL;
	bool goagain = false;
	int time_sum = 0;
	int timeout_left = 0;
	int delay = 0;
	#endif

	if (!cmd) {
	crm_err("Completed executor action (%s) does not match any known operations",
	action->id);
	return;
	}

	#ifdef PCMK__TIME_USE_CGT
	if (cmd->result.exit_status != action->rc) {
	cmd->epoch_rcchange = time(NULL);
	}
	#endif

	cmd->last_pid = action->pid;

	// Cast variable instead of function return to keep compilers happy
	code = services_result2ocf(action->standard, cmd->action, action->rc);
	pcmk__set_result(&(cmd->result), (int) code,
	action->status, services__exit_reason(action));

	rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;

	#ifdef PCMK__TIME_USE_CGT
	if (rsc != NULL) {
	rclass = rsc->class;
	#if PCMK__ENABLE_SERVICE
	if (pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_SERVICE,
	pcmk__str_casei)) {
	rclass = resources_find_service_class(rsc->type);
	}
	#endif
	}

	if (!pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_SYSTEMD, pcmk__str_casei)) {
	goto finalize;
	}

	if (pcmk__result_ok(&(cmd->result))
	&& pcmk__strcase_any_of(cmd->action, PCMK_ACTION_START,
	PCMK_ACTION_STOP, NULL)) {
	- /* systemd returns from start and stop actions after the action
	- * begins, not after it completes. We have to jump through a few
	- * hoops so that we don't report 'complete' to the rest of pacemaker
	- * until it's actually done.
	+ /* Getting results for when a start or stop action completes is now
	+ * handled by watching for JobRemoved() signals from systemd and
	+ * reacting to them. So, we can bypass the rest of the code in this
	+ * function for those actions, and simply finalize cmd.
	+ *
	+ * @TODO When monitors are handled in the same way, this function
	+ * can either be drastically simplified or done away with entirely.
	*/
	- goagain = true;
	- cmd->real_action = cmd->action;
	- cmd->action = pcmk__str_copy(PCMK_ACTION_MONITOR);
	+ services__copy_result(action, &(cmd->result));
	+ goto finalize;

	} else if (cmd->result.execution_status == PCMK_EXEC_PENDING &&
	pcmk__str_any_of(cmd->action, PCMK_ACTION_MONITOR, PCMK_ACTION_STATUS, NULL) &&
	cmd->interval_ms == 0 &&
	cmd->real_action == NULL) {
	/* If the state is Pending at the time of probe, execute follow-up monitor. */
	goagain = true;
	cmd->real_action = cmd->action;
	cmd->action = pcmk__str_copy(PCMK_ACTION_MONITOR);
	} else if (cmd->real_action != NULL) {
	// This is follow-up monitor to check whether start/stop/probe(monitor) completed
	if (cmd->result.execution_status == PCMK_EXEC_PENDING) {
	goagain = true;

	} else if (pcmk__result_ok(&(cmd->result))
	&& pcmk__str_eq(cmd->real_action, PCMK_ACTION_STOP,
	pcmk__str_casei)) {
	goagain = true;

	} else {
	int time_sum = time_diff_ms(NULL, &(cmd->t_first_run));
	int timeout_left = cmd->timeout_orig - time_sum;

	crm_debug("%s systemd %s is now complete (elapsed=%dms, "
	"remaining=%dms): %s (%d)",
	cmd->rsc_id, cmd->real_action, time_sum, timeout_left,
	crm_exit_str(cmd->result.exit_status),
	cmd->result.exit_status);
	cmd_original_times(cmd);

	// Monitors may return "not running", but start/stop shouldn't
	if ((cmd->result.execution_status == PCMK_EXEC_DONE)
	&& (cmd->result.exit_status == PCMK_OCF_NOT_RUNNING)) {

	if (pcmk__str_eq(cmd->real_action, PCMK_ACTION_START,
	pcmk__str_casei)) {
	cmd->result.exit_status = PCMK_OCF_UNKNOWN_ERROR;
	} else if (pcmk__str_eq(cmd->real_action, PCMK_ACTION_STOP,
	pcmk__str_casei)) {
	cmd->result.exit_status = PCMK_OCF_OK;
	}
	}
	}
	} else if (pcmk__str_any_of(cmd->action, PCMK_ACTION_MONITOR, PCMK_ACTION_STATUS, NULL)
	&& (cmd->interval_ms > 0)) {
	/* For monitors, excluding follow-up monitors, */
	/* if the pending state persists from the first notification until its timeout, */
	/* it will be treated as a timeout. */

	if ((cmd->result.execution_status == PCMK_EXEC_PENDING) &&
	(cmd->last_notify_op_status == PCMK_EXEC_PENDING)) {
	int time_left = time(NULL) - (cmd->epoch_rcchange + (cmd->timeout_orig/1000));

	if (time_left >= 0) {
	crm_notice("Giving up on %s %s (rc=%d): monitor pending timeout "
	"(first pending notification=%s timeout=%ds)",
	cmd->rsc_id, cmd->action, cmd->result.exit_status,
	pcmk__trim(ctime(&cmd->epoch_rcchange)), cmd->timeout_orig);
	pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
	PCMK_EXEC_TIMEOUT,
	"Investigate reason for timeout, and adjust "
	"configured operation timeout if necessary");
	cmd_original_times(cmd);
	}
	}
	}

	if (!goagain) {
	goto finalize;
	}

	time_sum = time_diff_ms(NULL, &(cmd->t_first_run));
	timeout_left = cmd->timeout_orig - time_sum;
	delay = cmd->timeout_orig / 10;

	if (delay >= timeout_left && timeout_left > 20) {
	delay = timeout_left/2;
	}

	delay = QB_MIN(2000, delay);
	if (delay < timeout_left) {
	cmd->start_delay = delay;
	cmd->timeout = timeout_left;

	if (pcmk__result_ok(&(cmd->result))) {
	crm_debug("%s %s may still be in progress: re-scheduling (elapsed=%dms, remaining=%dms, start_delay=%dms)",
	cmd->rsc_id, cmd->real_action, time_sum, timeout_left, delay);

	} else if (cmd->result.execution_status == PCMK_EXEC_PENDING) {
	crm_info("%s %s is still in progress: re-scheduling (elapsed=%dms, remaining=%dms, start_delay=%dms)",
	cmd->rsc_id, cmd->action, time_sum, timeout_left, delay);

	} else {
	crm_notice("%s %s failed: %s: Re-scheduling (remaining "
	"timeout %s) " QB_XS
	" exitstatus=%d elapsed=%dms start_delay=%dms)",
	cmd->rsc_id, cmd->action,
	crm_exit_str(cmd->result.exit_status),
	pcmk__readable_interval(timeout_left),
	cmd->result.exit_status, time_sum, delay);
	}

	cmd_reset(cmd);
	if (rsc) {
	rsc->active = NULL;
	}
	schedule_lrmd_cmd(rsc, cmd);

	/* Don't finalize cmd, we're not done with it yet */
	return;

	} else {
	crm_notice("Giving up on %s %s (rc=%d): timeout (elapsed=%dms, remaining=%dms)",
	cmd->rsc_id,
	(cmd->real_action? cmd->real_action : cmd->action),
	cmd->result.exit_status, time_sum, timeout_left);
	pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
	PCMK_EXEC_TIMEOUT,
	"Investigate reason for timeout, and adjust "
	"configured operation timeout if necessary");
	cmd_original_times(cmd);
	}
	#endif

	finalize:
	pcmk__set_result_output(&(cmd->result), services__grab_stdout(action),
	services__grab_stderr(action));
	cmd_finalize(cmd, rsc);
	}

	/*!
	* \internal
	* \brief Process the result of a fence device action (start, stop, or monitor)
	*
	* \param[in,out] cmd Fence device action that completed
	* \param[in] exit_status Fencer API exit status for action
	* \param[in] execution_status Fencer API execution status for action
	* \param[in] exit_reason Human-friendly detail, if action failed
	*/
	static void
	stonith_action_complete(lrmd_cmd_t *cmd, int exit_status,
	enum pcmk_exec_status execution_status,
	const char *exit_reason)
	{
	// This can be NULL if resource was removed before command completed
	lrmd_rsc_t *rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id);

	// Simplify fencer exit status to uniform exit status
	if (exit_status != CRM_EX_OK) {
	exit_status = PCMK_OCF_UNKNOWN_ERROR;
	}

	if (cmd->result.execution_status == PCMK_EXEC_CANCELLED) {
	/* An in-flight fence action was cancelled. The execution status is
	* already correct, so don't overwrite it.
	*/
	execution_status = PCMK_EXEC_CANCELLED;

	} else {
	/* Some execution status codes have specific meanings for the fencer
	* that executor clients may not expect, so map them to a simple error
	* status.
	*/
	switch (execution_status) {
	case PCMK_EXEC_NOT_CONNECTED:
	case PCMK_EXEC_INVALID:
	execution_status = PCMK_EXEC_ERROR;
	break;

	case PCMK_EXEC_NO_FENCE_DEVICE:
	/* This should be possible only for probes in practice, but
	* interpret for all actions to be safe.
	*/
	if (pcmk__str_eq(cmd->action, PCMK_ACTION_MONITOR,
	pcmk__str_none)) {
	exit_status = PCMK_OCF_NOT_RUNNING;

	} else if (pcmk__str_eq(cmd->action, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	exit_status = PCMK_OCF_OK;

	} else {
	exit_status = PCMK_OCF_NOT_INSTALLED;
	}
	execution_status = PCMK_EXEC_ERROR;
	break;

	case PCMK_EXEC_NOT_SUPPORTED:
	exit_status = PCMK_OCF_UNIMPLEMENT_FEATURE;
	break;

	default:
	break;
	}
	}

	pcmk__set_result(&cmd->result, exit_status, execution_status, exit_reason);

	// Certain successful actions change the known state of the resource
	if ((rsc != NULL) && pcmk__result_ok(&(cmd->result))) {

	if (pcmk__str_eq(cmd->action, PCMK_ACTION_START, pcmk__str_casei)) {
	pcmk__set_result(&rsc->fence_probe_result, CRM_EX_OK,
	PCMK_EXEC_DONE, NULL); // "running"

	} else if (pcmk__str_eq(cmd->action, PCMK_ACTION_STOP,
	pcmk__str_casei)) {
	pcmk__set_result(&rsc->fence_probe_result, CRM_EX_ERROR,
	PCMK_EXEC_NO_FENCE_DEVICE, NULL); // "not running"
	}
	}

	/* The recurring timer should not be running at this point in any case, but
	* as a failsafe, stop it if it is.
	*/
	stop_recurring_timer(cmd);

	/* Reschedule this command if appropriate. If a recurring command is not
	* rescheduled, its status must be PCMK_EXEC_CANCELLED, otherwise it will
	* not be removed from recurring_ops by cmd_finalize().
	*/
	if (rsc && (cmd->interval_ms > 0)
	&& (cmd->result.execution_status != PCMK_EXEC_CANCELLED)) {
	start_recurring_timer(cmd);
	}

	cmd_finalize(cmd, rsc);
	}

	static void
	lrmd_stonith_callback(stonith_t * stonith, stonith_callback_data_t * data)
	{
	if ((data == NULL) \|\| (data->userdata == NULL)) {
	crm_err("Ignoring fence action result: "
	"Invalid callback arguments (bug?)");
	} else {
	stonith_action_complete((lrmd_cmd_t *) data->userdata,
	stonith__exit_status(data),
	stonith__execution_status(data),
	stonith__exit_reason(data));
	}
	}

	void
	stonith_connection_failed(void)
	{
	GHashTableIter iter;
	lrmd_rsc_t *rsc = NULL;

	crm_warn("Connection to fencer lost (any pending operations for "
	"fence devices will be considered failed)");

	g_hash_table_iter_init(&iter, rsc_list);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &rsc)) {
	if (!pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH,
	pcmk__str_none)) {
	continue;
	}

	/* If we registered this fence device, we don't know whether the
	* fencer still has the registration or not. Cause future probes to
	* return an error until the resource is stopped or started
	* successfully. This is especially important if the controller also
	* went away (possibly due to a cluster layer restart) and won't
	* receive our client notification of any monitors finalized below.
	*/
	if (rsc->fence_probe_result.execution_status == PCMK_EXEC_DONE) {
	pcmk__set_result(&rsc->fence_probe_result, CRM_EX_ERROR,
	PCMK_EXEC_NOT_CONNECTED,
	"Lost connection to fencer");
	}

	// Consider any active, pending, or recurring operations as failed

	for (GList *op = rsc->recurring_ops; op != NULL; op = op->next) {
	lrmd_cmd_t *cmd = op->data;

	/* This won't free a recurring op but instead restart its timer.
	* If cmd is rsc->active, this will set rsc->active to NULL, so we
	* don't have to worry about finalizing it a second time below.
	*/
	stonith_action_complete(cmd,
	CRM_EX_ERROR, PCMK_EXEC_NOT_CONNECTED,
	"Lost connection to fencer");
	}

	if (rsc->active != NULL) {
	rsc->pending_ops = g_list_prepend(rsc->pending_ops, rsc->active);
	}
	while (rsc->pending_ops != NULL) {
	// This will free the op and remove it from rsc->pending_ops
	stonith_action_complete((lrmd_cmd_t *) rsc->pending_ops->data,
	CRM_EX_ERROR, PCMK_EXEC_NOT_CONNECTED,
	"Lost connection to fencer");
	}
	}
	}

	/*!
	* \internal
	* \brief Execute a stonith resource "start" action
	*
	* Start a stonith resource by registering it with the fencer.
	* (Stonith agents don't have a start command.)
	*
	* \param[in,out] stonith_api Connection to fencer
	* \param[in] rsc Stonith resource to start
	* \param[in] cmd Start command to execute
	*
	* \return pcmk_ok on success, -errno otherwise
	*/
	static int
	execd_stonith_start(stonith_t stonith_api, const lrmd_rsc_t rsc,
	const lrmd_cmd_t *cmd)
	{
	char *key = NULL;
	char *value = NULL;
	stonith_key_value_t *device_params = NULL;
	int rc = pcmk_ok;

	// Convert command parameters to stonith API key/values
	if (cmd->params) {
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, cmd->params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	device_params = stonith_key_value_add(device_params, key, value);
	}
	}

	/* The fencer will automatically register devices via CIB notifications
	* when the CIB changes, but to avoid a possible race condition between
	* the fencer receiving the notification and the executor requesting that
	* resource, the executor registers the device as well. The fencer knows how
	* to handle duplicate registrations.
	*/
	rc = stonith_api->cmds->register_device(stonith_api, st_opt_sync_call,
	cmd->rsc_id, rsc->provider,
	rsc->type, device_params);

	stonith_key_value_freeall(device_params, 1, 1);
	return rc;
	}

	/*!
	* \internal
	* \brief Execute a stonith resource "stop" action
	*
	* Stop a stonith resource by unregistering it with the fencer.
	* (Stonith agents don't have a stop command.)
	*
	* \param[in,out] stonith_api Connection to fencer
	* \param[in] rsc Stonith resource to stop
	*
	* \return pcmk_ok on success, -errno otherwise
	*/
	static inline int
	execd_stonith_stop(stonith_t stonith_api, const lrmd_rsc_t rsc)
	{
	/* @TODO Failure would indicate a problem communicating with fencer;
	* perhaps we should try reconnecting and retrying a few times?
	*/
	return stonith_api->cmds->remove_device(stonith_api, st_opt_sync_call,
	rsc->rsc_id);
	}

	/*!
	* \internal
	* \brief Initiate a stonith resource agent recurring "monitor" action
	*
	* \param[in,out] stonith_api Connection to fencer
	* \param[in,out] rsc Stonith resource to monitor
	* \param[in] cmd Monitor command being executed
	*
	* \return pcmk_ok if monitor was successfully initiated, -errno otherwise
	*/
	static inline int
	execd_stonith_monitor(stonith_t stonith_api, lrmd_rsc_t rsc, lrmd_cmd_t *cmd)
	{
	int rc = stonith_api->cmds->monitor(stonith_api, 0, cmd->rsc_id,
	pcmk__timeout_ms2s(cmd->timeout));

	rc = stonith_api->cmds->register_callback(stonith_api, rc, 0, 0, cmd,
	"lrmd_stonith_callback",
	lrmd_stonith_callback);
	if (rc == TRUE) {
	rsc->active = cmd;
	rc = pcmk_ok;
	} else {
	rc = -pcmk_err_generic;
	}
	return rc;
	}

	static void
	execute_stonith_action(lrmd_rsc_t rsc, lrmd_cmd_t cmd)
	{
	int rc = 0;
	bool do_monitor = FALSE;

	stonith_t *stonith_api = get_stonith_connection();

	if (pcmk__str_eq(cmd->action, PCMK_ACTION_MONITOR, pcmk__str_casei)
	&& (cmd->interval_ms == 0)) {
	// Probes don't require a fencer connection
	stonith_action_complete(cmd, rsc->fence_probe_result.exit_status,
	rsc->fence_probe_result.execution_status,
	rsc->fence_probe_result.exit_reason);
	return;

	} else if (stonith_api == NULL) {
	stonith_action_complete(cmd, PCMK_OCF_UNKNOWN_ERROR,
	PCMK_EXEC_NOT_CONNECTED,
	"No connection to fencer");
	return;

	} else if (pcmk__str_eq(cmd->action, PCMK_ACTION_START, pcmk__str_casei)) {
	rc = execd_stonith_start(stonith_api, rsc, cmd);
	if (rc == pcmk_ok) {
	do_monitor = TRUE;
	}

	} else if (pcmk__str_eq(cmd->action, PCMK_ACTION_STOP, pcmk__str_casei)) {
	rc = execd_stonith_stop(stonith_api, rsc);

	} else if (pcmk__str_eq(cmd->action, PCMK_ACTION_MONITOR,
	pcmk__str_casei)) {
	do_monitor = TRUE;

	} else {
	stonith_action_complete(cmd, PCMK_OCF_UNIMPLEMENT_FEATURE,
	PCMK_EXEC_ERROR,
	"Invalid fence device action (bug?)");
	return;
	}

	if (do_monitor) {
	rc = execd_stonith_monitor(stonith_api, rsc, cmd);
	if (rc == pcmk_ok) {
	// Don't clean up yet, we will find out result of the monitor later
	return;
	}
	}

	stonith_action_complete(cmd,
	((rc == pcmk_ok)? CRM_EX_OK : CRM_EX_ERROR),
	stonith__legacy2status(rc),
	((rc == -pcmk_err_generic)? NULL : pcmk_strerror(rc)));
	}

	static void
	execute_nonstonith_action(lrmd_rsc_t rsc, lrmd_cmd_t cmd)
	{
	svc_action_t *action = NULL;
	GHashTable *params_copy = NULL;

	pcmk__assert((rsc != NULL) && (cmd != NULL));

	crm_trace("Creating action, resource:%s action:%s class:%s provider:%s agent:%s",
	rsc->rsc_id, cmd->action, rsc->class, rsc->provider, rsc->type);

	params_copy = pcmk__str_table_dup(cmd->params);

	action = services__create_resource_action(rsc->rsc_id, rsc->class, rsc->provider,
	rsc->type,
	normalize_action_name(rsc, cmd->action),
	cmd->interval_ms, cmd->timeout,
	params_copy, cmd->service_flags);

	if (action == NULL) {
	pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
	PCMK_EXEC_ERROR, strerror(ENOMEM));
	cmd_finalize(cmd, rsc);
	return;
	}

	if (action->rc != PCMK_OCF_UNKNOWN) {
	services__copy_result(action, &(cmd->result));
	services_action_free(action);
	cmd_finalize(cmd, rsc);
	return;
	}

	action->cb_data = cmd;

	if (services_action_async(action, action_complete)) {
	/* The services library has taken responsibility for the action. It
	* could be pending, blocked, or merged into a duplicate recurring
	* action, in which case the action callback (action_complete())
	* will be called when the action completes, otherwise the callback has
	* already been called.
	*
	* action_complete() calls cmd_finalize() which can free cmd, so cmd
	* cannot be used here.
	*/
	} else {
	/* This is a recurring action that is not being cancelled and could not
	* be initiated. It has been rescheduled, and the action callback
	* (action_complete()) has been called, which in this case has already
	* called cmd_finalize(), which in this case should only reset (not
	* free) cmd.
	*/
	services__copy_result(action, &(cmd->result));
	services_action_free(action);
	}
	}

	static gboolean
	execute_resource_action(gpointer user_data)
	{
	lrmd_rsc_t rsc = (lrmd_rsc_t ) user_data;
	lrmd_cmd_t *cmd = NULL;

	CRM_CHECK(rsc != NULL, return FALSE);

	if (rsc->active) {
	crm_trace("%s is still active", rsc->rsc_id);
	return TRUE;
	}

	if (rsc->pending_ops) {
	GList *first = rsc->pending_ops;

	cmd = first->data;
	if (cmd->delay_id) {
	crm_trace
	("Command %s %s was asked to run too early, waiting for start_delay timeout of %dms",
	cmd->rsc_id, cmd->action, cmd->start_delay);
	return TRUE;
	}
	rsc->pending_ops = g_list_remove_link(rsc->pending_ops, first);
	g_list_free_1(first);

	#ifdef PCMK__TIME_USE_CGT
	get_current_time(&(cmd->t_run), &(cmd->t_first_run));
	#endif
	cmd->epoch_last_run = time(NULL);
	}

	if (!cmd) {
	crm_trace("Nothing further to do for %s", rsc->rsc_id);
	return TRUE;
	}

	rsc->active = cmd; /* only one op at a time for a rsc */
	if (cmd->interval_ms) {
	rsc->recurring_ops = g_list_append(rsc->recurring_ops, cmd);
	}

	log_execute(cmd);

	if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	execute_stonith_action(rsc, cmd);
	} else {
	execute_nonstonith_action(rsc, cmd);
	}

	return TRUE;
	}

	void
	free_rsc(gpointer data)
	{
	GList *gIter = NULL;
	lrmd_rsc_t *rsc = data;
	int is_stonith = pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH,
	pcmk__str_casei);

	gIter = rsc->pending_ops;
	while (gIter != NULL) {
	GList *next = gIter->next;
	lrmd_cmd_t *cmd = gIter->data;

	/* command was never executed */
	cmd->result.execution_status = PCMK_EXEC_CANCELLED;
	cmd_finalize(cmd, NULL);

	gIter = next;
	}
	/* frees list, but not list elements. */
	g_list_free(rsc->pending_ops);

	gIter = rsc->recurring_ops;
	while (gIter != NULL) {
	GList *next = gIter->next;
	lrmd_cmd_t *cmd = gIter->data;

	if (is_stonith) {
	cmd->result.execution_status = PCMK_EXEC_CANCELLED;
	/* If a stonith command is in-flight, just mark it as cancelled;
	* it is not safe to finalize/free the cmd until the stonith api
	* says it has either completed or timed out.
	*/
	if (rsc->active != cmd) {
	cmd_finalize(cmd, NULL);
	}
	} else {
	/* This command is already handed off to service library,
	* let service library cancel it and tell us via the callback
	* when it is cancelled. The rsc can be safely destroyed
	* even if we are waiting for the cancel result */
	services_action_cancel(rsc->rsc_id,
	normalize_action_name(rsc, cmd->action),
	cmd->interval_ms);
	}

	gIter = next;
	}
	/* frees list, but not list elements. */
	g_list_free(rsc->recurring_ops);

	free(rsc->rsc_id);
	free(rsc->class);
	free(rsc->provider);
	free(rsc->type);
	mainloop_destroy_trigger(rsc->work);

	free(rsc);
	}

	static int
	process_lrmd_signon(pcmk__client_t client, xmlNode request, int call_id,
	xmlNode **reply)
	{
	int rc = pcmk_ok;
	time_t now = time(NULL);
	const char *protocol_version =
	crm_element_value(request, PCMK__XA_LRMD_PROTOCOL_VERSION);
	const char *start_state = pcmk__env_option(PCMK__ENV_NODE_START_STATE);

	if (compare_version(protocol_version, LRMD_COMPATIBLE_PROTOCOL) < 0) {
	crm_err("Cluster API version must be greater than or equal to %s, not %s",
	LRMD_COMPATIBLE_PROTOCOL, protocol_version);
	rc = -EPROTO;
	}

	if (pcmk__xe_attr_is_true(request, PCMK__XA_LRMD_IS_IPC_PROVIDER)) {
	#ifdef PCMK__COMPILE_REMOTE
	if ((client->remote != NULL)
	&& pcmk_is_set(client->flags,
	pcmk__client_tls_handshake_complete)) {
	const char *op = crm_element_value(request, PCMK__XA_LRMD_OP);

	// This is a remote connection from a cluster node's controller
	ipc_proxy_add_provider(client);

	/* @TODO Allowing multiple proxies makes no sense given that clients
	* have no way to choose between them. Maybe always use the most
	* recent one and switch any existing IPC connections to use it,
	* by iterating over ipc_clients here, and if client->id doesn't
	* match the client's userdata, replace the userdata with the new
	* ID. After the iteration, call lrmd_remote_client_destroy() on any
	* of the replaced values in ipc_providers.
	*/

	/* If this was a register operation, also ask for new schema files but
	* only if it's supported by the protocol version.
	*/
	if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none) &&
	LRMD_SUPPORTS_SCHEMA_XFER(protocol_version)) {
	remoted_request_cib_schema_files();
	}
	} else {
	rc = -EACCES;
	}
	#else
	rc = -EPROTONOSUPPORT;
	#endif
	}

	*reply = create_lrmd_reply(__func__, rc, call_id);
	crm_xml_add(*reply, PCMK__XA_LRMD_OP, CRM_OP_REGISTER);
	crm_xml_add(*reply, PCMK__XA_LRMD_CLIENTID, client->id);
	crm_xml_add(*reply, PCMK__XA_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
	crm_xml_add_ll(*reply, PCMK__XA_UPTIME, now - start_time);

	if (start_state) {
	crm_xml_add(*reply, PCMK__XA_NODE_START_STATE, start_state);
	}

	return rc;
	}

	static int
	process_lrmd_rsc_register(pcmk__client_t client, uint32_t id, xmlNode request)
	{
	int rc = pcmk_ok;
	lrmd_rsc_t *rsc = build_rsc_from_xml(request);
	lrmd_rsc_t *dup = g_hash_table_lookup(rsc_list, rsc->rsc_id);

	if (dup &&
	pcmk__str_eq(rsc->class, dup->class, pcmk__str_casei) &&
	pcmk__str_eq(rsc->provider, dup->provider, pcmk__str_casei) && pcmk__str_eq(rsc->type, dup->type, pcmk__str_casei)) {

	crm_notice("Ignoring duplicate registration of '%s'", rsc->rsc_id);
	free_rsc(rsc);
	return rc;
	}

	g_hash_table_replace(rsc_list, rsc->rsc_id, rsc);
	crm_info("Cached agent information for '%s'", rsc->rsc_id);
	return rc;
	}

	static xmlNode *
	process_lrmd_get_rsc_info(xmlNode *request, int call_id)
	{
	int rc = pcmk_ok;
	xmlNode *rsc_xml = get_xpath_object("//" PCMK__XE_LRMD_RSC, request,
	LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, PCMK__XA_LRMD_RSC_ID);
	xmlNode *reply = NULL;
	lrmd_rsc_t *rsc = NULL;

	if (rsc_id == NULL) {
	rc = -ENODEV;
	} else {
	rsc = g_hash_table_lookup(rsc_list, rsc_id);
	if (rsc == NULL) {
	crm_info("Agent information for '%s' not in cache", rsc_id);
	rc = -ENODEV;
	}
	}

	reply = create_lrmd_reply(__func__, rc, call_id);
	if (rsc) {
	crm_xml_add(reply, PCMK__XA_LRMD_RSC_ID, rsc->rsc_id);
	crm_xml_add(reply, PCMK__XA_LRMD_CLASS, rsc->class);
	crm_xml_add(reply, PCMK__XA_LRMD_PROVIDER, rsc->provider);
	crm_xml_add(reply, PCMK__XA_LRMD_TYPE, rsc->type);
	}
	return reply;
	}

	static int
	process_lrmd_rsc_unregister(pcmk__client_t *client, uint32_t id,
	xmlNode *request)
	{
	int rc = pcmk_ok;
	lrmd_rsc_t *rsc = NULL;
	xmlNode *rsc_xml = get_xpath_object("//" PCMK__XE_LRMD_RSC, request,
	LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, PCMK__XA_LRMD_RSC_ID);

	if (!rsc_id) {
	return -ENODEV;
	}

	rsc = g_hash_table_lookup(rsc_list, rsc_id);
	if (rsc == NULL) {
	crm_info("Ignoring unregistration of resource '%s', which is not registered",
	rsc_id);
	return pcmk_ok;
	}

	if (rsc->active) {
	/* let the caller know there are still active ops on this rsc to watch for */
	crm_trace("Operation (%p) still in progress for unregistered resource %s",
	rsc->active, rsc_id);
	rc = -EINPROGRESS;
	}

	g_hash_table_remove(rsc_list, rsc_id);

	return rc;
	}

	static int
	process_lrmd_rsc_exec(pcmk__client_t client, uint32_t id, xmlNode request)
	{
	lrmd_rsc_t *rsc = NULL;
	lrmd_cmd_t *cmd = NULL;
	xmlNode *rsc_xml = get_xpath_object("//" PCMK__XE_LRMD_RSC, request,
	LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, PCMK__XA_LRMD_RSC_ID);
	int call_id;

	if (!rsc_id) {
	return -EINVAL;
	}
	if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) {
	crm_info("Resource '%s' not found (%d active resources)",
	rsc_id, g_hash_table_size(rsc_list));
	return -ENODEV;
	}

	cmd = create_lrmd_cmd(request, client);
	call_id = cmd->call_id;

	/* Don't reference cmd after handing it off to be scheduled.
	* The cmd could get merged and freed. */
	schedule_lrmd_cmd(rsc, cmd);

	return call_id;
	}

	static int
	cancel_op(const char rsc_id, const char action, guint interval_ms)
	{
	GList *gIter = NULL;
	lrmd_rsc_t *rsc = g_hash_table_lookup(rsc_list, rsc_id);

	/* How to cancel an action.
	* 1. Check pending ops list, if it hasn't been handed off
	* to the service library or stonith recurring list remove
	* it there and that will stop it.
	* 2. If it isn't in the pending ops list, then it's either a
	* recurring op in the stonith recurring list, or the service
	* library's recurring list. Stop it there
	* 3. If not found in any lists, then this operation has either
	* been executed already and is not a recurring operation, or
	* never existed.
	*/
	if (!rsc) {
	return -ENODEV;
	}

	for (gIter = rsc->pending_ops; gIter != NULL; gIter = gIter->next) {
	lrmd_cmd_t *cmd = gIter->data;

	if (action_matches(cmd, action, interval_ms)) {
	cmd->result.execution_status = PCMK_EXEC_CANCELLED;
	cmd_finalize(cmd, rsc);
	return pcmk_ok;
	}
	}

	if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	/* The service library does not handle stonith operations.
	* We have to handle recurring stonith operations ourselves. */
	for (gIter = rsc->recurring_ops; gIter != NULL; gIter = gIter->next) {
	lrmd_cmd_t *cmd = gIter->data;

	if (action_matches(cmd, action, interval_ms)) {
	cmd->result.execution_status = PCMK_EXEC_CANCELLED;
	if (rsc->active != cmd) {
	cmd_finalize(cmd, rsc);
	}
	return pcmk_ok;
	}
	}
	} else if (services_action_cancel(rsc_id,
	normalize_action_name(rsc, action),
	interval_ms) == TRUE) {
	/* The service library will tell the action_complete callback function
	* this action was cancelled, which will destroy the cmd and remove
	* it from the recurring_op list. Do not do that in this function
	* if the service library says it cancelled it. */
	return pcmk_ok;
	}

	return -EOPNOTSUPP;
	}

	static void
	cancel_all_recurring(lrmd_rsc_t * rsc, const char *client_id)
	{
	GList *cmd_list = NULL;
	GList *cmd_iter = NULL;

	/* Notice a copy of each list is created when concat is called.
	* This prevents odd behavior from occurring when the cmd_list
	* is iterated through later on. It is possible the cancel_op
	* function may end up modifying the recurring_ops and pending_ops
	* lists. If we did not copy those lists, our cmd_list iteration
	* could get messed up.*/
	if (rsc->recurring_ops) {
	cmd_list = g_list_concat(cmd_list, g_list_copy(rsc->recurring_ops));
	}
	if (rsc->pending_ops) {
	cmd_list = g_list_concat(cmd_list, g_list_copy(rsc->pending_ops));
	}
	if (!cmd_list) {
	return;
	}

	for (cmd_iter = cmd_list; cmd_iter; cmd_iter = cmd_iter->next) {
	lrmd_cmd_t *cmd = cmd_iter->data;

	if (cmd->interval_ms == 0) {
	continue;
	}

	if (client_id && !pcmk__str_eq(cmd->client_id, client_id, pcmk__str_casei)) {
	continue;
	}

	cancel_op(rsc->rsc_id, cmd->action, cmd->interval_ms);
	}
	/* frees only the copied list data, not the cmds */
	g_list_free(cmd_list);
	}

	static int
	process_lrmd_rsc_cancel(pcmk__client_t client, uint32_t id, xmlNode request)
	{
	xmlNode *rsc_xml = get_xpath_object("//" PCMK__XE_LRMD_RSC, request,
	LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, PCMK__XA_LRMD_RSC_ID);
	const char *action = crm_element_value(rsc_xml, PCMK__XA_LRMD_RSC_ACTION);
	guint interval_ms = 0;

	crm_element_value_ms(rsc_xml, PCMK__XA_LRMD_RSC_INTERVAL, &interval_ms);

	if (!rsc_id \|\| !action) {
	return -EINVAL;
	}

	return cancel_op(rsc_id, action, interval_ms);
	}

	static void
	add_recurring_op_xml(xmlNode reply, lrmd_rsc_t rsc)
	{
	xmlNode *rsc_xml = pcmk__xe_create(reply, PCMK__XE_LRMD_RSC);

	crm_xml_add(rsc_xml, PCMK__XA_LRMD_RSC_ID, rsc->rsc_id);
	for (GList *item = rsc->recurring_ops; item != NULL; item = item->next) {
	lrmd_cmd_t *cmd = item->data;
	xmlNode *op_xml = pcmk__xe_create(rsc_xml, PCMK__XE_LRMD_RSC_OP);

	crm_xml_add(op_xml, PCMK__XA_LRMD_RSC_ACTION,
	pcmk__s(cmd->real_action, cmd->action));
	crm_xml_add_ms(op_xml, PCMK__XA_LRMD_RSC_INTERVAL, cmd->interval_ms);
	crm_xml_add_int(op_xml, PCMK__XA_LRMD_TIMEOUT, cmd->timeout_orig);
	}
	}

	static xmlNode *
	process_lrmd_get_recurring(xmlNode *request, int call_id)
	{
	int rc = pcmk_ok;
	const char *rsc_id = NULL;
	lrmd_rsc_t *rsc = NULL;
	xmlNode *reply = NULL;
	xmlNode *rsc_xml = NULL;

	// Resource ID is optional
	rsc_xml = pcmk__xe_first_child(request, PCMK__XE_LRMD_CALLDATA, NULL, NULL);
	if (rsc_xml) {
	rsc_xml = pcmk__xe_first_child(rsc_xml, PCMK__XE_LRMD_RSC, NULL, NULL);
	}
	if (rsc_xml) {
	rsc_id = crm_element_value(rsc_xml, PCMK__XA_LRMD_RSC_ID);
	}

	// If resource ID is specified, resource must exist
	if (rsc_id != NULL) {
	rsc = g_hash_table_lookup(rsc_list, rsc_id);
	if (rsc == NULL) {
	crm_info("Resource '%s' not found (%d active resources)",
	rsc_id, g_hash_table_size(rsc_list));
	rc = -ENODEV;
	}
	}

	reply = create_lrmd_reply(__func__, rc, call_id);

	// If resource ID is not specified, check all resources
	if (rsc_id == NULL) {
	GHashTableIter iter;
	char *key = NULL;

	g_hash_table_iter_init(&iter, rsc_list);
	while (g_hash_table_iter_next(&iter, (gpointer *) &key,
	(gpointer *) &rsc)) {
	add_recurring_op_xml(reply, rsc);
	}
	} else if (rsc) {
	add_recurring_op_xml(reply, rsc);
	}
	return reply;
	}

	void
	process_lrmd_message(pcmk__client_t client, uint32_t id, xmlNode request)
	{
	int rc = pcmk_ok;
	int call_id = 0;
	const char *op = crm_element_value(request, PCMK__XA_LRMD_OP);
	int do_reply = 0;
	int do_notify = 0;
	xmlNode *reply = NULL;

	/* Certain IPC commands may be done only by privileged users (i.e. root or
	* hacluster), because they would otherwise provide a means of bypassing
	* ACLs.
	*/
	bool allowed = pcmk_is_set(client->flags, pcmk__client_privileged);

	crm_trace("Processing %s operation from %s", op, client->id);
	crm_element_value_int(request, PCMK__XA_LRMD_CALLID, &call_id);

	if (pcmk__str_eq(op, CRM_OP_IPC_FWD, pcmk__str_none)) {
	#ifdef PCMK__COMPILE_REMOTE
	if (allowed) {
	ipc_proxy_forward_client(client, request);
	} else {
	rc = -EACCES;
	}
	#else
	rc = -EPROTONOSUPPORT;
	#endif
	do_reply = 1;
	} else if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
	rc = process_lrmd_signon(client, request, call_id, &reply);
	do_reply = 1;
	} else if (pcmk__str_eq(op, LRMD_OP_RSC_REG, pcmk__str_none)) {
	if (allowed) {
	rc = process_lrmd_rsc_register(client, id, request);
	do_notify = 1;
	} else {
	rc = -EACCES;
	}
	do_reply = 1;
	} else if (pcmk__str_eq(op, LRMD_OP_RSC_INFO, pcmk__str_none)) {
	if (allowed) {
	reply = process_lrmd_get_rsc_info(request, call_id);
	} else {
	rc = -EACCES;
	}
	do_reply = 1;
	} else if (pcmk__str_eq(op, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
	if (allowed) {
	rc = process_lrmd_rsc_unregister(client, id, request);
	/* don't notify anyone about failed un-registers */
	if (rc == pcmk_ok \|\| rc == -EINPROGRESS) {
	do_notify = 1;
	}
	} else {
	rc = -EACCES;
	}
	do_reply = 1;
	} else if (pcmk__str_eq(op, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
	if (allowed) {
	rc = process_lrmd_rsc_exec(client, id, request);
	} else {
	rc = -EACCES;
	}
	do_reply = 1;
	} else if (pcmk__str_eq(op, LRMD_OP_RSC_CANCEL, pcmk__str_none)) {
	if (allowed) {
	rc = process_lrmd_rsc_cancel(client, id, request);
	} else {
	rc = -EACCES;
	}
	do_reply = 1;
	} else if (pcmk__str_eq(op, LRMD_OP_POKE, pcmk__str_none)) {
	do_notify = 1;
	do_reply = 1;
	} else if (pcmk__str_eq(op, LRMD_OP_CHECK, pcmk__str_none)) {
	if (allowed) {
	xmlNode *wrapper = pcmk__xe_first_child(request,
	PCMK__XE_LRMD_CALLDATA,
	NULL, NULL);
	xmlNode *data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);

	const char *timeout = NULL;

	CRM_LOG_ASSERT(data != NULL);
	timeout = crm_element_value(data, PCMK__XA_LRMD_WATCHDOG);
	pcmk__valid_stonith_watchdog_timeout(timeout);
	} else {
	rc = -EACCES;
	}
	} else if (pcmk__str_eq(op, LRMD_OP_ALERT_EXEC, pcmk__str_none)) {
	if (allowed) {
	rc = process_lrmd_alert_exec(client, id, request);
	} else {
	rc = -EACCES;
	}
	do_reply = 1;
	} else if (pcmk__str_eq(op, LRMD_OP_GET_RECURRING, pcmk__str_none)) {
	if (allowed) {
	reply = process_lrmd_get_recurring(request, call_id);
	} else {
	rc = -EACCES;
	}
	do_reply = 1;
	} else {
	rc = -EOPNOTSUPP;
	do_reply = 1;
	crm_err("Unknown IPC request '%s' from client %s",
	op, pcmk__client_name(client));
	}

	if (rc == -EACCES) {
	crm_warn("Rejecting IPC request '%s' from unprivileged client %s",
	op, pcmk__client_name(client));
	}

	crm_debug("Processed %s operation from %s: rc=%d, reply=%d, notify=%d",
	op, client->id, rc, do_reply, do_notify);

	if (do_reply) {
	int send_rc = pcmk_rc_ok;

	if (reply == NULL) {
	reply = create_lrmd_reply(__func__, rc, call_id);
	}
	send_rc = lrmd_server_send_reply(client, id, reply);
	pcmk__xml_free(reply);
	if (send_rc != pcmk_rc_ok) {
	crm_warn("Reply to client %s failed: %s " QB_XS " rc=%d",
	pcmk__client_name(client), pcmk_rc_str(send_rc), send_rc);
	}
	}

	if (do_notify) {
	send_generic_notify(rc, request);
	}
	}
	diff --git a/lib/services/services_private.h b/lib/services/services_private.h
	index 4bd147fa7e..de7e7a4573 100644
	--- a/lib/services/services_private.h
	+++ b/lib/services/services_private.h
	@@ -1,116 +1,116 @@
	/*
	* Copyright 2010-2011 Red Hat, Inc.
	* Later changes copyright 2012-2025 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__SERVICES_SERVICES_PRIVATE__H
	#define PCMK__SERVICES_SERVICES_PRIVATE__H

	#include <unistd.h> // uid_t, gid_t

	#include <glib.h> // G_GNUC_INTERNAL, gboolean, guint, etc.
	#if HAVE_DBUS
	#include <dbus/dbus.h> // DBusPendingCall
	#endif

	#include <crm/common/mainloop.h> // mainloop_io_t
	#include <crm/services.h> // svc_action_t

	#ifdef __cplusplus
	extern "C" {
	#endif

	#define MAX_ARGC 255
	struct svc_action_private_s {
	char *exec;
	char *exit_reason;
	char *args[MAX_ARGC];

	uid_t uid;
	gid_t gid;

	guint repeat_timer;
	void (callback) (svc_action_t op);
	void (fork_callback) (svc_action_t op);

	int stderr_fd;
	mainloop_io_t *stderr_gsource;

	int stdout_fd;
	mainloop_io_t *stdout_gsource;

	int stdin_fd;
	#if HAVE_DBUS
	DBusPendingCall* pending;
	unsigned timerid;
	#endif

	#if SUPPORT_SYSTEMD
	- char *job_path;
	+ char *job_path; // D-Bus object path for enqueued start/stop job
	#endif // SUPPORT_SYSTEMD
	};

	G_GNUC_INTERNAL
	const char services__action_kind(const svc_action_t action);

	G_GNUC_INTERNAL
	GList services_os_get_single_directory_list(const char root, gboolean files,
	gboolean executable);

	G_GNUC_INTERNAL
	GList services_os_get_directory_list(const char root, gboolean files, gboolean executable);

	G_GNUC_INTERNAL
	int services__execute_file(svc_action_t *op);

	G_GNUC_INTERNAL
	gboolean cancel_recurring_action(svc_action_t * op);

	G_GNUC_INTERNAL
	gboolean recurring_action_timer(gpointer data);

	G_GNUC_INTERNAL
	int services__finalize_async_op(svc_action_t *op);

	G_GNUC_INTERNAL
	int services__generic_error(const svc_action_t *op);

	G_GNUC_INTERNAL
	int services__not_installed_error(const svc_action_t *op);

	G_GNUC_INTERNAL
	int services__authorization_error(const svc_action_t *op);

	G_GNUC_INTERNAL
	int services__configuration_error(const svc_action_t *op, bool is_fatal);

	G_GNUC_INTERNAL
	void services__handle_exec_error(svc_action_t * op, int error);

	G_GNUC_INTERNAL
	void services__set_cancelled(svc_action_t *action);

	G_GNUC_INTERNAL
	void services_add_inflight_op(svc_action_t *op);

	G_GNUC_INTERNAL
	void services_untrack_op(const svc_action_t *op);

	G_GNUC_INTERNAL
	gboolean is_op_blocked(const char *rsc);

	#if HAVE_DBUS
	G_GNUC_INTERNAL
	void services_set_op_pending(svc_action_t op, DBusPendingCall pending);
	#endif

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__SERVICES_SERVICES_PRIVATE__H
	diff --git a/lib/services/systemd.c b/lib/services/systemd.c
	index aec7e74088..1f12fa9141 100644
	--- a/lib/services/systemd.c
	+++ b/lib/services/systemd.c
	@@ -1,1262 +1,1415 @@
	/*
	* Copyright 2012-2025 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/services.h>
	#include <crm/services_internal.h>
	#include <crm/common/mainloop.h>

	+#include <inttypes.h> // PRIu32
	#include <stdbool.h>
	+#include <stdint.h> // uint32_t
	#include <stdio.h> // fopen(), NULL, etc.
	#include <sys/stat.h>
	#include <gio/gio.h>
	#include <services_private.h>
	#include <systemd.h>
	#include <dbus/dbus.h>
	#include <pcmk-dbus.h>

	static void invoke_unit_by_path(svc_action_t op, const char unit);

	/* Systemd D-Bus interface
	* https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.systemd1.html
	*/
	#define BUS_NAME "org.freedesktop.systemd1"
	#define BUS_NAME_MANAGER BUS_NAME ".Manager"
	#define BUS_NAME_UNIT BUS_NAME ".Unit"
	#define BUS_PATH "/org/freedesktop/systemd1"

	/*!
	* \internal
	* \brief Prepare a systemd action
	*
	* \param[in,out] op Action to prepare
	*
	* \return Standard Pacemaker return code
	*/
	int
	services__systemd_prepare(svc_action_t *op)
	{
	op->opaque->exec = strdup("systemd-dbus");
	if (op->opaque->exec == NULL) {
	return ENOMEM;
	}
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Map a systemd result to a standard OCF result
	*
	* \param[in] exit_status Systemd result
	*
	* \return Standard OCF result
	*/
	enum ocf_exitcode
	services__systemd2ocf(int exit_status)
	{
	// This library uses OCF codes for systemd actions
	return (enum ocf_exitcode) exit_status;
	}

	static inline DBusMessage *
	systemd_new_method(const char *method)
	{
	crm_trace("Calling: %s on " BUS_NAME_MANAGER, method);
	return dbus_message_new_method_call(BUS_NAME, BUS_PATH, BUS_NAME_MANAGER,
	method);
	}

	/*
	* Functions to manage a static DBus connection
	*/

	static DBusConnection* systemd_proxy = NULL;

	static inline DBusPendingCall *
	systemd_send(DBusMessage *msg,
	void(done)(DBusPendingCall pending, void *user_data),
	void *user_data, int timeout)
	{
	return pcmk_dbus_send(msg, systemd_proxy, done, user_data, timeout);
	}

	static inline DBusMessage *
	systemd_send_recv(DBusMessage msg, DBusError error, int timeout)
	{
	return pcmk_dbus_send_recv(msg, systemd_proxy, error, timeout);
	}

	/*!
	* \internal
	* \brief Send a method to systemd without arguments, and wait for reply
	*
	* \param[in] method Method to send
	*
	* \return Systemd reply on success, NULL (and error will be logged) otherwise
	*
	* \note The caller must call dbus_message_unref() on the reply after
	* handling it.
	*/
	static DBusMessage *
	systemd_call_simple_method(const char *method)
	{
	DBusMessage *msg = NULL;
	DBusMessage *reply = NULL;
	DBusError error;

	/* Don't call systemd_init() here, because that calls this */
	CRM_CHECK(systemd_proxy, return NULL);

	msg = systemd_new_method(method);

	if (msg == NULL) {
	crm_err("Could not create message to send %s to systemd", method);
	return NULL;
	}

	dbus_error_init(&error);
	reply = systemd_send_recv(msg, &error, DBUS_TIMEOUT_USE_DEFAULT);
	dbus_message_unref(msg);

	if (dbus_error_is_set(&error)) {
	crm_err("Could not send %s to systemd: %s (%s)",
	method, error.message, error.name);
	dbus_error_free(&error);
	return NULL;

	} else if (reply == NULL) {
	crm_err("Could not send %s to systemd: no reply received", method);
	return NULL;
	}

	return reply;
	}

	/*!
	* \internal
	* \brief Subscribe to D-Bus signals from systemd
	*
	* Systemd does not broadcast signal messages unless at least one client has
	* called the \c Subscribe() method. Also, a D-Bus client ignores broadcast
	* messages unless an appropriate match rule is set, so we set one here.
	*
	* \return Standard Pacemaker return code
	*/
	static int
	subscribe_to_signals(void)
	{
	const char *match_rule = "type='signal',"
	"sender='" BUS_NAME "',"
	"interface='" BUS_NAME_MANAGER "',"
	"path='" BUS_PATH "'";
	DBusMessage *reply = NULL;
	DBusError error;

	/* Tell D-Bus to accept signal messages from systemd.
	* https://dbus.freedesktop.org/doc/dbus-specification.html#message-bus-routing-match-rules
	*/
	dbus_error_init(&error);
	dbus_bus_add_match(systemd_proxy, match_rule, &error);

	if (dbus_error_is_set(&error)) {
	crm_err("Could not listen for systemd DBus signals: %s " QB_XS " (%s)",
	error.message, error.name);
	dbus_error_free(&error);
	return ECOMM;
	}

	// Tell systemd to broadcast signals
	reply = systemd_call_simple_method("Subscribe");
	if (reply == NULL) {
	dbus_bus_remove_match(systemd_proxy, match_rule, &error);
	return ECOMM;
	}

	dbus_message_unref(reply);
	return pcmk_rc_ok;
	}

	static bool
	systemd_init(void)
	{
	static int need_init = 1;
	// https://dbus.freedesktop.org/doc/api/html/group__DBusConnection.html

	if (systemd_proxy
	&& dbus_connection_get_is_connected(systemd_proxy) == FALSE) {
	crm_warn("Connection to System DBus is closed. Reconnecting...");
	pcmk_dbus_disconnect(systemd_proxy);
	systemd_proxy = NULL;
	need_init = 1;
	}

	if (need_init) {
	need_init = 0;
	systemd_proxy = pcmk_dbus_connect();

	if (subscribe_to_signals() != pcmk_rc_ok) {
	pcmk_dbus_disconnect(systemd_proxy);
	systemd_proxy = NULL;
	}
	}

	return (systemd_proxy != NULL);
	}

	static inline char *
	systemd_get_property(const char unit, const char name,
	void (callback)(const char name, const char value, void userdata),
	void userdata, DBusPendingCall *pending, int timeout)
	{
	return systemd_proxy?
	pcmk_dbus_get_property(systemd_proxy, BUS_NAME, unit, BUS_NAME_UNIT,
	name, callback, userdata, pending, timeout)
	: NULL;
	}

	void
	systemd_cleanup(void)
	{
	if (systemd_proxy) {
	pcmk_dbus_disconnect(systemd_proxy);
	systemd_proxy = NULL;
	}
	}

	/*
	* end of systemd_proxy functions
	*/

	/*!
	* \internal
	* \brief Check whether a file name represents a manageable systemd unit
	*
	* \param[in] name File name to check
	*
	* \return Pointer to "dot" before filename extension if so, NULL otherwise
	*/
	static const char *
	systemd_unit_extension(const char *name)
	{
	if (name) {
	const char *dot = strrchr(name, '.');

	if (dot && (!strcmp(dot, ".service")
	\|\| !strcmp(dot, ".socket")
	\|\| !strcmp(dot, ".mount")
	\|\| !strcmp(dot, ".timer")
	\|\| !strcmp(dot, ".path"))) {
	return dot;
	}
	}
	return NULL;
	}

	static char *
	systemd_unit_name(const char *name, bool add_instance_name)
	{
	const char *dot = NULL;

	if (pcmk__str_empty(name)) {
	return NULL;
	}

	/* Services that end with an @ sign are systemd templates. They expect an
	* instance name to follow the service name. If no instance name was
	* provided, just add "pacemaker" to the string as the instance name. It
	* doesn't seem to matter for purposes of looking up whether a service
	* exists or not.
	*
	* A template can be specified either with or without the unit extension,
	* so this block handles both cases.
	*/
	dot = systemd_unit_extension(name);

	if (dot) {
	if (dot != name && *(dot-1) == '@') {
	return crm_strdup_printf("%.*spacemaker%s",
	(int) (dot - name), name, dot);
	} else {
	return pcmk__str_copy(name);
	}

	} else if (add_instance_name && *(name+strlen(name)-1) == '@') {
	return crm_strdup_printf("%spacemaker.service", name);

	} else {
	return crm_strdup_printf("%s.service", name);
	}
	}

	static void
	systemd_daemon_reload_complete(DBusPendingCall pending, void user_data)
	{
	DBusError error;
	DBusMessage *reply = NULL;
	unsigned int reload_count = GPOINTER_TO_UINT(user_data);

	dbus_error_init(&error);
	if(pending) {
	reply = dbus_pending_call_steal_reply(pending);
	}

	if (pcmk_dbus_find_error(pending, reply, &error)) {
	crm_warn("Could not issue systemd reload %d: %s",
	reload_count, error.message);
	dbus_error_free(&error);

	} else {
	crm_trace("Reload %d complete", reload_count);
	}

	if(pending) {
	dbus_pending_call_unref(pending);
	}
	if(reply) {
	dbus_message_unref(reply);
	}
	}

	static bool
	systemd_daemon_reload(int timeout)
	{
	static unsigned int reload_count = 0;
	DBusMessage *msg = systemd_new_method("Reload");

	reload_count++;
	pcmk__assert(msg != NULL);
	systemd_send(msg, systemd_daemon_reload_complete,
	GUINT_TO_POINTER(reload_count), timeout);
	dbus_message_unref(msg);

	return TRUE;
	}

	/*!
	* \internal
	* \brief Set an action result based on a method error
	*
	* \param[in,out] op Action to set result for
	* \param[in] error Method error
	*/
	static void
	set_result_from_method_error(svc_action_t op, const DBusError error)
	{
	services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	"Unable to invoke systemd DBus method");

	if (strstr(error->name, "org.freedesktop.systemd1.InvalidName")
	\|\| strstr(error->name, "org.freedesktop.systemd1.LoadFailed")
	\|\| strstr(error->name, "org.freedesktop.systemd1.NoSuchUnit")) {

	if (pcmk__str_eq(op->action, PCMK_ACTION_STOP, pcmk__str_casei)) {
	crm_trace("Masking systemd stop failure (%s) for %s "
	"because unknown service can be considered stopped",
	error->name, pcmk__s(op->rsc, "unknown resource"));
	services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
	return;
	}

	services__format_result(op, PCMK_OCF_NOT_INSTALLED,
	PCMK_EXEC_NOT_INSTALLED,
	"systemd unit %s not found", op->agent);
	}

	crm_info("DBus request for %s of systemd unit %s%s%s failed: %s",
	op->action, op->agent,
	((op->rsc == NULL)? "" : " for resource "), pcmk__s(op->rsc, ""),
	error->message);
	}

	/*!
	* \internal
	* \brief Extract unit path from LoadUnit reply, and execute action
	*
	* \param[in] reply LoadUnit reply
	* \param[in,out] op Action to execute (or NULL to just return path)
	*
	* \return DBus object path for specified unit if successful (only valid for
	* lifetime of \p reply), otherwise NULL
	*/
	static const char *
	execute_after_loadunit(DBusMessage reply, svc_action_t op)
	{
	const char *path = NULL;
	DBusError error;

	/* path here is not used other than as a non-NULL flag to indicate that a
	* request was indeed sent
	*/
	if (pcmk_dbus_find_error((void *) &path, reply, &error)) {
	if (op != NULL) {
	set_result_from_method_error(op, &error);
	}
	dbus_error_free(&error);

	} else if (!pcmk_dbus_type_check(reply, NULL, DBUS_TYPE_OBJECT_PATH,
	__func__, __LINE__)) {
	if (op != NULL) {
	services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	"systemd DBus method had unexpected reply");
	crm_info("Could not load systemd unit %s for %s: "
	"DBus reply has unexpected type", op->agent, op->id);
	} else {
	crm_info("Could not load systemd unit: "
	"DBus reply has unexpected type");
	}

	} else {
	dbus_message_get_args (reply, NULL,
	DBUS_TYPE_OBJECT_PATH, &path,
	DBUS_TYPE_INVALID);
	}

	if (op != NULL) {
	if (path != NULL) {
	invoke_unit_by_path(op, path);

	} else if (!(op->synchronous)) {
	services__format_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	"No DBus object found for systemd unit %s",
	op->agent);
	services__finalize_async_op(op);
	}
	}

	return path;
	}

	/*!
	* \internal
	* \brief Execute a systemd action after its LoadUnit completes
	*
	* \param[in,out] pending If not NULL, DBus call associated with LoadUnit
	* \param[in,out] user_data Action to execute
	*/
	static void
	loadunit_completed(DBusPendingCall pending, void user_data)
	{
	DBusMessage *reply = NULL;
	svc_action_t *op = user_data;

	crm_trace("LoadUnit result for %s arrived", op->id);

	// Grab the reply
	if (pending != NULL) {
	reply = dbus_pending_call_steal_reply(pending);
	}

	// The call is no longer pending
	CRM_LOG_ASSERT(pending == op->opaque->pending);
	services_set_op_pending(op, NULL);

	// Execute the desired action based on the reply
	execute_after_loadunit(reply, user_data);
	if (reply != NULL) {
	dbus_message_unref(reply);
	}
	}

	/*!
	* \internal
	* \brief Execute a systemd action, given the unit name
	*
	* \param[in] arg_name Unit name (possibly without ".service" extension)
	* \param[in,out] op Action to execute (if NULL, just get object path)
	* \param[out] path If non-NULL and \p op is NULL or synchronous, where
	* to store DBus object path for specified unit
	*
	* \return Standard Pacemaker return code (for NULL \p op, pcmk_rc_ok means unit
	* was found; for synchronous actions, pcmk_rc_ok means unit was
	* executed, with the actual result stored in \p op; for asynchronous
	* actions, pcmk_rc_ok means action was initiated)
	* \note It is the caller's responsibility to free the path.
	*/
	static int
	invoke_unit_by_name(const char arg_name, svc_action_t op, char **path)
	{
	DBusMessage *msg;
	DBusMessage *reply = NULL;
	DBusPendingCall *pending = NULL;
	char *name = NULL;

	if (pcmk__str_empty(arg_name)) {
	return EINVAL;
	}

	if (!systemd_init()) {
	if (op != NULL) {
	services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	"No DBus connection");
	}
	return ENOTCONN;
	}

	/* Create a LoadUnit DBus method (equivalent to GetUnit if already loaded),
	* which makes the unit usable via further DBus methods.
	*
	* <method name="LoadUnit">
	* <arg name="name" type="s" direction="in"/>
	* <arg name="unit" type="o" direction="out"/>
	* </method>
	*/
	msg = systemd_new_method("LoadUnit");
	pcmk__assert(msg != NULL);

	// Add the (expanded) unit name as the argument
	name = systemd_unit_name(arg_name,
	(op == NULL)
	\|\| pcmk__str_eq(op->action, PCMK_ACTION_META_DATA,
	pcmk__str_none));
	CRM_LOG_ASSERT(dbus_message_append_args(msg, DBUS_TYPE_STRING, &name,
	DBUS_TYPE_INVALID));
	free(name);

	if ((op == NULL) \|\| op->synchronous) {
	// For synchronous ops, wait for a reply and extract the result
	const char *unit = NULL;
	int rc = pcmk_rc_ok;

	reply = systemd_send_recv(msg, NULL,
	(op? op->timeout : DBUS_TIMEOUT_USE_DEFAULT));
	dbus_message_unref(msg);

	unit = execute_after_loadunit(reply, op);
	if (unit == NULL) {
	rc = ENOENT;
	if (path != NULL) {
	*path = NULL;
	}
	} else if (path != NULL) {
	*path = strdup(unit);
	if (*path == NULL) {
	rc = ENOMEM;
	}
	}

	if (reply != NULL) {
	dbus_message_unref(reply);
	}
	return rc;
	}

	// For asynchronous ops, initiate the LoadUnit call and return
	pending = systemd_send(msg, loadunit_completed, op, op->timeout);
	if (pending == NULL) {
	services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	"Unable to send DBus message");
	dbus_message_unref(msg);
	return ECOMM;
	}

	// LoadUnit was successfully initiated
	services__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
	services_set_op_pending(op, pending);
	dbus_message_unref(msg);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Compare two strings alphabetically (case-insensitive)
	*
	* \param[in] a First string to compare
	* \param[in] b Second string to compare
	*
	* \return 0 if strings are equal, -1 if a < b, 1 if a > b
	*
	* \note Usable as a GCompareFunc with g_list_sort().
	* NULL is considered less than non-NULL.
	*/
	static gint
	sort_str(gconstpointer a, gconstpointer b)
	{
	if (!a && !b) {
	return 0;
	} else if (!a) {
	return -1;
	} else if (!b) {
	return 1;
	}
	return strcasecmp(a, b);
	}

	GList *
	systemd_unit_listall(void)
	{
	int nfiles = 0;
	GList *units = NULL;
	DBusMessageIter args;
	DBusMessageIter unit;
	DBusMessageIter elem;
	DBusMessage *reply = NULL;

	if (!systemd_init()) {
	return NULL;
	}

	/*
	" <method name=\"ListUnitFiles\">\n" \
	" <arg name=\"files\" type=\"a(ss)\" direction=\"out\"/>\n" \
	" </method>\n" \
	*/

	reply = systemd_call_simple_method("ListUnitFiles");
	if (reply == NULL) {
	return NULL;
	}
	if (!dbus_message_iter_init(reply, &args)) {
	crm_err("Could not list systemd unit files: systemd reply has no arguments");
	dbus_message_unref(reply);
	return NULL;
	}
	if (!pcmk_dbus_type_check(reply, &args, DBUS_TYPE_ARRAY,
	__func__, __LINE__)) {
	crm_err("Could not list systemd unit files: systemd reply has invalid arguments");
	dbus_message_unref(reply);
	return NULL;
	}

	dbus_message_iter_recurse(&args, &unit);
	for (; dbus_message_iter_get_arg_type(&unit) != DBUS_TYPE_INVALID;
	dbus_message_iter_next(&unit)) {

	DBusBasicValue value;
	const char *match = NULL;
	char *unit_name = NULL;
	char *basename = NULL;

	if(!pcmk_dbus_type_check(reply, &unit, DBUS_TYPE_STRUCT, __func__, __LINE__)) {
	crm_warn("Skipping systemd reply argument with unexpected type");
	continue;
	}

	dbus_message_iter_recurse(&unit, &elem);
	if(!pcmk_dbus_type_check(reply, &elem, DBUS_TYPE_STRING, __func__, __LINE__)) {
	crm_warn("Skipping systemd reply argument with no string");
	continue;
	}

	dbus_message_iter_get_basic(&elem, &value);
	if (value.str == NULL) {
	crm_debug("ListUnitFiles reply did not provide a string");
	continue;
	}
	crm_trace("DBus ListUnitFiles listed: %s", value.str);

	match = systemd_unit_extension(value.str);
	if (match == NULL) {
	// This is not a unit file type we know how to manage
	crm_debug("ListUnitFiles entry '%s' is not supported as resource",
	value.str);
	continue;
	}

	// ListUnitFiles returns full path names, we just want base name
	basename = strrchr(value.str, '/');
	if (basename) {
	basename = basename + 1;
	} else {
	basename = value.str;
	}

	if (!strcmp(match, ".service")) {
	// Service is the "default" unit type, so strip it
	unit_name = strndup(basename, match - basename);
	} else {
	unit_name = strdup(basename);
	}

	nfiles++;
	units = g_list_prepend(units, unit_name);
	}

	dbus_message_unref(reply);

	crm_trace("Found %d manageable systemd unit files", nfiles);
	units = g_list_sort(units, sort_str);
	return units;
	}

	bool
	systemd_unit_exists(const char *name)
	{
	char *path = NULL;
	char *state = NULL;
	int rc = false;

	/* Note: Makes a blocking dbus calls
	* Used by resources_find_service_class() when resource class=service
	*/
	if ((invoke_unit_by_name(name, NULL, &path) != pcmk_rc_ok)
	\|\| (path == NULL)) {
	goto done;
	}

	/* A successful LoadUnit is not sufficient to determine the unit's
	* existence; it merely means the LoadUnit request received a reply.
	* We must make another blocking call to check the LoadState property.
	*/
	state = systemd_get_property(path, "LoadState", NULL, NULL, NULL,
	DBUS_TIMEOUT_USE_DEFAULT);
	rc = pcmk__str_any_of(state, "loaded", "masked", NULL);

	done:
	free(path);
	free(state);
	return rc;
	}

	// @TODO Use XML string constants and maybe a real XML object
	#define METADATA_FORMAT \
	"<?xml " PCMK_XA_VERSION "=\"1.0\"?>\n" \
	"<" PCMK_XE_RESOURCE_AGENT " " \
	PCMK_XA_NAME "=\"%s\" " \
	PCMK_XA_VERSION "=\"" PCMK_DEFAULT_AGENT_VERSION "\">\n" \
	" <" PCMK_XE_VERSION ">1.1</" PCMK_XE_VERSION ">\n" \
	" <" PCMK_XE_LONGDESC " " PCMK_XA_LANG "=\"" PCMK__VALUE_EN "\">\n" \
	" %s\n" \
	" </" PCMK_XE_LONGDESC ">\n" \
	" <" PCMK_XE_SHORTDESC " " PCMK_XA_LANG "=\"" PCMK__VALUE_EN "\">" \
	"systemd unit file for %s" \
	"</" PCMK_XE_SHORTDESC ">\n" \
	" <" PCMK_XE_PARAMETERS "/>\n" \
	" <" PCMK_XE_ACTIONS ">\n" \
	" <" PCMK_XE_ACTION " " PCMK_XA_NAME "=\"" PCMK_ACTION_START "\"" \
	" " PCMK_META_TIMEOUT "=\"100s\" />\n" \
	" <" PCMK_XE_ACTION " " PCMK_XA_NAME "=\"" PCMK_ACTION_STOP "\"" \
	" " PCMK_META_TIMEOUT "=\"100s\" />\n" \
	" <" PCMK_XE_ACTION " " PCMK_XA_NAME "=\"" PCMK_ACTION_STATUS "\"" \
	" " PCMK_META_TIMEOUT "=\"100s\" />\n" \
	" <" PCMK_XE_ACTION " " PCMK_XA_NAME "=\"" PCMK_ACTION_MONITOR "\"" \
	" " PCMK_META_TIMEOUT "=\"100s\"" \
	" " PCMK_META_INTERVAL "=\"60s\" />\n" \
	" <" PCMK_XE_ACTION " " PCMK_XA_NAME "=\"" PCMK_ACTION_META_DATA "\"" \
	" " PCMK_META_TIMEOUT "=\"5s\" />\n" \
	" </" PCMK_XE_ACTIONS ">\n" \
	" <" PCMK_XE_SPECIAL " " PCMK_XA_TAG "=\"systemd\"/>\n" \
	"</" PCMK_XE_RESOURCE_AGENT ">\n"

	static char *
	systemd_unit_metadata(const char *name, int timeout)
	{
	char *meta = NULL;
	char *desc = NULL;
	char *path = NULL;

	if (invoke_unit_by_name(name, NULL, &path) == pcmk_rc_ok) {
	/* TODO: Worth a making blocking call for? Probably not. Possibly if cached. */
	desc = systemd_get_property(path, "Description", NULL, NULL, NULL,
	timeout);
	} else {
	desc = crm_strdup_printf("Systemd unit file for %s", name);
	}

	if (pcmk__xml_needs_escape(desc, pcmk__xml_escape_text)) {
	gchar *escaped = pcmk__xml_escape(desc, pcmk__xml_escape_text);

	meta = crm_strdup_printf(METADATA_FORMAT, name, escaped, name);
	g_free(escaped);

	} else {
	meta = crm_strdup_printf(METADATA_FORMAT, name, desc, name);
	}

	free(desc);
	free(path);
	return meta;
	}

	/*!
	* \internal
	* \brief Determine result of method from reply
	*
	* \param[in] reply Reply to start, stop, or restart request
	* \param[in,out] op Action that was executed
	*/
	static void
	process_unit_method_reply(DBusMessage reply, svc_action_t op)
	{
	+ bool start_stop = pcmk__strcase_any_of(op->action, PCMK_ACTION_START,
	+ PCMK_ACTION_STOP, NULL);
	DBusError error;

	dbus_error_init(&error);

	/* The first use of error here is not used other than as a non-NULL flag to
	* indicate that a request was indeed sent
	*/
	if (pcmk_dbus_find_error((void *) &error, reply, &error)) {
	set_result_from_method_error(op, &error);
	dbus_error_free(&error);

	} else if (!pcmk_dbus_type_check(reply, NULL, DBUS_TYPE_OBJECT_PATH,
	__func__, __LINE__)) {
	+ const char *reason = "systemd D-Bus method had unexpected reply";
	+
	crm_info("DBus request for %s of %s succeeded but "
	"return type was unexpected",
	op->action, pcmk__s(op->rsc, "unknown resource"));
	- services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE,
	- "systemd DBus method had unexpected reply");
	+
	+ if (!op->synchronous && start_stop) {
	+ /* The start or stop job is enqueued but is not complete. We need a
	+ * job path to detect completion in job_removed_filter().
	+ */
	+ services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	+ reason);
	+
	+ } else {
	+ /* Something weird happened, but the action is finished and there
	+ * was no D-Bus error. So call it a success.
	+ */
	+ services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, reason);
	+ }

	} else {
	const char *path = NULL;

	dbus_message_get_args(reply, NULL,
	DBUS_TYPE_OBJECT_PATH, &path,
	DBUS_TYPE_INVALID);

	crm_debug("DBus request for %s of %s using %s succeeded",
	op->action, pcmk__s(op->rsc, "unknown resource"), path);
	- pcmk__str_update(&(op->opaque->job_path), path);
	- services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
	+
	+ if (!op->synchronous && start_stop) {
	+ // Should be set to unknown/pending already
	+ services__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
	+ pcmk__str_update(&(op->opaque->job_path), path);
	+
	+ } else {
	+ services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
	+ }
	+ }
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Process a systemd \c JobRemoved signal for a given service action
	+ *
	+ * This filter is expected to be added with \c finalize_async_action_dbus() as
	+ * the \c free_data_function. Then if \p message is a \c JobRemoved signal for
	+ * the action specified by \p user_data, the action's result is set, the filter
	+ * is removed, and the action is finalized.
	+ *
	+ * \param[in,out] connection D-Bus connection
	+ * \param[in] message D-Bus message
	+ * \param[in,out] user_data Service action (\c svc_action_t)
	+ *
	+ * \retval \c DBUS_HANDLER_RESULT_HANDLED if \p message is a \c JobRemoved
	+ * signal for \p user_data
	+ * \retval \c DBUS_HANDLER_RESULT_NOT_YET_HANDLED otherwise (on error, if
	+ * \p message is not a \c JobRemoved signal, or if the signal is for
	+ * some other action's job)
	+ */
	+static DBusHandlerResult
	+job_removed_filter(DBusConnection connection, DBusMessage message,
	+ void *user_data)
	+{
	+ svc_action_t *action = user_data;
	+ const char *action_name = NULL;
	+ uint32_t job_id = 0;
	+ const char *bus_path = NULL;
	+ const char *unit_name = NULL;
	+ const char *result = NULL;
	+ DBusError error;
	+
	+ CRM_CHECK((connection != NULL) && (message != NULL),
	+ return DBUS_HANDLER_RESULT_NOT_YET_HANDLED);
	+
	+ // action should always be set when the filter is added
	+ if ((action == NULL)
	+ \|\| !dbus_message_is_signal(message, BUS_NAME_MANAGER, "JobRemoved")) {
	+ return DBUS_HANDLER_RESULT_NOT_YET_HANDLED;
	+ }
	+
	+ dbus_error_init(&error);
	+ if (!dbus_message_get_args(message, &error,
	+ DBUS_TYPE_UINT32, &job_id,
	+ DBUS_TYPE_OBJECT_PATH, &bus_path,
	+ DBUS_TYPE_STRING, &unit_name,
	+ DBUS_TYPE_STRING, &result,
	+ DBUS_TYPE_INVALID)) {
	+ crm_err("Could not interpret systemd DBus signal: %s " QB_XS " (%s)",
	+ error.message, error.name);
	+ dbus_error_free(&error);
	+ return DBUS_HANDLER_RESULT_NOT_YET_HANDLED;
	+ }
	+
	+ if (!pcmk__str_eq(bus_path, action->opaque->job_path, pcmk__str_none)) {
	+ // This filter is not for this job
	+ return DBUS_HANDLER_RESULT_NOT_YET_HANDLED;
	}
	+
	+ action_name = pcmk__s(action->action, "(unknown)");
	+
	+ crm_trace("Setting %s result for %s (JobRemoved id=%" PRIu32 ", result=%s",
	+ action_name, unit_name, job_id, result);
	+
	+ if (pcmk__str_eq(result, "done", pcmk__str_none)) {
	+ services__set_result(action, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
	+
	+ } else if (pcmk__str_eq(result, "timeout", pcmk__str_none)) {
	+ services__format_result(action, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_TIMEOUT,
	+ "systemd %s job for %s timed out",
	+ action_name, unit_name);
	+
	+ } else {
	+ services__format_result(action, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	+ "systemd %s job for %s failed with result '%s'",
	+ action_name, unit_name, result);
	+ }
	+
	+ /* This instance of the filter was specifically for the given action.
	+ *
	+ * The action gets finalized by services__finalize_async_op() via the
	+ * filter's free_data_function.
	+ */
	+ dbus_connection_remove_filter(systemd_proxy, job_removed_filter, action);
	+ return DBUS_HANDLER_RESULT_HANDLED;
	+}
	+
	+/*!
	+ * \internal
	+ * \brief \c DBusFreeFunction wrapper for \c services__finalize_async_op()
	+ *
	+ * \param[in,out] action Asynchronous service action to finalize
	+ */
	+static void
	+finalize_async_action_dbus(void *action)
	+{
	+ services__finalize_async_op((svc_action_t *) action);
	}

	/*!
	* \internal
	* \brief Process the completion of an asynchronous unit start, stop, or restart
	*
	* \param[in,out] pending If not NULL, DBus call associated with request
	* \param[in,out] user_data Action that was executed
	*/
	static void
	unit_method_complete(DBusPendingCall pending, void user_data)
	{
	DBusMessage *reply = NULL;
	svc_action_t *op = user_data;

	crm_trace("Result for %s arrived", op->id);

	// Grab the reply
	if (pending != NULL) {
	reply = dbus_pending_call_steal_reply(pending);
	}

	// The call is no longer pending
	CRM_LOG_ASSERT(pending == op->opaque->pending);
	services_set_op_pending(op, NULL);

	- // Determine result and finalize action
	process_unit_method_reply(reply, op);
	- services__finalize_async_op(op);
	+
	if (reply != NULL) {
	dbus_message_unref(reply);
	}
	+
	+ if ((op->status == PCMK_EXEC_PENDING)
	+ && pcmk__strcase_any_of(op->action, PCMK_ACTION_START, PCMK_ACTION_STOP,
	+ NULL)) {
	+ /* Start and stop method calls return when the job is enqueued, not when
	+ * it's complete. Start and stop actions must be finalized after the job
	+ * is complete, because the action callback function may use it. We add
	+ * a message filter to process the JobRemoved signal, which indicates
	+ * completion.
	+ *
	+ * The filter takes ownership of op, which will be finalized when the
	+ * filter is later removed.
	+ */
	+ if (dbus_connection_add_filter(systemd_proxy, job_removed_filter, op,
	+ finalize_async_action_dbus)) {
	+ return;
	+ }
	+ crm_err("Could not add D-Bus filter for systemd JobRemoved signals");
	+ services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	+ "Failed to add D-Bus filter for systemd "
	+ "JobRemoved signal");
	+ }
	+ services__finalize_async_op(op);
	}

	/* When the cluster manages a systemd resource, we create a unit file override
	* to order the service "before" pacemaker. The "before" relationship won't
	* actually be used, since systemd won't ever start the resource -- we're
	* interested in the reverse shutdown ordering it creates, to ensure that
	* systemd doesn't stop the resource at shutdown while pacemaker is still
	* running.
	*
	* @TODO Add start timeout
	*/
	#define SYSTEMD_UNIT_OVERRIDE_TEMPLATE \
	"[Unit]\n" \
	"Description=Cluster Controlled %s\n" \
	"Before=pacemaker.service pacemaker_remote.service\n"

	#define SYSTEMD_SERVICE_OVERRIDE \
	"\n" \
	"[Service]\n" \
	"Restart=no\n"

	/*!
	* \internal
	* \brief Get runtime drop-in directory path for a systemd unit
	*
	* \param[in] unit_name Systemd unit (with extension)
	*
	* \return Drop-in directory path
	*/
	static GString *
	get_override_dir(const char *unit_name)
	{
	GString *buf = g_string_sized_new(128);

	pcmk__g_strcat(buf, "/run/systemd/system/", unit_name, ".d", NULL);
	return buf;
	}

	/*!
	* \internal
	* \brief Append systemd override filename to a directory path
	*
	* \param[in,out] buf Buffer containing directory path to append to
	*/
	static inline void
	append_override_basename(GString *buf)
	{
	g_string_append(buf, "/50-pacemaker.conf");
	}

	/*!
	* \internal
	* \brief Create a runtime override file for a systemd unit
	*
	* The systemd daemon is then reloaded. This file does not survive a reboot.
	*
	* \param[in] agent Systemd resource agent
	* \param[in] timeout Timeout for systemd daemon reload
	*
	* \return Standard Pacemaker return code
	*
	* \note Any configuration in \c /etc takes precedence over our drop-in.
	* \todo Document this in Pacemaker Explained or Administration?
	*/
	static int
	systemd_create_override(const char *agent, int timeout)
	{
	char *unit_name = NULL;
	GString *filename = NULL;
	GString *override = NULL;
	FILE *fp = NULL;
	int fd = 0;
	int rc = pcmk_rc_ok;

	unit_name = systemd_unit_name(agent, false);
	CRM_CHECK(!pcmk__str_empty(unit_name),
	rc = EINVAL; goto done);

	filename = get_override_dir(unit_name);
	rc = pcmk__build_path(filename->str, 0755);
	if (rc != pcmk_rc_ok) {
	crm_err("Could not create systemd override directory %s: %s",
	filename->str, pcmk_rc_str(rc));
	goto done;
	}

	append_override_basename(filename);
	fp = fopen(filename->str, "w");
	if (fp == NULL) {
	rc = errno;
	crm_err("Cannot open systemd override file %s for writing: %s",
	filename->str, pcmk_rc_str(rc));
	goto done;
	}

	// Ensure the override file is world-readable (avoid systemd warning in log)
	fd = fileno(fp);
	if ((fd < 0) \|\| (fchmod(fd, S_IRUSR\|S_IWUSR\|S_IRGRP\|S_IROTH) < 0)) {
	rc = errno;
	crm_err("Failed to set permissions on systemd override file %s: %s",
	filename->str, pcmk_rc_str(rc));
	goto done;
	}

	override = g_string_sized_new(2 * sizeof(SYSTEMD_UNIT_OVERRIDE_TEMPLATE));
	g_string_printf(override, SYSTEMD_UNIT_OVERRIDE_TEMPLATE, unit_name);
	if (pcmk__ends_with_ext(unit_name, ".service")) {
	g_string_append(override, SYSTEMD_SERVICE_OVERRIDE);
	}

	if (fputs(override->str, fp) == EOF) {
	rc = EIO;
	crm_err("Cannot write to systemd override file %s", filename->str);
	}

	done:
	if (fp != NULL) {
	fclose(fp);
	}

	if (rc == pcmk_rc_ok) {
	// @TODO Make sure the reload succeeds
	systemd_daemon_reload(timeout);

	} else if (fp != NULL) {
	// File was created, so remove it
	unlink(filename->str);
	}

	free(unit_name);

	// coverity[check_after_deref] False positive
	if (filename != NULL) {
	g_string_free(filename, TRUE);
	}
	if (override != NULL) {
	g_string_free(override, TRUE);
	}
	return rc;
	}

	static void
	systemd_remove_override(const char *agent, int timeout)
	{
	char *unit_name = systemd_unit_name(agent, false);
	GString *filename = NULL;

	CRM_CHECK(!pcmk__str_empty(unit_name), goto done);

	filename = get_override_dir(unit_name);
	append_override_basename(filename);

	if (unlink(filename->str) < 0) {
	int rc = errno;

	if (rc != ENOENT) {
	// Stop may be called when already stopped, which is fine
	crm_warn("Cannot remove systemd override file %s: %s",
	filename->str, pcmk_rc_str(rc));
	}

	} else {
	systemd_daemon_reload(timeout);
	}

	done:
	free(unit_name);

	// coverity[check_after_deref] False positive
	if (filename != NULL) {
	g_string_free(filename, TRUE);
	}
	}

	/*!
	* \internal
	* \brief Parse result of systemd status check
	*
	* Set a status action's exit status and execution status based on a DBus
	* property check result, and finalize the action if asynchronous.
	*
	* \param[in] name DBus interface name for property that was checked
	* \param[in] state Property value
	* \param[in,out] userdata Status action that check was done for
	*/
	static void
	parse_status_result(const char name, const char state, void *userdata)
	{
	svc_action_t *op = userdata;

	crm_trace("Resource %s has %s='%s'",
	pcmk__s(op->rsc, "(unspecified)"), name,
	pcmk__s(state, "<null>"));

	if (pcmk__str_eq(state, "active", pcmk__str_none)) {
	services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);

	} else if (pcmk__str_eq(state, "reloading", pcmk__str_none)) {
	services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);

	} else if (pcmk__str_eq(state, "activating", pcmk__str_none)) {
	services__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);

	} else if (pcmk__str_eq(state, "deactivating", pcmk__str_none)) {
	services__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);

	} else {
	services__set_result(op, PCMK_OCF_NOT_RUNNING, PCMK_EXEC_DONE, state);
	}

	if (!(op->synchronous)) {
	services_set_op_pending(op, NULL);
	services__finalize_async_op(op);
	}
	}

	/*!
	* \internal
	* \brief Invoke a systemd unit, given its DBus object path
	*
	* \param[in,out] op Action to execute
	* \param[in] unit DBus object path of systemd unit to invoke
	*/
	static void
	invoke_unit_by_path(svc_action_t op, const char unit)
	{
	const char *method = NULL;
	DBusMessage *msg = NULL;
	DBusMessage *reply = NULL;

	if (pcmk__str_any_of(op->action, PCMK_ACTION_MONITOR, PCMK_ACTION_STATUS,
	NULL)) {
	DBusPendingCall *pending = NULL;
	char *state;

	state = systemd_get_property(unit, "ActiveState",
	(op->synchronous? NULL : parse_status_result),
	op, (op->synchronous? NULL : &pending),
	op->timeout);
	if (op->synchronous) {
	parse_status_result("ActiveState", state, op);
	free(state);

	} else if (pending == NULL) { // Could not get ActiveState property
	services__format_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	"Could not get state for unit %s from DBus",
	op->agent);
	services__finalize_async_op(op);

	} else {
	services_set_op_pending(op, pending);
	}
	return;

	} else if (pcmk__str_eq(op->action, PCMK_ACTION_START, pcmk__str_none)) {
	int rc = pcmk_rc_ok;

	method = "StartUnit";
	rc = systemd_create_override(op->agent, op->timeout);
	if (rc != pcmk_rc_ok) {
	services__format_result(op, pcmk_rc2ocf(rc), PCMK_EXEC_ERROR,
	"Failed to create systemd override file "
	"for %s",
	pcmk__s(op->agent, "(unspecified)"));
	if (!(op->synchronous)) {
	services__finalize_async_op(op);
	}
	return;
	}

	} else if (pcmk__str_eq(op->action, PCMK_ACTION_STOP, pcmk__str_none)) {
	method = "StopUnit";
	systemd_remove_override(op->agent, op->timeout);

	} else if (pcmk__str_eq(op->action, "restart", pcmk__str_none)) {
	method = "RestartUnit";

	} else {
	services__format_result(op, PCMK_OCF_UNIMPLEMENT_FEATURE,
	PCMK_EXEC_ERROR,
	"Action %s not implemented "
	"for systemd resources",
	pcmk__s(op->action, "(unspecified)"));
	if (!(op->synchronous)) {
	services__finalize_async_op(op);
	}
	return;
	}

	crm_trace("Calling %s for unit path %s%s%s",
	method, unit,
	((op->rsc == NULL)? "" : " for resource "), pcmk__s(op->rsc, ""));

	msg = systemd_new_method(method);
	pcmk__assert(msg != NULL);

	/* (ss) */
	{
	const char *replace_s = "replace";
	char *name = systemd_unit_name(op->agent,
	pcmk__str_eq(op->action,
	PCMK_ACTION_META_DATA,
	pcmk__str_none));

	CRM_LOG_ASSERT(dbus_message_append_args(msg, DBUS_TYPE_STRING, &name, DBUS_TYPE_INVALID));
	CRM_LOG_ASSERT(dbus_message_append_args(msg, DBUS_TYPE_STRING, &replace_s, DBUS_TYPE_INVALID));

	free(name);
	}

	if (op->synchronous) {
	reply = systemd_send_recv(msg, NULL, op->timeout);
	dbus_message_unref(msg);
	process_unit_method_reply(reply, op);
	if (reply != NULL) {
	dbus_message_unref(reply);
	}

	} else {
	DBusPendingCall *pending = systemd_send(msg, unit_method_complete, op,
	op->timeout);

	dbus_message_unref(msg);
	if (pending == NULL) {
	services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	"Unable to send DBus message");
	services__finalize_async_op(op);

	} else {
	services_set_op_pending(op, pending);
	}
	}
	}

	static gboolean
	systemd_timeout_callback(gpointer p)
	{
	svc_action_t * op = p;

	op->opaque->timerid = 0;
	crm_info("%s action for systemd unit %s named '%s' timed out",
	op->action, op->agent, op->rsc);
	services__format_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_TIMEOUT,
	"%s action for systemd unit %s "
	"did not complete in time", op->action, op->agent);
	- services__finalize_async_op(op);
	+
	+ if (op->opaque->job_path != NULL) {
	+ // A filter owns this op
	+ dbus_connection_remove_filter(systemd_proxy, job_removed_filter, op);
	+
	+ } else {
	+ services__finalize_async_op(op);
	+ }
	return FALSE;
	}

	/*!
	* \internal
	* \brief Execute a systemd action
	*
	* \param[in,out] op Action to execute
	*
	* \return Standard Pacemaker return code
	* \retval EBUSY Recurring operation could not be initiated
	* \retval pcmk_rc_error Synchronous action failed
	* \retval pcmk_rc_ok Synchronous action succeeded, or asynchronous action
	* should not be freed (because it's pending or because
	* it failed to execute and was already freed)
	*
	* \note If the return value for an asynchronous action is not pcmk_rc_ok, the
	* caller is responsible for freeing the action.
	*/
	int
	services__execute_systemd(svc_action_t *op)
	{
	pcmk__assert(op != NULL);

	if (pcmk__str_empty(op->action) \|\| pcmk__str_empty(op->agent)) {
	services__set_result(op, PCMK_OCF_NOT_CONFIGURED, PCMK_EXEC_ERROR_FATAL,
	"Bug in action caller");
	goto done;
	}

	if (!systemd_init()) {
	services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	"No DBus connection");
	goto done;
	}

	crm_debug("Performing %ssynchronous %s op on systemd unit %s%s%s",
	(op->synchronous? "" : "a"), op->action, op->agent,
	((op->rsc == NULL)? "" : " for resource "), pcmk__s(op->rsc, ""));

	if (pcmk__str_eq(op->action, PCMK_ACTION_META_DATA, pcmk__str_casei)) {
	op->stdout_data = systemd_unit_metadata(op->agent, op->timeout);
	services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
	goto done;
	}

	/* invoke_unit_by_name() should always override these values, which are here
	* just as a fail-safe in case there are any code paths that neglect to
	*/
	services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	"Bug in service library");

	if (invoke_unit_by_name(op->agent, op, NULL) == pcmk_rc_ok) {
	+ // @TODO Why plus 5000? No explanation in fccd046.
	op->opaque->timerid = pcmk__create_timer(op->timeout + 5000,
	systemd_timeout_callback, op);
	services_add_inflight_op(op);
	return pcmk_rc_ok;
	}

	done:
	if (op->synchronous) {
	return (op->rc == PCMK_OCF_OK)? pcmk_rc_ok : pcmk_rc_error;
	} else {
	return services__finalize_async_op(op);
	}
	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 12:58 PM (1 d, 2 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1664863
Default Alt Text: (118 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions