diff --git a/include/crm/lrmd.h b/include/crm/lrmd.h index ffbc325498..067461015d 100644 --- a/include/crm/lrmd.h +++ b/include/crm/lrmd.h @@ -1,417 +1,455 @@ /* * Copyright (c) 2012 David Vossel * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * */ /** * \file * \brief Local Resource Manager * \ingroup lrm */ #ifndef LRMD__H #define LRMD__H typedef struct lrmd_s lrmd_t; typedef struct lrmd_key_value_s { char *key; char *value; struct lrmd_key_value_s *next; } lrmd_key_value_t; /* *INDENT-OFF* */ +#define DEFAULT_REMOTE_KEY_LOCATION "/etc/pacemaker/authkey" +#define ALT_REMOTE_KEY_LOCATION "/etc/corosync/authkey" +#define DEFAULT_REMOTE_PORT 1984 +#define DEFAULT_REMOTE_USERNAME "lrmd" + #define F_LRMD_OPERATION "lrmd_op" #define F_LRMD_CLIENTNAME "lrmd_clientname" #define F_LRMD_CLIENTID "lrmd_clientid" +#define F_LRMD_REMOTE_MSG_TYPE "lrmd_remote_msg_type" +#define F_LRMD_REMOTE_MSG_ID "lrmd_remote_msg_id" #define F_LRMD_CALLBACK_TOKEN "lrmd_async_id" #define F_LRMD_CALLID "lrmd_callid" #define F_LRMD_CANCEL_CALLID "lrmd_cancel_callid" #define F_LRMD_CALLOPTS "lrmd_callopt" #define F_LRMD_CALLDATA "lrmd_calldata" #define F_LRMD_RC "lrmd_rc" #define F_LRMD_EXEC_RC "lrmd_exec_rc" #define F_LRMD_OP_STATUS "lrmd_exec_op_status" #define F_LRMD_TIMEOUT "lrmd_timeout" #define F_LRMD_CLASS "lrmd_class" #define F_LRMD_PROVIDER "lrmd_provider" #define F_LRMD_TYPE "lrmd_type" #define F_LRMD_ORIGIN "lrmd_origin" #define F_LRMD_RSC_RUN_TIME "lrmd_run_time" #define F_LRMD_RSC_RCCHANGE_TIME "lrmd_rcchange_time" #define F_LRMD_RSC_EXEC_TIME "lrmd_exec_time" #define F_LRMD_RSC_QUEUE_TIME "lrmd_queue_time" #define F_LRMD_RSC_ID "lrmd_rsc_id" #define F_LRMD_RSC_ACTION "lrmd_rsc_action" #define F_LRMD_RSC_USERDATA_STR "lrmd_rsc_userdata_str" #define F_LRMD_RSC_OUTPUT "lrmd_rsc_output" #define F_LRMD_RSC_START_DELAY "lrmd_rsc_start_delay" #define F_LRMD_RSC_INTERVAL "lrmd_rsc_interval" #define F_LRMD_RSC_METADATA "lrmd_rsc_metadata_res" #define F_LRMD_RSC_DELETED "lrmd_rsc_deleted" #define F_LRMD_RSC "lrmd_rsc" #define LRMD_OP_RSC_CHK_REG "lrmd_rsc_check_register" #define LRMD_OP_RSC_REG "lrmd_rsc_register" #define LRMD_OP_RSC_EXEC "lrmd_rsc_exec" #define LRMD_OP_RSC_CANCEL "lrmd_rsc_cancel" #define LRMD_OP_RSC_UNREG "lrmd_rsc_unregister" #define LRMD_OP_RSC_INFO "lrmd_rsc_info" #define LRMD_OP_RSC_METADATA "lrmd_rsc_metadata" #define T_LRMD "lrmd" #define T_LRMD_REPLY "lrmd_reply" #define T_LRMD_NOTIFY "lrmd_notify" /* *INDENT-ON* */ +/*! + * \brief Create a new local lrmd connection + */ lrmd_t *lrmd_api_new(void); + +/*! + * \brief Create a new remote lrmd connection using tls backend + */ +lrmd_t *lrmd_remote_api_new(const char *server, int port); + +/*! + * \brief Use after lrmd_poll returns 1. + * + * \param fd to poll on + * \param timeout in ms + * + * \retval true - connection is still up + * \retval false - disconnected + */ bool lrmd_dispatch(lrmd_t *lrmd); + +/*! + * \brief Poll for a specified timeout period to determine if a message + * is ready for dispatch. + * \retval 1 msg is ready + * \retval 0 timeout occured + * \retval negative error code + */ +int lrmd_poll(lrmd_t *lrmd, int timeout); + +/*! + * \brief Destroy lrmd object + */ void lrmd_api_delete(lrmd_t * lrmd); lrmd_key_value_t *lrmd_key_value_add(lrmd_key_value_t *kvp, const char *key, const char *value); /* *INDENT-OFF* */ /* Reserved for future use */ enum lrmd_call_options { lrmd_opt_none = 0x00000000, /* lrmd_opt_sync_call = 0x00000001, //Not implemented, patches welcome. */ /*! Only notify the client originating a exec() the results */ lrmd_opt_notify_orig_only = 0x00000002, /*! Drop recurring operations initiated by a client when client disconnects. * This call_option is only valid when registering a resource. */ lrmd_opt_drop_recurring = 0x00000003, /*! Only send out notifications for recurring operations whenthe result changes */ lrmd_opt_notify_changes_only = 0x00000004, }; enum lrmd_callback_event { lrmd_event_register, lrmd_event_unregister, lrmd_event_exec_complete, lrmd_event_disconnect, }; enum lrmd_exec_rc { PCMK_EXECRA_OK = 0, PCMK_EXECRA_UNKNOWN_ERROR = 1, PCMK_EXECRA_INVALID_PARAM = 2, PCMK_EXECRA_UNIMPLEMENT_FEATURE = 3, PCMK_EXECRA_INSUFFICIENT_PRIV = 4, PCMK_EXECRA_NOT_INSTALLED = 5, PCMK_EXECRA_NOT_CONFIGURED = 6, PCMK_EXECRA_NOT_RUNNING = 7, PCMK_EXECRA_RUNNING_MASTER = 8, PCMK_EXECRA_FAILED_MASTER = 9, /* For status command only */ PCMK_EXECRA_STATUS_UNKNOWN = 14, }; /* *INDENT-ON* */ typedef struct lrmd_event_data_s { /*! Type of event, register, unregister, call_completed... */ enum lrmd_callback_event type; /*! The resource this event occurred on. */ const char *rsc_id; /*! The action performed, start, stop, monitor... */ const char *op_type; /*! The userdata string given do exec() api function */ const char *user_data; /*! The client api call id associated with this event */ int call_id; /*! The operation's timeout period in ms. */ int timeout; /*! The operation's recurring interval in ms. */ int interval; /*! The operation's start delay value in ms. */ int start_delay; /*! This operation that just completed is on a deleted rsc. */ int rsc_deleted; /*! The executed ra return code */ enum lrmd_exec_rc rc; /*! The lrmd status returned for exec_complete events */ int op_status; /*! stdout from resource agent operation */ const char *output; /*! Timestamp of when op ran */ unsigned int t_run; /*! Timestamp of last rc change */ unsigned int t_rcchange; /*! Time in length op took to execute */ unsigned int exec_time; /*! Time in length spent in queue */ unsigned int queue_time; /* This is a GHashTable containing the * parameters given to the operation */ void *params; } lrmd_event_data_t; lrmd_event_data_t *lrmd_copy_event(lrmd_event_data_t *event); void lrmd_free_event(lrmd_event_data_t *event); typedef struct lrmd_rsc_info_s { char *id; char *type; char *class; char *provider; } lrmd_rsc_info_t; lrmd_rsc_info_t *lrmd_copy_rsc_info(lrmd_rsc_info_t *rsc_info); void lrmd_free_rsc_info(lrmd_rsc_info_t *rsc_info); typedef void (*lrmd_event_callback)(lrmd_event_data_t *event); typedef struct lrmd_list_s { const char *val; struct lrmd_list_s *next; } lrmd_list_t; void lrmd_list_freeall(lrmd_list_t *head); typedef struct lrmd_api_operations_s { /*! * \brief Connect from the lrmd. * * \retval 0, success * \retval negative error code on failure */ int (*connect) (lrmd_t *lrmd, const char *client_name, int *fd); /*! * \brief Disconnect from the lrmd. * * \retval 0, success * \retval negative error code on failure */ int (*disconnect)(lrmd_t *lrmd); /*! * \brief Register a resource with the lrmd. * * \note Synchronous, guaranteed to occur in daemon before function returns. * * \retval 0, success * \retval negative error code on failure */ int (*register_rsc) (lrmd_t *lrmd, const char *rsc_id, const char *class, const char *provider, const char *agent, enum lrmd_call_options options); /*! * \brief Retrieve registration info for a rsc * * \retval info on success * \retval NULL on failure */ lrmd_rsc_info_t *(*get_rsc_info) (lrmd_t *lrmd, const char *rsc_id, enum lrmd_call_options options); /*! * \brief Unregister a resource from the lrmd. * * \note All pending and recurring operations will be cancelled * automatically. * * \note Synchronous, guaranteed to occur in daemon before function returns. * * \retval 0, success * \retval -1, success, but operations are currently executing on the rsc which will * return once they are completed. * \retval negative error code on failure * */ int (*unregister_rsc) (lrmd_t *lrmd, const char *rsc_id, enum lrmd_call_options options); /*! * \brief Sets the callback to receive lrmd events on. */ void (*set_callback) (lrmd_t *lrmd, lrmd_event_callback callback); /*! * \brief Issue a command on a resource * * \note Asynchronous, command is queued in daemon on function return, but * execution of command is not synced. * * \note Operations on individual resources are guaranteed to occur * in the order the client api calls them in. * * \note Operations between different resources are not guaranteed * to occur in any specific order in relation to one another * regardless of what order the client api is called in. * \retval call_id to track async event result on success * \retval negative error code on failure */ int (*exec)(lrmd_t *lrmd, const char *rsc_id, const char *action, const char *userdata, /* userdata string given back in event notification */ int interval, /* ms */ int timeout, /* ms */ int start_delay, /* ms */ enum lrmd_call_options options, lrmd_key_value_t *params); /* ownership of params is given up to api here */ /*! * \brief Cancel a recurring command. * * \note Synchronous, guaranteed to occur in daemon before function returns. * * \note The cancel is completed async from this call. * We can be guaranteed the cancel has completed once * the callback receives an exec_complete event with * the lrmd_op_status signifying that the operation is * cancelled. * \note For each resource, cancel operations and exec operations * are processed in the order they are received. * It is safe to assume that for a single resource, a cancel * will occur in the lrmd before an exec if the client's cancel * api call occurs before the exec api call. * * It is not however safe to assume any operation on one resource will * occur before an operation on another resource regardless of * the order the client api is called in. * * \retval 0, cancel command sent. * \retval negative error code on failure */ int (*cancel)(lrmd_t *lrmd, const char *rsc_id, const char *action, int interval); /*! * \brief Get the metadata documentation for a resource. * * \note Value is returned in output. Output must be freed when set * * \retval lrmd_ok success * \retval negative error code on failure */ int (*get_metadata) (lrmd_t *lrmd, const char *class, const char *provider, const char *agent, char **output, enum lrmd_call_options options); /*! * \brief Retrieve a list of installed resource agents. * * \note if class is not provided, all known agents will be returned * \note list must be freed using lrmd_list_freeall() * * \retval num items in list on success * \retval negative error code on failure */ int (*list_agents)(lrmd_t *lrmd, lrmd_list_t **agents, const char *class, const char *provider); /*! * \brief Retrieve a list of resource agent providers * * \note When the agent is provided, only the agent's provider will be returned * \note When no agent is supplied, all providers will be returned. * \note List must be freed using lrmd_list_freeall() * * \retval num items in list on success * \retval negative error code on failure */ int (*list_ocf_providers)(lrmd_t *lrmd, const char *agent, lrmd_list_t **providers); /*! * \brief Retrieve a list of standards supported by this machine/installation * * \note List must be freed using lrmd_list_freeall() * * \retval num items in list on success * \retval negative error code on failure */ int (*list_standards)(lrmd_t *lrmd, lrmd_list_t **standards); } lrmd_api_operations_t; struct lrmd_s { lrmd_api_operations_t *cmds; void *private; }; static inline const char * lrmd_event_rc2str(enum lrmd_exec_rc rc) { switch(rc) { case PCMK_EXECRA_OK: return "ok"; case PCMK_EXECRA_UNKNOWN_ERROR: return "unknown error"; case PCMK_EXECRA_INVALID_PARAM: return "invalid parameter"; case PCMK_EXECRA_UNIMPLEMENT_FEATURE: return "unimplemented feature"; case PCMK_EXECRA_INSUFFICIENT_PRIV: return "insufficient privileges"; case PCMK_EXECRA_NOT_INSTALLED: return "not installed"; case PCMK_EXECRA_NOT_CONFIGURED: return "not configured"; case PCMK_EXECRA_NOT_RUNNING: return "not running"; case PCMK_EXECRA_RUNNING_MASTER: return "master"; case PCMK_EXECRA_FAILED_MASTER: return "master (failed)"; case PCMK_EXECRA_STATUS_UNKNOWN: return "status: unknown"; default: break; } return ""; } static inline const char * lrmd_event_type2str(enum lrmd_callback_event type) { switch (type) { case lrmd_event_register: return "register"; case lrmd_event_unregister: return "unregister"; case lrmd_event_exec_complete: return "exec_complete"; case lrmd_event_disconnect: return "disconnect"; } return "unknown"; } #endif diff --git a/include/crm_internal.h b/include/crm_internal.h index 664e3e0b7f..929a0b0a52 100644 --- a/include/crm_internal.h +++ b/include/crm_internal.h @@ -1,301 +1,306 @@ /* crm_internal.h */ /* * Copyright (C) 2006 - 2008 * Andrew Beekhof * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef CRM_INTERNAL__H # define CRM_INTERNAL__H # include # include # include # include # include # include # include /* Dynamic loading of libraries */ void *find_library_function(void **handle, const char *lib, const char *fn, int fatal); void *convert_const_pointer(const void *ptr); /* For ACLs */ char *uid2username(uid_t uid); void determine_request_user(char *user, xmlNode * request, const char *field); # if ENABLE_ACL # include static inline gboolean is_privileged(const char *user) { if (user == NULL) { return FALSE; } else if (strcmp(user, CRM_DAEMON_USER) == 0) { return TRUE; } else if (strcmp(user, "root") == 0) { return TRUE; } return FALSE; } # endif /* CLI option processing*/ # ifdef HAVE_GETOPT_H # include # else # define no_argument 0 # define required_argument 1 # endif # define pcmk_option_default 0x00000 # define pcmk_option_hidden 0x00001 # define pcmk_option_paragraph 0x00002 # define pcmk_option_example 0x00004 struct crm_option { /* Fields from 'struct option' in getopt.h */ /* name of long option */ const char *name; /* * one of no_argument, required_argument, and optional_argument: * whether option takes an argument */ int has_arg; /* if not NULL, set *flag to val when option found */ int *flag; /* if flag not NULL, value to set *flag to; else return value */ int val; /* Custom fields */ const char *desc; long flags; }; void crm_set_options(const char *short_options, const char *usage, struct crm_option *long_options, const char *app_desc); int crm_get_option(int argc, char **argv, int *index); int crm_get_option_long(int argc, char **argv, int *index, const char **longname); void crm_help(char cmd, int exit_code); /* Cluster Option Processing */ typedef struct pe_cluster_option_s { const char *name; const char *alt_name; const char *type; const char *values; const char *default_value; gboolean(*is_valid) (const char *); const char *description_short; const char *description_long; } pe_cluster_option; const char *cluster_option(GHashTable * options, gboolean(*validate) (const char *), const char *name, const char *old_name, const char *def_value); const char *get_cluster_pref(GHashTable * options, pe_cluster_option * option_list, int len, const char *name); void config_metadata(const char *name, const char *version, const char *desc_short, const char *desc_long, pe_cluster_option * option_list, int len); void verify_all_options(GHashTable * options, pe_cluster_option * option_list, int len); gboolean check_time(const char *value); gboolean check_timer(const char *value); gboolean check_boolean(const char *value); gboolean check_number(const char *value); /* Shared PE/crmd functionality */ void filter_action_parameters(xmlNode * param_set, const char *version); void filter_reload_parameters(xmlNode * param_set, const char *restart_string); /* Resource operation updates */ xmlNode *create_operation_update(xmlNode * parent, lrmd_event_data_t *event, const char *caller_version, int target_rc, const char *origin, int level); /* char2score */ extern int node_score_red; extern int node_score_green; extern int node_score_yellow; extern int node_score_infinity; /* Assorted convenience functions */ static inline int crm_strlen_zero(const char *s) { return !s || *s == '\0'; } char *add_list_element(char *list, const char *value); char *generate_series_filename(const char *directory, const char *series, int sequence, gboolean bzip); int get_last_sequence(const char *directory, const char *series); void write_last_sequence(const char *directory, const char *series, int sequence, int max); int crm_pid_active(long pid); void crm_make_daemon(const char *name, gboolean daemonize, const char *pidfile); gboolean crm_is_writable(const char *dir, const char *file, const char *user, const char *group, gboolean need_both); char *generate_op_key(const char *rsc_id, const char *op_type, int interval); char *generate_notify_key(const char *rsc_id, const char *notify_type, const char *op_type); char *generate_transition_magic_v202(const char *transition_key, int op_status); char *generate_transition_magic(const char *transition_key, int op_status, int op_rc); char *generate_transition_key(int action, int transition_id, int target_rc, const char *node); static inline long long crm_clear_bit(const char *function, const char *target, long long word, long long bit) { long long rc = (word & ~bit); if(rc == word) { /* Unchanged */ } else if (target) { crm_trace("Bit 0x%.8llx for %s cleared by %s", bit, target, function); } else { crm_trace("Bit 0x%.8llx cleared by %s", bit, function); } return rc; } static inline long long crm_set_bit(const char *function, const char *target, long long word, long long bit) { long long rc = (word|bit); if(rc == word) { /* Unchanged */ } else if (target) { crm_trace("Bit 0x%.8llx for %s set by %s", bit, target, function); } else { crm_trace("Bit 0x%.8llx set by %s", bit, function); } return rc; } # define set_bit(word, bit) word = crm_set_bit(__PRETTY_FUNCTION__, NULL, word, bit) # define clear_bit(word, bit) word = crm_clear_bit(__PRETTY_FUNCTION__, NULL, word, bit) void g_hash_destroy_str(gpointer data); long long crm_int_helper(const char *text, char **end_text); char *crm_concat(const char *prefix, const char *suffix, char join); char *generate_hash_key(const char *crm_msg_reference, const char *sys); /*! remote tcp/tls helper functions */ gboolean crm_recv_remote_msg(void *session, char **recv_buf, gboolean encrypted, int total_timeout_ms, int *disconnected); char *crm_recv_remote_raw(void *data, gboolean encrypted, size_t max_recv, size_t *recv_len, int *disconnected); int crm_send_remote_msg(void *session, xmlNode * msg, gboolean encrypted); int crm_recv_remote_ready(void *session, gboolean encrypted, int timeout_ms); xmlNode *crm_parse_remote_buffer(char **msg_buf); int crm_remote_tcp_connect(const char *host, int port); #ifdef HAVE_GNUTLS_GNUTLS_H /*! * \internal * \brief Initiate the client handshake after establishing the tcp socket. * \note This is a blocking function, it will block until the entire handshake * is complete or until the timeout period is reached. * \retval 0 success * \retval negative, failure */ int crm_initiate_client_tls_handshake(void *session_data, int timeout_ms); /*! * \internal * \brief Create client or server session for anon DH encryption credentials * \param sock, the socket the session will use for transport * \param type, GNUTLS_SERVER or GNUTLS_CLIENT * \param credentials, gnutls_anon_server_credentials_t or gnutls_anon_client_credentials_t * * \retval gnutls_session * on success * \retval NULL on failure */ void *crm_create_anon_tls_session(int sock, int type, void *credentials); /*! * \internal * \brief Create client or server session for PSK credentials * \param sock, the socket the session will use for transport * \param type, GNUTLS_SERVER or GNUTLS_CLIENT * \param credentials, gnutls_psk_server_credentials_t or gnutls_osk_client_credentials_t * * \retval gnutls_session * on success * \retval NULL on failure */ void *create_psk_tls_session(int csock, int type, void *credentials); #endif #define REMOTE_MSG_TERMINATOR "\r\n\r\n" const char *daemon_option(const char *option); void set_daemon_option(const char *option, const char *value); gboolean daemon_option_enabled(const char *daemon, const char *option); void strip_text_nodes(xmlNode *xml); +enum lrmd_client_type { + LRMD_CLIENT_IPC, + LRMD_CLIENT_TLS, +}; + # define crm_config_err(fmt...) { crm_config_error = TRUE; crm_err(fmt); } # define crm_config_warn(fmt...) { crm_config_warning = TRUE; crm_warn(fmt); } # define attrd_channel T_ATTRD # define F_ATTRD_KEY "attr_key" # define F_ATTRD_ATTRIBUTE "attr_name" # define F_ATTRD_TASK "task" # define F_ATTRD_VALUE "attr_value" # define F_ATTRD_SET "attr_set" # define F_ATTRD_SECTION "attr_section" # define F_ATTRD_DAMPEN "attr_dampening" # define F_ATTRD_IGNORE_LOCALLY "attr_ignore_locally" # define F_ATTRD_HOST "attr_host" # define F_ATTRD_USER "attr_user" # if SUPPORT_COROSYNC # if CS_USES_LIBQB # include # include typedef struct qb_ipc_request_header cs_ipc_header_request_t; typedef struct qb_ipc_response_header cs_ipc_header_response_t; # else # include # include # include static inline int qb_to_cs_error(int a) { return a; } typedef coroipc_request_header_t cs_ipc_header_request_t; typedef coroipc_response_header_t cs_ipc_header_response_t; # endif # else typedef struct { int size __attribute__ ((aligned(8))); int id __attribute__ ((aligned(8))); } __attribute__ ((aligned(8))) cs_ipc_header_request_t; typedef struct { int size __attribute__ ((aligned(8))); int id __attribute__ ((aligned(8))); int error __attribute__ ((aligned(8))); } __attribute__ ((aligned(8))) cs_ipc_header_response_t; # endif #endif /* CRM_INTERNAL__H */ diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c index 2d9bc33dd1..c2c6694291 100644 --- a/lib/lrmd/lrmd_client.c +++ b/lib/lrmd/lrmd_client.c @@ -1,998 +1,1636 @@ /* * Copyright (c) 2012 David Vossel * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include + +#ifdef HAVE_GNUTLS_GNUTLS_H +# undef KEYFILE +# include +#endif + +#include +#include +#include +#include + CRM_TRACE_INIT_DATA(lrmd); static stonith_t *stonith_api = NULL; +static int lrmd_api_disconnect(lrmd_t *lrmd); +static int lrmd_connected(lrmd_t *lrmd); + +#ifdef HAVE_GNUTLS_GNUTLS_H + +#define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000 /* 5 seconds */ +gnutls_psk_client_credentials_t psk_cred_s; +int lrmd_tls_set_key(gnutls_datum_t *key, const char *location); +static void lrmd_tls_disconnect(lrmd_t *lrmd); +static int global_remote_msg_id = 0; +int lrmd_tls_send_msg(gnutls_session *session, xmlNode *msg, uint32_t id, const char *msg_type); +static void lrmd_tls_connection_destroy(gpointer userdata); +#endif + typedef struct lrmd_private_s { + enum lrmd_client_type type; char *token; - crm_ipc_t *ipc; mainloop_io_t *source; + /* IPC parameters */ + crm_ipc_t *ipc; + + /* TLS parameters */ +#ifdef HAVE_GNUTLS_GNUTLS_H + char *server; + int port; + gnutls_psk_client_credentials_t psk_cred_c; + gnutls_session *session; + int sock; + + char *recv_buf; + + GList *pending_notify; + crm_trigger_t *process_notify; +#endif + lrmd_event_callback callback; } lrmd_private_t; static lrmd_list_t * lrmd_list_add(lrmd_list_t * head, const char *value) { lrmd_list_t *p, *end; p = calloc(1, sizeof(lrmd_list_t)); p->val = strdup(value); end = head; while (end && end->next) { end = end->next; } if (end) { end->next = p; } else { head = p; } return head; } void lrmd_list_freeall(lrmd_list_t * head) { lrmd_list_t *p; while (head) { char *val = (char *)head->val; p = head->next; free(val); free(head); head = p; } } lrmd_key_value_t * lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value) { lrmd_key_value_t *p, *end; p = calloc(1, sizeof(lrmd_key_value_t)); p->key = strdup(key); p->value = strdup(value); end = head; while (end && end->next) { end = end->next; } if (end) { end->next = p; } else { head = p; } return head; } static void lrmd_key_value_freeall(lrmd_key_value_t * head) { lrmd_key_value_t *p; while (head) { p = head->next; free(head->key); free(head->value); free(head); head = p; } } static void dup_attr(gpointer key, gpointer value, gpointer user_data) { g_hash_table_replace(user_data, strdup(key), strdup(value)); } lrmd_event_data_t * lrmd_copy_event(lrmd_event_data_t * event) { lrmd_event_data_t *copy = NULL; copy = calloc(1, sizeof(lrmd_event_data_t)); /* This will get all the int values. * we just have to be careful not to leave any * dangling pointers to strings. */ memcpy(copy, event, sizeof(lrmd_event_data_t)); copy->rsc_id = event->rsc_id ? strdup(event->rsc_id) : NULL; copy->op_type = event->op_type ? strdup(event->op_type) : NULL; copy->user_data = event->user_data ? strdup(event->user_data) : NULL; copy->output = event->output ? strdup(event->output) : NULL; if (event->params) { copy->params = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); if (copy->params != NULL) { g_hash_table_foreach(event->params, dup_attr, copy->params); } } return copy; } void lrmd_free_event(lrmd_event_data_t * event) { if (!event) { return; } /* free gives me grief if i try to cast */ free((char *)event->rsc_id); free((char *)event->op_type); free((char *)event->user_data); free((char *)event->output); if (event->params) { g_hash_table_destroy(event->params); } free(event); } static int -lrmd_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata) +lrmd_dispatch_internal(lrmd_t *lrmd, xmlNode *msg) { const char *type; - lrmd_t *lrmd = userdata; lrmd_private_t *native = lrmd->private; lrmd_event_data_t event = { 0, }; - xmlNode *msg; if (!native->callback) { /* no callback set */ + crm_trace("notify event received but client has not set callback"); return 1; } - msg = string2xml(buffer); type = crm_element_value(msg, F_LRMD_OPERATION); crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id); event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID); if (crm_str_eq(type, LRMD_OP_RSC_REG, TRUE)) { event.type = lrmd_event_register; } else if (crm_str_eq(type, LRMD_OP_RSC_UNREG, TRUE)) { event.type = lrmd_event_unregister; } else if (crm_str_eq(type, LRMD_OP_RSC_EXEC, TRUE)) { crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout); crm_element_value_int(msg, F_LRMD_RSC_INTERVAL, &event.interval); crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay); crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc); crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status); crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted); crm_element_value_int(msg, F_LRMD_RSC_RUN_TIME, (int *)&event.t_run); crm_element_value_int(msg, F_LRMD_RSC_RCCHANGE_TIME, (int *)&event.t_rcchange); crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time); crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time); event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION); event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR); event.output = crm_element_value(msg, F_LRMD_RSC_OUTPUT); event.type = lrmd_event_exec_complete; event.params = xml2list(msg); + } else { + return 1; } + crm_trace("op %s notify event received", type); native->callback(&event); if (event.params) { g_hash_table_destroy(event.params); } + return 1; +} + +static int +lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata) +{ + lrmd_t *lrmd = userdata; + lrmd_private_t *native = lrmd->private; + xmlNode *msg; + int rc; + + if (!native->callback) { + /* no callback set */ + return 1; + } + + msg = string2xml(buffer); + rc = lrmd_dispatch_internal(lrmd, msg); free_xml(msg); + return rc; +} + +#ifdef HAVE_GNUTLS_GNUTLS_H +static void +lrmd_free_xml(gpointer userdata) +{ + free_xml((xmlNode *) userdata); +} + +static int +lrmd_tls_connected(lrmd_t *lrmd) +{ + lrmd_private_t *native = lrmd->private; + if (native->session) { + return TRUE; + } + + return FALSE; +} + +static int +lrmd_tls_dispatch(gpointer userdata) +{ + lrmd_t *lrmd = userdata; + lrmd_private_t *native = lrmd->private; + xmlNode *xml = NULL; + int rc = 0; + int disconnected = 0; + + if (lrmd_tls_connected(lrmd) == FALSE) { + crm_trace("tls dispatch triggered after disconnect"); + return 0; + } + + crm_trace("tls_dispatch triggered"); + + /* First check if there are any pending notifies to process that came + * while we were waiting for replies earlier. */ + if (native->pending_notify) { + GList *iter = NULL; + crm_trace("Processing pending notifies"); + for (iter = native->pending_notify; iter; iter = iter->next) { + lrmd_dispatch_internal(lrmd, iter->data); + } + g_list_free_full(native->pending_notify, lrmd_free_xml); + native->pending_notify = NULL; + } + + /* Next read the current buffer and see if there are any messages to handle. */ + rc = crm_recv_remote_ready(native->session, TRUE, 0); + if (rc == 0) { + /* nothing to read, see if any full messages are already in buffer. */ + xml = crm_parse_remote_buffer(&native->recv_buf); + } else if (rc < 0) { + disconnected = 1; + } else { + crm_recv_remote_msg(native->session, &native->recv_buf, TRUE, -1, &disconnected); + xml = crm_parse_remote_buffer(&native->recv_buf); + } + while (xml) { + lrmd_dispatch_internal(lrmd, xml); + free_xml(xml); + xml = crm_parse_remote_buffer(&native->recv_buf); + } + + if (disconnected) { + crm_info("Server disconnected while reading remote server msg."); + lrmd_tls_disconnect(lrmd); + return 0; + } return 1; + } +#endif + +/* Not used with mainloop */ +int lrmd_poll(lrmd_t *lrmd, int timeout) +{ + int fd = -1; + lrmd_private_t *native = lrmd->private; + + switch (native->type) { + case LRMD_CLIENT_IPC: + fd = crm_ipc_get_fd(native->ipc); + break; + case LRMD_CLIENT_TLS: +#ifdef HAVE_GNUTLS_GNUTLS_H + if (native->pending_notify) { + return 1; + } else if (native->recv_buf && strstr(native->recv_buf, REMOTE_MSG_TERMINATOR)) { + return 1; + } + + fd = native->sock; +#endif + break; + } + + return crm_recv_remote_ready(GUINT_TO_POINTER(fd), FALSE, 0); } /* Not used with mainloop */ bool lrmd_dispatch(lrmd_t * lrmd) { - gboolean stay_connected = TRUE; lrmd_private_t *private = NULL; CRM_ASSERT(lrmd != NULL); + private = lrmd->private; - while (crm_ipc_ready(private->ipc)) { - if (crm_ipc_read(private->ipc) > 0) { - const char *msg = crm_ipc_buffer(private->ipc); + switch (private->type) { + case LRMD_CLIENT_IPC: + while (crm_ipc_ready(private->ipc)) { + if (crm_ipc_read(private->ipc) > 0) { + const char *msg = crm_ipc_buffer(private->ipc); + lrmd_ipc_dispatch(msg, strlen(msg), lrmd); + } + } + break; + case LRMD_CLIENT_TLS: +#ifdef HAVE_GNUTLS_GNUTLS_H + lrmd_tls_dispatch(lrmd); +#endif + break; + } - lrmd_dispatch_internal(msg, strlen(msg), lrmd); - } - if (crm_ipc_connected(private->ipc) == FALSE) { - crm_err("Connection closed"); - stay_connected = FALSE; - } + if (lrmd_connected(lrmd) == FALSE) { + crm_err("Connection closed"); + return FALSE; } - return stay_connected; + return TRUE; } static xmlNode * lrmd_create_op(const char *token, const char *op, xmlNode * data, enum lrmd_call_options options) { xmlNode *op_msg = create_xml_node(NULL, "lrmd_command"); CRM_CHECK(op_msg != NULL, return NULL); CRM_CHECK(token != NULL, return NULL); crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command"); crm_xml_add(op_msg, F_TYPE, T_LRMD); crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token); crm_xml_add(op_msg, F_LRMD_OPERATION, op); crm_trace("Sending call options: %.8lx, %d", (long)options, options); crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options); if (data != NULL) { add_message_xml(op_msg, F_LRMD_CALLDATA, data); } return op_msg; } static void -lrmd_connection_destroy(gpointer userdata) +lrmd_ipc_connection_destroy(gpointer userdata) { lrmd_t *lrmd = userdata; lrmd_private_t *native = lrmd->private; - crm_info("connection destroyed"); + crm_info("IPC connection destroyed"); /* Prevent these from being cleaned up in lrmd_api_disconnect() */ native->ipc = NULL; native->source = NULL; if (native->callback) { lrmd_event_data_t event = { 0, }; event.type = lrmd_event_disconnect; native->callback(&event); } } +#ifdef HAVE_GNUTLS_GNUTLS_H +static void +lrmd_tls_connection_destroy(gpointer userdata) +{ + lrmd_t *lrmd = userdata; + lrmd_private_t *native = lrmd->private; + + crm_info("TLS connection destroyed"); + + if (native->session) { + gnutls_bye(*native->session, GNUTLS_SHUT_RDWR); + gnutls_deinit(*native->session); + gnutls_free(native->session); + } + if (native->psk_cred_c) { + gnutls_psk_free_client_credentials(native->psk_cred_c); + } + if (native->sock) { + close(native->sock); + } + if (native->process_notify) { + mainloop_destroy_trigger(native->process_notify); + native->process_notify = NULL; + } + if (native->pending_notify) { + g_list_free_full(native->pending_notify, lrmd_free_xml); + native->pending_notify = NULL; + } + + free(native->recv_buf); + native->recv_buf = NULL; + native->source = 0; + native->sock = 0; + native->psk_cred_c = NULL; + native->session = NULL; + native->sock = 0; + + if (native->callback) { + lrmd_event_data_t event = { 0, }; + event.type = lrmd_event_disconnect; + native->callback(&event); + } + return; +} + +int +lrmd_tls_send_msg(gnutls_session *session, xmlNode *msg, uint32_t id, const char *msg_type) +{ + int rc = -1; + + crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id); + crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type); + + rc = crm_send_remote_msg(session, msg, TRUE); + + if (rc < 0) { + crm_err("Failed to send remote lrmd tls msg, rc = %d" , rc); + return rc; + } + + return rc; +} + +static xmlNode * +lrmd_tls_recv_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id, int *disconnected) +{ + lrmd_private_t *native = lrmd->private; + xmlNode *xml = NULL; + time_t start = time(NULL); + const char *msg_type = NULL; + int reply_id = 0; + int remaining_timeout = 0; + + if (total_timeout == 0) { + total_timeout = 10000; + } else if (total_timeout == -1) { + total_timeout = 30000; + } + + while (!xml) { + + xml = crm_parse_remote_buffer(&native->recv_buf); + if (!xml) { + /* read some more off the tls buffer if we still have time left. */ + if (remaining_timeout) { + remaining_timeout = remaining_timeout - ((time(NULL) - start) * 1000); + } else { + remaining_timeout = total_timeout; + } + if (remaining_timeout <= 0) { + return NULL; + } + + crm_recv_remote_msg(native->session, &native->recv_buf, TRUE, remaining_timeout, disconnected); + xml = crm_parse_remote_buffer(&native->recv_buf); + if (!xml || *disconnected) { + return NULL; + } + } + + CRM_ASSERT(xml != NULL); + + crm_element_value_int(xml, F_LRMD_REMOTE_MSG_ID, &reply_id); + msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE); + + if (!msg_type) { + crm_err("Empty msg type received while waiting for reply"); + free_xml(xml); + xml = NULL; + } else if (safe_str_eq(msg_type, "notify")) { + /* got a notify while waiting for reply, trigger the notify to be processed later */ + crm_info("queueing notify"); + native->pending_notify = g_list_append(native->pending_notify, xml); + if (native->process_notify) { + crm_info("notify trigger set."); + mainloop_set_trigger(native->process_notify); + } + xml = NULL; + } else if (safe_str_neq(msg_type, "reply")) { + /* msg isn't a reply, make some noise */ + crm_err("Expected a reply, got %s", msg_type); + free_xml(xml); + xml = NULL; + } else if (reply_id != expected_reply_id) { + crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id); + free_xml(xml); + xml = NULL; + } + } + + if (native->recv_buf && native->process_notify) { + mainloop_set_trigger(native->process_notify); + } + + return xml; +} + +static int +lrmd_tls_send_recv(lrmd_t *lrmd, xmlNode *msg, int timeout, xmlNode **reply) +{ + int rc = 0; + int disconnected = 0; + xmlNode *xml = NULL; + lrmd_private_t *native = lrmd->private; + + if (lrmd_tls_connected(lrmd) == FALSE) { + return -1; + } + + global_remote_msg_id++; + if (global_remote_msg_id <= 0) { + global_remote_msg_id = 1; + } + + rc = lrmd_tls_send_msg(native->session, msg, global_remote_msg_id, "request"); + if (rc <= 0) { + crm_err("Remote lrmd send failed, disconnecting"); + lrmd_tls_disconnect(lrmd); + return -ENOTCONN; + } + + xml = lrmd_tls_recv_reply(lrmd, timeout, global_remote_msg_id, &disconnected); + + if (disconnected) { + crm_err("Remote lrmd server disconnected while waiting for reply with id %d. ", global_remote_msg_id); + lrmd_tls_disconnect(lrmd); + rc = -ENOTCONN; + } else if (!xml) { + crm_err("Remote lrmd never received reply for request id %d. timeout: %dms ", global_remote_msg_id, timeout); + rc = -ECOMM; + } + + if (reply) { + *reply = xml; + } else { + free_xml(xml); + } + + return rc; +} +#endif + +static int +lrmd_send_xml(lrmd_t *lrmd, xmlNode *msg, int timeout, xmlNode **reply) +{ + int rc = -1; + lrmd_private_t *native = lrmd->private; + + switch (native->type) { + case LRMD_CLIENT_IPC: + rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply); + break; + case LRMD_CLIENT_TLS: +#ifdef HAVE_GNUTLS_GNUTLS_H + rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply); +#endif + break; + } + + return rc; +} + +static int +lrmd_connected(lrmd_t *lrmd) +{ + lrmd_private_t *native = lrmd->private; + + switch (native->type) { + case LRMD_CLIENT_IPC: + return crm_ipc_connected(native->ipc); + break; + case LRMD_CLIENT_TLS: +#ifdef HAVE_GNUTLS_GNUTLS_H + return lrmd_tls_connected(lrmd); +#endif + break; + } + + return 0; +} + static int lrmd_send_command(lrmd_t * lrmd, const char *op, xmlNode * data, xmlNode ** output_data, int timeout, /* ms. defaults to 1000 if set to 0 */ enum lrmd_call_options options) { int rc = pcmk_ok; int reply_id = -1; lrmd_private_t *native = lrmd->private; xmlNode *op_msg = NULL; xmlNode *op_reply = NULL; - if (!native->ipc) { + if (!lrmd_connected(lrmd)) { return -ENOTCONN; } if (op == NULL) { crm_err("No operation specified"); return -EINVAL; } CRM_CHECK(native->token != NULL,;); + crm_trace("sending %s op to lrmd", op); op_msg = lrmd_create_op(native->token, op, data, options); if (op_msg == NULL) { return -EINVAL; } crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout); - rc = crm_ipc_send(native->ipc, op_msg, crm_ipc_client_response, timeout, &op_reply); + rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply); free_xml(op_msg); if (rc < 0) { crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc); rc = -ECOMM; goto done; } rc = pcmk_ok; crm_element_value_int(op_reply, F_LRMD_CALLID, &reply_id); - crm_trace("reply received"); + crm_trace("%s op reply received", op); if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) { rc = -ENOMSG; goto done; } + crm_log_xml_trace(op_reply, "Reply"); + if (output_data) { *output_data = op_reply; op_reply = NULL; /* Prevent subsequent free */ } - crm_log_xml_trace(op_reply, "Reply"); - done: - if (crm_ipc_connected(native->ipc) == FALSE) { + if (lrmd_connected(lrmd) == FALSE) { crm_err("LRMD disconnected"); } free_xml(op_reply); return rc; } static int -lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd) +lrmd_handshake(lrmd_t *lrmd, const char *name) +{ + int rc = pcmk_ok; + lrmd_private_t *native = lrmd->private; + xmlNode *reply = NULL; + xmlNode *hello = create_xml_node(NULL, "lrmd_command"); + + crm_xml_add(hello, F_TYPE, T_LRMD); + crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER); + crm_xml_add(hello, F_LRMD_CLIENTNAME, name); + + rc = lrmd_send_xml(lrmd, hello, -1, &reply); + + if (rc < 0) { + crm_perror(LOG_DEBUG, "Couldn't complete registration with the lrmd API: %d", rc); + rc = -ECOMM; + } else if (reply == NULL) { + crm_err("Did not receive registration reply"); + rc = -EPROTO; + } else { + const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION); + const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID); + + if (safe_str_neq(msg_type, CRM_OP_REGISTER)) { + crm_err("Invalid registration message: %s", msg_type); + crm_log_xml_err(reply, "Bad reply"); + rc = -EPROTO; + } else if (tmp_ticket == NULL) { + crm_err("No registration token provided"); + crm_log_xml_err(reply, "Bad reply"); + rc = -EPROTO; + } else { + crm_trace("Obtained registration token: %s", tmp_ticket); + native->token = strdup(tmp_ticket); + rc = pcmk_ok; + } + } + + free_xml(reply); + free_xml(hello); + + return rc; +} + +static int +lrmd_ipc_connect(lrmd_t * lrmd, int *fd) { int rc = pcmk_ok; lrmd_private_t *native = lrmd->private; static struct ipc_client_callbacks lrmd_callbacks = { - .dispatch = lrmd_dispatch_internal, - .destroy = lrmd_connection_destroy + .dispatch = lrmd_ipc_dispatch, + .destroy = lrmd_ipc_connection_destroy }; crm_info("Connecting to lrmd"); if (fd) { /* No mainloop */ native->ipc = crm_ipc_new("lrmd", 0); if (native->ipc && crm_ipc_connect(native->ipc)) { *fd = crm_ipc_get_fd(native->ipc); } else if (native->ipc) { rc = -ENOTCONN; } } else { native->source = mainloop_add_ipc_client("lrmd", G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks); native->ipc = mainloop_get_ipc_client(native->source); } if (native->ipc == NULL) { crm_debug("Could not connect to the LRMD API"); rc = -ENOTCONN; } - if (!rc) { - xmlNode *reply = NULL; - xmlNode *hello = create_xml_node(NULL, "lrmd_command"); + return rc; +} - crm_xml_add(hello, F_TYPE, T_LRMD); - crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER); - crm_xml_add(hello, F_LRMD_CLIENTNAME, name); +#ifdef HAVE_GNUTLS_GNUTLS_H +int lrmd_tls_set_key(gnutls_datum_t *key, const char *location) +{ + FILE *stream; + int read_len = 256; + int cur_len = 0; + int buf_len = read_len; + static char *key_cache = NULL; + static size_t key_cache_len = 0; + static time_t key_cache_updated; + + if (key_cache) { + time_t now = time(NULL); + + if ((now - key_cache_updated) < 60) { + key->data = gnutls_malloc(key_cache_len + 1); + key->size = key_cache_len; + memcpy(key->data, key_cache, key_cache_len); + + crm_debug("using cached LRMD key"); + return 0; + } else { + key_cache_len = 0; + key_cache_updated = 0; + free(key_cache); + key_cache = NULL; + crm_debug("clearing lrmd key cache"); + } + } - rc = crm_ipc_send(native->ipc, hello, crm_ipc_client_response, -1, &reply); + stream = fopen(location, "r"); + if (!stream) { + return -1; + } - if (rc < 0) { - crm_perror(LOG_DEBUG, "Couldn't complete registration with the lrmd API: %d", rc); - rc = -ECOMM; - } else if (reply == NULL) { - crm_err("Did not receive registration reply"); - rc = -EPROTO; - } else { - const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION); - const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID); - - if (safe_str_neq(msg_type, CRM_OP_REGISTER)) { - crm_err("Invalid registration message: %s", msg_type); - crm_log_xml_err(reply, "Bad reply"); - rc = -EPROTO; - } else if (tmp_ticket == NULL) { - crm_err("No registration token provided"); - crm_log_xml_err(reply, "Bad reply"); - rc = -EPROTO; - } else { - crm_trace("Obtained registration token: %s", tmp_ticket); - native->token = strdup(tmp_ticket); - rc = pcmk_ok; - } + key->data = gnutls_malloc(read_len); + while (!feof(stream)) { + char next; + if (cur_len == buf_len) { + buf_len = cur_len + read_len; + key->data = gnutls_realloc(key->data, buf_len); + } + next = fgetc(stream); + if (next == EOF && feof(stream)) { + break; } - free_xml(reply); - free_xml(hello); + key->data[cur_len] = next; + cur_len++; + } + fclose(stream); + + key->size = cur_len; + if (!cur_len) { + gnutls_free(key->data); + key->data = 0; + return -1; } + if (!key_cache) { + key_cache = calloc(1, key->size+1); + memcpy(key_cache, key->data, key->size); + + key_cache_len = key->size; + key_cache_updated = time(NULL); + } + + return 0; +} + +static int +lrmd_tls_key_cb(gnutls_session_t session, char **username, gnutls_datum_t *key) +{ + int rc = 0; + + if (lrmd_tls_set_key(key, DEFAULT_REMOTE_KEY_LOCATION)) { + rc = lrmd_tls_set_key(key, ALT_REMOTE_KEY_LOCATION); + } + if (rc) { + crm_err("No lrmd remote key found"); + return -1; + } + + *username = gnutls_malloc(strlen(DEFAULT_REMOTE_USERNAME) + 1); + strcpy(*username, DEFAULT_REMOTE_USERNAME); + return rc; } +#endif static int -lrmd_api_disconnect(lrmd_t * lrmd) +lrmd_tls_connect(lrmd_t *lrmd, int *fd) { +#ifdef HAVE_GNUTLS_GNUTLS_H + static struct mainloop_fd_callbacks lrmd_tls_callbacks = + { + .dispatch = lrmd_tls_dispatch, + .destroy = lrmd_tls_connection_destroy, + }; + lrmd_private_t *native = lrmd->private; + static int gnutls_init = 0; + int sock; - crm_info("Disconnecting from lrmd service"); + if (!gnutls_init) { + gnutls_global_init(); + } + + gnutls_psk_allocate_client_credentials(&native->psk_cred_c); + gnutls_psk_set_client_credentials_function(native->psk_cred_c, lrmd_tls_key_cb); + sock = crm_remote_tcp_connect(native->server, native->port); + if (sock <= 0) { + crm_warn("Could not establish remote lrmd connection to %s", native->server); + lrmd_tls_connection_destroy(lrmd); + return -ENOTCONN; + } + + native->sock = sock; + native->session = create_psk_tls_session(sock, GNUTLS_CLIENT, native->psk_cred_c); + + if (crm_initiate_client_tls_handshake(native->session, LRMD_CLIENT_HANDSHAKE_TIMEOUT) != 0) { + crm_err("Session creation for %s:%d failed", native->server, native->port); + gnutls_deinit(*native->session); + gnutls_free(native->session); + native->session = NULL; + lrmd_tls_connection_destroy(lrmd); + return -1; + } + + crm_info("Remote lrmd client TLS connection established with server %s:%d", native->server, native->port); + + if (fd) { + *fd = sock; + } else { + char name[256] = { 0, }; + snprintf(name, 128, "remote-lrmd-%s:%d", native->server, native->port); + + native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_tls_dispatch, lrmd); + native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd, &lrmd_tls_callbacks); + } + return pcmk_ok; +#else + crm_err("TLS not enabled for this build."); + return -ENOTCONN; +#endif +} + +static int +lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd) +{ + int rc = -ENOTCONN; + lrmd_private_t *native = lrmd->private; + + switch (native->type) { + case LRMD_CLIENT_IPC: + rc = lrmd_ipc_connect(lrmd, fd); + break; + case LRMD_CLIENT_TLS: + rc = lrmd_tls_connect(lrmd, fd); + break; + } + + if (rc == pcmk_ok) { + rc = lrmd_handshake(lrmd, name); + } + + if (rc != pcmk_ok) { + lrmd_api_disconnect(lrmd); + } + + return rc; +} +static void +lrmd_ipc_disconnect(lrmd_t *lrmd) +{ + lrmd_private_t *native = lrmd->private; if (native->source != NULL) { /* Attached to mainloop */ mainloop_del_ipc_client(native->source); native->source = NULL; native->ipc = NULL; } else if(native->ipc) { /* Not attached to mainloop */ crm_ipc_t *ipc = native->ipc; native->ipc = NULL; crm_ipc_close(ipc); crm_ipc_destroy(ipc); } +} + +#ifdef HAVE_GNUTLS_GNUTLS_H +static void +lrmd_tls_disconnect(lrmd_t *lrmd) +{ + lrmd_private_t *native = lrmd->private; + + if (native->session) { + gnutls_bye(*native->session, GNUTLS_SHUT_RDWR); + gnutls_deinit(*native->session); + gnutls_free(native->session); + native->session = 0; + } + + if (native->source != NULL) { + /* Attached to mainloop */ + mainloop_del_ipc_client(native->source); + native->source = NULL; + + } else if(native->sock) { + close(native->sock); + } + + if (native->pending_notify) { + g_list_free_full(native->pending_notify, lrmd_free_xml); + native->pending_notify = NULL; + } +} +#endif + +static int +lrmd_api_disconnect(lrmd_t *lrmd) +{ + lrmd_private_t *native = lrmd->private; + + crm_info("Disconnecting from lrmd service"); + switch (native->type) { + case LRMD_CLIENT_IPC: + lrmd_ipc_disconnect(lrmd); + break; + case LRMD_CLIENT_TLS: +#ifdef HAVE_GNUTLS_GNUTLS_H + lrmd_tls_disconnect(lrmd); +#endif + break; + } free(native->token); native->token = NULL; return 0; } static int lrmd_api_register_rsc(lrmd_t * lrmd, const char *rsc_id, const char *class, const char *provider, const char *type, enum lrmd_call_options options) { int rc = pcmk_ok; xmlNode *data = NULL; if (!class || !type || !rsc_id) { return -EINVAL; } if (safe_str_eq(class, "ocf") && !provider) { return -EINVAL; } data = create_xml_node(NULL, F_LRMD_RSC); crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__); crm_xml_add(data, F_LRMD_RSC_ID, rsc_id); crm_xml_add(data, F_LRMD_CLASS, class); crm_xml_add(data, F_LRMD_PROVIDER, provider); crm_xml_add(data, F_LRMD_TYPE, type); rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options); free_xml(data); return rc; } static int lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options) { int rc = pcmk_ok; xmlNode *data = create_xml_node(NULL, F_LRMD_RSC); crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__); crm_xml_add(data, F_LRMD_RSC_ID, rsc_id); rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options); free_xml(data); return rc; } lrmd_rsc_info_t * lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info) { lrmd_rsc_info_t *copy = NULL; copy = calloc(1, sizeof(lrmd_rsc_info_t)); copy->id = strdup(rsc_info->id); copy->type = strdup(rsc_info->type); copy->class = strdup(rsc_info->class); if (rsc_info->provider) { copy->provider = strdup(rsc_info->provider); } return copy; } void lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info) { if (!rsc_info) { return; } free(rsc_info->id); free(rsc_info->type); free(rsc_info->class); free(rsc_info->provider); free(rsc_info); } static lrmd_rsc_info_t * lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options) { lrmd_rsc_info_t *rsc_info = NULL; xmlNode *data = create_xml_node(NULL, F_LRMD_RSC); xmlNode *output = NULL; const char *class = NULL; const char *provider = NULL; const char *type = NULL; crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__); crm_xml_add(data, F_LRMD_RSC_ID, rsc_id); lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options); free_xml(data); if (!output) { return NULL; } class = crm_element_value(output, F_LRMD_CLASS); provider = crm_element_value(output, F_LRMD_PROVIDER); type = crm_element_value(output, F_LRMD_TYPE); if (!class || !type) { free_xml(output); return NULL; } else if (safe_str_eq(class, "ocf") && !provider) { free_xml(output); return NULL; } rsc_info = calloc(1, sizeof(lrmd_rsc_info_t)); rsc_info->id = strdup(rsc_id); rsc_info->class = strdup(class); if (provider) { rsc_info->provider = strdup(provider); } rsc_info->type = strdup(type); free_xml(output); return rsc_info; } static void lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback) { lrmd_private_t *native = lrmd->private; native->callback = callback; } static int stonith_get_metadata(const char *provider, const char *type, char **output) { int rc = pcmk_ok; stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type, provider, output, 0); if (*output == NULL) { rc = -EIO; } return rc; } static int lsb_get_metadata(const char *type, char **output) { #define lsb_metadata_template \ "\n"\ "\n"\ "\n"\ " 1.0\n"\ " \n"\ " %s"\ " \n"\ " %s\n"\ " \n"\ " \n"\ " \n"\ " \n"\ " \n"\ " \n"\ " \n"\ " \n"\ " \n"\ " \n"\ " \n"\ " \n"\ " %s\n"\ " %s\n"\ " %s\n"\ " %s\n"\ " %s\n"\ " %s\n"\ " %s\n"\ " \n"\ "\n" #define LSB_INITSCRIPT_INFOBEGIN_TAG "### BEGIN INIT INFO" #define LSB_INITSCRIPT_INFOEND_TAG "### END INIT INFO" #define PROVIDES "# Provides:" #define REQ_START "# Required-Start:" #define REQ_STOP "# Required-Stop:" #define SHLD_START "# Should-Start:" #define SHLD_STOP "# Should-Stop:" #define DFLT_START "# Default-Start:" #define DFLT_STOP "# Default-Stop:" #define SHORT_DSCR "# Short-Description:" #define DESCRIPTION "# Description:" #define lsb_meta_helper_free_value(m) \ if ((m) != NULL) { \ xmlFree(m); \ (m) = NULL; \ } #define lsb_meta_helper_get_value(buffer, ptr, keyword) \ if (!ptr && !strncasecmp(buffer, keyword, strlen(keyword))) { \ (ptr) = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST buffer+strlen(keyword)); \ continue; \ } char ra_pathname[PATH_MAX] = { 0, }; FILE *fp; GString *meta_data = NULL; char buffer[1024]; char *provides = NULL; char *req_start = NULL; char *req_stop = NULL; char *shld_start = NULL; char *shld_stop = NULL; char *dflt_start = NULL; char *dflt_stop = NULL; char *s_dscrpt = NULL; char *xml_l_dscrpt = NULL; GString *l_dscrpt = NULL; snprintf(ra_pathname, sizeof(ra_pathname), "%s%s%s", type[0] == '/' ? "" : LSB_ROOT_DIR, type[0] == '/' ? "" : "/", type); if (!(fp = fopen(ra_pathname, "r"))) { return -EIO; } /* Enter into the lsb-compliant comment block */ while (fgets(buffer, sizeof(buffer), fp)) { /* Now suppose each of the following eight arguments contain only one line */ lsb_meta_helper_get_value(buffer, provides, PROVIDES) lsb_meta_helper_get_value(buffer, req_start, REQ_START) lsb_meta_helper_get_value(buffer, req_stop, REQ_STOP) lsb_meta_helper_get_value(buffer, shld_start, SHLD_START) lsb_meta_helper_get_value(buffer, shld_stop, SHLD_STOP) lsb_meta_helper_get_value(buffer, dflt_start, DFLT_START) lsb_meta_helper_get_value(buffer, dflt_stop, DFLT_STOP) lsb_meta_helper_get_value(buffer, s_dscrpt, SHORT_DSCR) /* Long description may cross multiple lines */ if ((l_dscrpt == NULL) && (0 == strncasecmp(buffer, DESCRIPTION, strlen(DESCRIPTION)))) { l_dscrpt = g_string_new(buffer + strlen(DESCRIPTION)); /* Between # and keyword, more than one space, or a tab character, * indicates the continuation line. Extracted from LSB init script standard */ while (fgets(buffer, sizeof(buffer), fp)) { if (!strncmp(buffer, "# ", 3) || !strncmp(buffer, "#\t", 2)) { buffer[0] = ' '; l_dscrpt = g_string_append(l_dscrpt, buffer); } else { fputs(buffer, fp); break; /* Long description ends */ } } continue; } if (l_dscrpt) { xml_l_dscrpt = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST(l_dscrpt->str)); } if (!strncasecmp(buffer, LSB_INITSCRIPT_INFOEND_TAG, strlen(LSB_INITSCRIPT_INFOEND_TAG))) { /* Get to the out border of LSB comment block */ break; } if (buffer[0] != '#') { break; /* Out of comment block in the beginning */ } } fclose(fp); meta_data = g_string_new(""); g_string_sprintf(meta_data, lsb_metadata_template, type, (xml_l_dscrpt == NULL) ? type : xml_l_dscrpt, (s_dscrpt == NULL) ? type : s_dscrpt, (provides == NULL) ? "" : provides, (req_start == NULL) ? "" : req_start, (req_stop == NULL) ? "" : req_stop, (shld_start == NULL) ? "" : shld_start, (shld_stop == NULL) ? "" : shld_stop, (dflt_start == NULL) ? "" : dflt_start, (dflt_stop == NULL) ? "" : dflt_stop); lsb_meta_helper_free_value(xml_l_dscrpt); lsb_meta_helper_free_value(s_dscrpt); lsb_meta_helper_free_value(provides); lsb_meta_helper_free_value(req_start); lsb_meta_helper_free_value(req_stop); lsb_meta_helper_free_value(shld_start); lsb_meta_helper_free_value(shld_stop); lsb_meta_helper_free_value(dflt_start); lsb_meta_helper_free_value(dflt_stop); if (l_dscrpt) { g_string_free(l_dscrpt, TRUE); } *output = strdup(meta_data->str); g_string_free(meta_data, TRUE); return pcmk_ok; } static int generic_get_metadata(const char *standard, const char *provider, const char *type, char **output) { svc_action_t *action = resources_action_create(type, standard, provider, type, "meta-data", 0, 5000, NULL); if (!(services_action_sync(action))) { crm_err("Failed to retrieve meta-data for %s:%s:%s", standard, provider, type); services_action_free(action); return -EIO; } if (!action->stdout_data) { crm_err("Failed to retrieve meta-data for %s:%s:%s", standard, provider, type); services_action_free(action); return -EIO; } *output = strdup(action->stdout_data); services_action_free(action); return pcmk_ok; } static int lrmd_api_get_metadata(lrmd_t * lrmd, const char *class, const char *provider, const char *type, char **output, enum lrmd_call_options options) { if (!class || !type) { return -EINVAL; } if (safe_str_eq(class, "stonith")) { return stonith_get_metadata(provider, type, output); } else if (safe_str_eq(class, "lsb")) { return lsb_get_metadata(type, output); } return generic_get_metadata(class, provider, type, output); } static int lrmd_api_exec(lrmd_t * lrmd, const char *rsc_id, const char *action, const char *userdata, int interval, /* ms */ int timeout, /* ms */ int start_delay, /* ms */ enum lrmd_call_options options, lrmd_key_value_t * params) { int rc = pcmk_ok; xmlNode *data = create_xml_node(NULL, F_LRMD_RSC); xmlNode *args = create_xml_node(data, XML_TAG_ATTRS); lrmd_key_value_t *tmp = NULL; crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__); crm_xml_add(data, F_LRMD_RSC_ID, rsc_id); crm_xml_add(data, F_LRMD_RSC_ACTION, action); crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata); crm_xml_add_int(data, F_LRMD_RSC_INTERVAL, interval); crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout); crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay); for (tmp = params; tmp; tmp = tmp->next) { hash2field((gpointer) tmp->key, (gpointer) tmp->value, args); } rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options); free_xml(data); lrmd_key_value_freeall(params); return rc; } static int lrmd_api_cancel(lrmd_t * lrmd, const char *rsc_id, const char *action, int interval) { int rc = pcmk_ok; xmlNode *data = create_xml_node(NULL, F_LRMD_RSC); crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__); crm_xml_add(data, F_LRMD_RSC_ACTION, action); crm_xml_add(data, F_LRMD_RSC_ID, rsc_id); crm_xml_add_int(data, F_LRMD_RSC_INTERVAL, interval); rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0); free_xml(data); return rc; } static int list_stonith_agents(lrmd_list_t ** resources) { int rc = 0; stonith_key_value_t *stonith_resources = NULL; stonith_key_value_t *dIter = NULL; stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL, &stonith_resources, 0); for (dIter = stonith_resources; dIter; dIter = dIter->next) { rc++; if(resources) { *resources = lrmd_list_add(*resources, dIter->value); } } stonith_key_value_freeall(stonith_resources, 1, 0); return rc; } static int lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class, const char *provider) { int rc = 0; if (safe_str_eq(class, "stonith")) { rc += list_stonith_agents(resources); } else { GListPtr gIter = NULL; GList *agents = resources_list_agents(class, provider); for (gIter = agents; gIter != NULL; gIter = gIter->next) { *resources = lrmd_list_add(*resources, (const char *)gIter->data); rc++; } g_list_free_full(agents, free); if (!class) { rc += list_stonith_agents(resources); } } if(rc == 0) { crm_notice("No agents found for class %s", class); rc = -EPROTONOSUPPORT; } return rc; } static int does_provider_have_agent(const char *agent, const char *provider, const char *class) { int found = 0; GList *agents = NULL; GListPtr gIter2 = NULL; agents = resources_list_agents(class, provider); for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) { if (safe_str_eq(agent, gIter2->data)) { found = 1; } } g_list_free_full(agents, free); return found; } static int lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers) { int rc = pcmk_ok; char *provider = NULL; GList *ocf_providers = NULL; GListPtr gIter = NULL; ocf_providers = resources_list_providers("ocf"); for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) { provider = gIter->data; if (!agent || does_provider_have_agent(agent, provider, "ocf")) { *providers = lrmd_list_add(*providers, (const char *)gIter->data); rc++; } } g_list_free_full(ocf_providers, free); return rc; } static int lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported) { int rc = 0; GList *standards = NULL; GListPtr gIter = NULL; standards = resources_list_standards(); for (gIter = standards; gIter != NULL; gIter = gIter->next) { *supported = lrmd_list_add(*supported, (const char *)gIter->data); rc++; } if(list_stonith_agents(NULL) > 0) { *supported = lrmd_list_add(*supported, "stonith"); rc++; } g_list_free_full(standards, free); return rc; } lrmd_t * lrmd_api_new(void) { lrmd_t *new_lrmd = NULL; lrmd_private_t *pvt = NULL; new_lrmd = calloc(1, sizeof(lrmd_t)); pvt = calloc(1, sizeof(lrmd_private_t)); new_lrmd->cmds = calloc(1, sizeof(lrmd_api_operations_t)); + pvt->type = LRMD_CLIENT_IPC; new_lrmd->private = pvt; new_lrmd->cmds->connect = lrmd_api_connect; new_lrmd->cmds->disconnect = lrmd_api_disconnect; new_lrmd->cmds->register_rsc = lrmd_api_register_rsc; new_lrmd->cmds->unregister_rsc = lrmd_api_unregister_rsc; new_lrmd->cmds->get_rsc_info = lrmd_api_get_rsc_info; new_lrmd->cmds->set_callback = lrmd_api_set_callback; new_lrmd->cmds->get_metadata = lrmd_api_get_metadata; new_lrmd->cmds->exec = lrmd_api_exec; new_lrmd->cmds->cancel = lrmd_api_cancel; new_lrmd->cmds->list_agents = lrmd_api_list_agents; new_lrmd->cmds->list_ocf_providers = lrmd_api_list_ocf_providers; new_lrmd->cmds->list_standards = lrmd_api_list_standards; if (!stonith_api) { stonith_api = stonith_api_new(); } return new_lrmd; } +lrmd_t * +lrmd_remote_api_new(const char *server, int port) +{ +#ifdef HAVE_GNUTLS_GNUTLS_H + lrmd_t *new_lrmd = lrmd_api_new(); + lrmd_private_t *native = new_lrmd->private; + + native->type = LRMD_CLIENT_TLS; + native->server = strdup(server); + native->port = port ? port : DEFAULT_REMOTE_PORT; + return new_lrmd; +#else + crm_err("GNUTLS is not enabled for this build, remote LRMD client can not be created"); + return NULL; +#endif + +} + void lrmd_api_delete(lrmd_t * lrmd) { if (!lrmd) { return; } lrmd->cmds->disconnect(lrmd); /* no-op if already disconnected */ free(lrmd->cmds); +#ifdef HAVE_GNUTLS_GNUTLS_H + { /* this is done to avoid compiler warnings */ + lrmd_private_t *native = lrmd->private; + free(native->server); + } +#endif free(lrmd->private); free(lrmd); } diff --git a/lrmd/Makefile.am b/lrmd/Makefile.am index b12c825afd..3a9027605c 100644 --- a/lrmd/Makefile.am +++ b/lrmd/Makefile.am @@ -1,40 +1,40 @@ # Copyright (c) 2012 David Vossel # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # MAINTAINERCLEANFILES = Makefile.in testdir = $(datadir)/$(PACKAGE)/tests/lrmd test_SCRIPTS = regression.py lrmdlibdir = $(CRM_DAEMON_DIR) ## binary progs lrmdlib_PROGRAMS = lrmd lrmd_test -lrmd_SOURCES = main.c lrmd.c +lrmd_SOURCES = main.c lrmd.c tls_backend.c lrmd_LDADD = $(top_builddir)/lib/common/libcrmcommon.la \ $(top_builddir)/lib/services/libcrmservice.la \ $(top_builddir)/lib/lrmd/liblrmd.la \ $(top_builddir)/lib/fencing/libstonithd.la lrmd_test_SOURCES = test.c lrmd_test_LDADD = $(top_builddir)/lib/common/libcrmcommon.la \ $(top_builddir)/lib/lrmd/liblrmd.la \ $(top_builddir)/lib/cib/libcib.la \ $(top_builddir)/lib/services/libcrmservice.la \ $(top_builddir)/lib/pengine/libpe_status.la \ $(top_builddir)/pengine/libpengine.la diff --git a/lrmd/lrmd.c b/lrmd/lrmd.c index 3fd7ab4877..ffabf7178d 100644 --- a/lrmd/lrmd.c +++ b/lrmd/lrmd.c @@ -1,1225 +1,1221 @@ /* * Copyright (c) 2012 David Vossel * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * */ #include #include #include #include #include #include -#include #include #include #ifdef HAVE_SYS_TIMEB_H # include #endif GHashTable *rsc_list = NULL; GHashTable *client_list = NULL; typedef struct lrmd_cmd_s { int timeout; int interval; int start_delay; int call_id; int exec_rc; int lrmd_op_status; int call_opts; /* Timer ids, must be removed on cmd destruction. */ int delay_id; int stonith_recurring_id; int rsc_deleted; char *client_id; char *origin; char *rsc_id; char *action; char *output; char *userdata_str; #ifdef HAVE_SYS_TIMEB_H /* Timestamp of when op ran */ struct timeb t_run; /* Timestamp of when op was queued */ struct timeb t_queue; /* Timestamp of last rc change */ struct timeb t_rcchange; #endif int first_notify_sent; int last_notify_rc; int last_notify_op_status; int last_pid; GHashTable *params; } lrmd_cmd_t; static void cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc); static gboolean lrmd_rsc_dispatch(gpointer user_data); static void cancel_all_recurring(lrmd_rsc_t *rsc, const char *client_id); static void log_finished(lrmd_cmd_t *cmd, int exec_time, int queue_time) { char pid_str[32] = { 0, }; int log_level = LOG_INFO; if (cmd->last_pid) { snprintf(pid_str, 32, "%d", cmd->last_pid); } if (safe_str_eq(cmd->action, "monitor")) { log_level = LOG_DEBUG; } #ifdef HAVE_SYS_TIMEB_H do_crm_log(log_level, "finished - rsc:%s action:%s call_id:%d %s%s exit-code:%d exec-time:%dms queue-time:%dms", cmd->rsc_id, cmd->action, cmd->call_id, cmd->last_pid ? "pid:" : "", pid_str, cmd->exec_rc, exec_time, queue_time); #else do_crm_log(log_level, "finished - rsc:%s action:%s call_id:%d %s%s exit-code:%d", cmd->rsc_id, cmd->action, cmd->call_id, cmd->last_pid ? "pid:" : "", pid_str, cmd->exec_rc); #endif } static void log_execute(lrmd_cmd_t *cmd) { int log_level = LOG_INFO; if (safe_str_eq(cmd->action, "monitor")) { log_level = LOG_DEBUG; } do_crm_log(log_level, "executing - rsc:%s action:%s call_id:%d", cmd->rsc_id, cmd->action, cmd->call_id); } static lrmd_rsc_t * build_rsc_from_xml(xmlNode * msg) { xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, msg, LOG_ERR); lrmd_rsc_t *rsc = NULL; rsc = calloc(1, sizeof(lrmd_rsc_t)); crm_element_value_int(msg, F_LRMD_CALLOPTS, &rsc->call_opts); rsc->rsc_id = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ID); rsc->class = crm_element_value_copy(rsc_xml, F_LRMD_CLASS); rsc->provider = crm_element_value_copy(rsc_xml, F_LRMD_PROVIDER); rsc->type = crm_element_value_copy(rsc_xml, F_LRMD_TYPE); rsc->work = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_rsc_dispatch, rsc); return rsc; } static lrmd_cmd_t * create_lrmd_cmd(xmlNode * msg, lrmd_client_t * client) { int call_options = 0; xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, msg, LOG_ERR); lrmd_cmd_t *cmd = NULL; cmd = calloc(1, sizeof(lrmd_cmd_t)); crm_element_value_int(msg, F_LRMD_CALLOPTS, &call_options); cmd->call_opts = call_options; cmd->client_id = strdup(client->id); crm_element_value_int(msg, F_LRMD_CALLID, &cmd->call_id); crm_element_value_int(rsc_xml, F_LRMD_RSC_INTERVAL, &cmd->interval); crm_element_value_int(rsc_xml, F_LRMD_TIMEOUT, &cmd->timeout); crm_element_value_int(rsc_xml, F_LRMD_RSC_START_DELAY, &cmd->start_delay); cmd->origin = crm_element_value_copy(rsc_xml, F_LRMD_ORIGIN); cmd->action = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ACTION); cmd->userdata_str = crm_element_value_copy(rsc_xml, F_LRMD_RSC_USERDATA_STR); cmd->rsc_id = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ID); cmd->params = xml2list(rsc_xml); return cmd; } static void free_lrmd_cmd(lrmd_cmd_t * cmd) { if (cmd->stonith_recurring_id) { g_source_remove(cmd->stonith_recurring_id); } if (cmd->delay_id) { g_source_remove(cmd->delay_id); } if (cmd->params) { g_hash_table_destroy(cmd->params); } free(cmd->origin); free(cmd->action); free(cmd->userdata_str); free(cmd->rsc_id); free(cmd->output); free(cmd->client_id); free(cmd); } static gboolean stonith_recurring_op_helper(gpointer data) { lrmd_cmd_t *cmd = data; lrmd_rsc_t *rsc; cmd->stonith_recurring_id = 0; if (!cmd->rsc_id) { return FALSE; } rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id); CRM_ASSERT(rsc != NULL); /* take it out of recurring_ops list, and put it in the pending ops * to be executed */ rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd); rsc->pending_ops = g_list_append(rsc->pending_ops, cmd); #ifdef HAVE_SYS_TIMEB_H ftime(&cmd->t_queue); #endif mainloop_set_trigger(rsc->work); return FALSE; } static gboolean start_delay_helper(gpointer data) { lrmd_cmd_t *cmd = data; lrmd_rsc_t *rsc = NULL; cmd->delay_id = 0; rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL; if (rsc) { mainloop_set_trigger(rsc->work); } return FALSE; } static void schedule_lrmd_cmd(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd) { CRM_CHECK(cmd != NULL, return); CRM_CHECK(rsc != NULL, return); crm_trace("Scheduling %s on %s", cmd->action, rsc->rsc_id); rsc->pending_ops = g_list_append(rsc->pending_ops, cmd); #ifdef HAVE_SYS_TIMEB_H ftime(&cmd->t_queue); #endif mainloop_set_trigger(rsc->work); if (cmd->start_delay) { cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd); } } static void send_reply(lrmd_client_t * client, int rc, uint32_t id, int call_id) { int send_rc = 0; xmlNode *reply = NULL; reply = create_xml_node(NULL, T_LRMD_REPLY); crm_xml_add(reply, F_LRMD_ORIGIN, __FUNCTION__); crm_xml_add_int(reply, F_LRMD_RC, rc); crm_xml_add_int(reply, F_LRMD_CALLID, call_id); - send_rc = crm_ipcs_send(client->channel, id, reply, FALSE); + send_rc = lrmd_server_send_reply(client, id, reply); free_xml(reply); if (send_rc < 0) { crm_warn("LRMD reply to %s failed: %d", client->name, send_rc); } } static void send_client_notify(gpointer key, gpointer value, gpointer user_data) { xmlNode *update_msg = user_data; lrmd_client_t *client = value; if (client == NULL) { crm_err("Asked to send event to NULL client"); return; - } else if (client->channel == NULL) { - crm_trace("Asked to send event to disconnected client"); - return; } else if (client->name == NULL) { crm_trace("Asked to send event to client with no name"); return; } - if (crm_ipcs_send(client->channel, 0, update_msg, TRUE) <= 0) { + if (lrmd_server_send_notify(client, update_msg) <= 0) { crm_warn("Notification of client %s/%s failed", client->name, client->id); } } #ifdef HAVE_SYS_TIMEB_H static int time_diff_ms(struct timeb *now, struct timeb *old) { int sec = difftime(now->time, old->time); int ms = now->millitm - old->millitm; if (old->time == 0) { return 0; } return (sec * 1000) + ms; } #endif static void send_cmd_complete_notify(lrmd_cmd_t * cmd) { int exec_time = 0; int queue_time = 0; xmlNode *notify = NULL; #ifdef HAVE_SYS_TIMEB_H struct timeb now = { 0, }; ftime(&now); exec_time = time_diff_ms(&now, &cmd->t_run); queue_time = time_diff_ms(&cmd->t_run, &cmd->t_queue); #endif log_finished(cmd, exec_time, queue_time); /* if the first notify result for a cmd has already been sent earlier, and the * the option to only send notifies on result changes is set. Check to see * if the last result is the same as the new one. If so, suppress this update */ if (cmd->first_notify_sent && (cmd->call_opts & lrmd_opt_notify_changes_only)) { if (cmd->last_notify_rc == cmd->exec_rc && cmd->last_notify_op_status == cmd->lrmd_op_status) { /* only send changes */ return; } } cmd->first_notify_sent = 1; cmd->last_notify_rc = cmd->exec_rc; cmd->last_notify_op_status = cmd->lrmd_op_status; notify = create_xml_node(NULL, T_LRMD_NOTIFY); crm_xml_add(notify, F_LRMD_ORIGIN, __FUNCTION__); crm_xml_add_int(notify, F_LRMD_TIMEOUT, cmd->timeout); crm_xml_add_int(notify, F_LRMD_RSC_INTERVAL, cmd->interval); crm_xml_add_int(notify, F_LRMD_RSC_START_DELAY, cmd->start_delay); crm_xml_add_int(notify, F_LRMD_EXEC_RC, cmd->exec_rc); crm_xml_add_int(notify, F_LRMD_OP_STATUS, cmd->lrmd_op_status); crm_xml_add_int(notify, F_LRMD_CALLID, cmd->call_id); crm_xml_add_int(notify, F_LRMD_RSC_DELETED, cmd->rsc_deleted); #ifdef HAVE_SYS_TIMEB_H crm_xml_add_int(notify, F_LRMD_RSC_RUN_TIME, cmd->t_run.time); crm_xml_add_int(notify, F_LRMD_RSC_RCCHANGE_TIME, cmd->t_rcchange.time); crm_xml_add_int(notify, F_LRMD_RSC_EXEC_TIME, exec_time); crm_xml_add_int(notify, F_LRMD_RSC_QUEUE_TIME, queue_time); #endif crm_xml_add(notify, F_LRMD_OPERATION, LRMD_OP_RSC_EXEC); crm_xml_add(notify, F_LRMD_RSC_ID, cmd->rsc_id); crm_xml_add(notify, F_LRMD_RSC_ACTION, cmd->action); crm_xml_add(notify, F_LRMD_RSC_USERDATA_STR, cmd->userdata_str); crm_xml_add(notify, F_LRMD_RSC_OUTPUT, cmd->output); if (cmd->params) { char *key = NULL; char *value = NULL; GHashTableIter iter; xmlNode *args = create_xml_node(notify, XML_TAG_ATTRS); g_hash_table_iter_init(&iter, cmd->params); while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) { hash2field((gpointer) key, (gpointer) value, args); } } if (cmd->client_id && (cmd->call_opts & lrmd_opt_notify_orig_only)) { lrmd_client_t *client = g_hash_table_lookup(client_list, cmd->client_id); if (client) { send_client_notify(client->id, client, notify); } } else { g_hash_table_foreach(client_list, send_client_notify, notify); } free_xml(notify); } static void send_generic_notify(int rc, xmlNode * request) { int call_id = 0; xmlNode *notify = NULL; xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR); const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID); const char *op = crm_element_value(request, F_LRMD_OPERATION); crm_element_value_int(request, F_LRMD_CALLID, &call_id); notify = create_xml_node(NULL, T_LRMD_NOTIFY); crm_xml_add(notify, F_LRMD_ORIGIN, __FUNCTION__); crm_xml_add_int(notify, F_LRMD_RC, rc); crm_xml_add_int(notify, F_LRMD_CALLID, call_id); crm_xml_add(notify, F_LRMD_OPERATION, op); crm_xml_add(notify, F_LRMD_RSC_ID, rsc_id); g_hash_table_foreach(client_list, send_client_notify, notify); free_xml(notify); } static void cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc) { crm_trace("Resource operation rsc:%s action:%s completed (%p %p)", cmd->rsc_id, cmd->action, rsc?rsc->active:NULL, cmd); if (rsc && (rsc->active == cmd)) { rsc->active = NULL; mainloop_set_trigger(rsc->work); } if (!rsc) { cmd->rsc_deleted = 1; } send_cmd_complete_notify(cmd); /* crmd expects lrmd to automatically cancel recurring ops after rsc stops */ if (rsc && safe_str_eq(cmd->action, "stop")) { cancel_all_recurring(rsc, NULL); } if (cmd->interval && (cmd->lrmd_op_status == PCMK_LRM_OP_CANCELLED)) { if (rsc) { rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd); rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd); } free_lrmd_cmd(cmd); } else if (cmd->interval == 0) { if (rsc) { rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd); } free_lrmd_cmd(cmd); } else { /* Clear all the values pertaining just to the last iteration of a recurring op. */ cmd->lrmd_op_status = 0; cmd->last_pid = 0; memset(&cmd->t_run, 0, sizeof(cmd->t_run)); memset(&cmd->t_queue, 0, sizeof(cmd->t_queue)); free(cmd->output); cmd->output = NULL; } } static int lsb2uniform_rc(const char *action, int rc) { if (rc < 0) { return PCMK_EXECRA_UNKNOWN_ERROR; } /* status has different return codes that everything else. */ if (!safe_str_eq(action, "status") && !safe_str_eq(action, "monitor")) { if (rc > PCMK_LSB_NOT_RUNNING) { return PCMK_EXECRA_UNKNOWN_ERROR; } return rc; } switch (rc) { case PCMK_LSB_STATUS_OK: return PCMK_EXECRA_OK; case PCMK_LSB_STATUS_NOT_INSTALLED: return PCMK_EXECRA_NOT_INSTALLED; case PCMK_LSB_STATUS_VAR_PID: case PCMK_LSB_STATUS_VAR_LOCK: case PCMK_LSB_STATUS_NOT_RUNNING: return PCMK_EXECRA_NOT_RUNNING; default: return PCMK_EXECRA_UNKNOWN_ERROR; } return PCMK_EXECRA_UNKNOWN_ERROR; } static int ocf2uniform_rc(int rc) { if (rc < 0 || rc > PCMK_OCF_FAILED_MASTER) { return PCMK_EXECRA_UNKNOWN_ERROR; } return rc; } static int stonith2uniform_rc(const char *action, int rc) { if (rc == -ENODEV) { if (safe_str_eq(action, "stop")) { rc = PCMK_EXECRA_OK; } else if (safe_str_eq(action, "start")) { rc = PCMK_EXECRA_NOT_INSTALLED; } else { rc = PCMK_EXECRA_NOT_RUNNING; } } else if (rc != 0) { rc = PCMK_EXECRA_UNKNOWN_ERROR; } return rc; } static int get_uniform_rc(const char *standard, const char *action, int rc) { if (safe_str_eq(standard, "ocf")) { return ocf2uniform_rc(rc); } else if (safe_str_eq(standard, "stonith")) { return stonith2uniform_rc(action, rc); } else if (safe_str_eq(standard, "systemd")) { return rc; } else if (safe_str_eq(standard, "upstart")) { return rc; } else { return lsb2uniform_rc(action, rc); } } void client_disconnect_cleanup(const char *client_id) { GHashTableIter iter; lrmd_rsc_t *rsc = NULL; char *key = NULL; g_hash_table_iter_init(&iter, rsc_list); while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & rsc)) { if (rsc->call_opts & lrmd_opt_drop_recurring) { /* This client is disconnecting, drop any recurring operations * it may have initiated on the resource */ cancel_all_recurring(rsc, client_id); } } } static void action_complete(svc_action_t * action) { lrmd_rsc_t *rsc; lrmd_cmd_t *cmd = action->cb_data; if (!cmd) { crm_err("LRMD action (%s) completed does not match any known operations.", action->id); return; } #ifdef HAVE_SYS_TIMEB_H if (cmd->exec_rc != action->rc) { ftime(&cmd->t_rcchange); } #endif cmd->last_pid = action->pid; cmd->exec_rc = get_uniform_rc(action->standard, cmd->action, action->rc); cmd->lrmd_op_status = action->status; rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL; if (action->stdout_data) { cmd->output = strdup(action->stdout_data); } cmd_finalize(cmd, rsc); } static void stonith_action_complete(lrmd_cmd_t *cmd, int rc) { int recurring = cmd->interval; lrmd_rsc_t *rsc = NULL; cmd->exec_rc = get_uniform_rc("stonith", cmd->action, rc); rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id); if (cmd->lrmd_op_status == PCMK_LRM_OP_CANCELLED) { recurring = 0; /* do nothing */ } else if (rc) { /* Attempt to map return codes to op status if possible */ switch (rc) { case -EPROTONOSUPPORT: cmd->lrmd_op_status = PCMK_LRM_OP_NOTSUPPORTED; break; case -ETIME: cmd->lrmd_op_status = PCMK_LRM_OP_TIMEOUT; break; default: cmd->lrmd_op_status = PCMK_LRM_OP_ERROR; } } else { /* command successful */ cmd->lrmd_op_status = PCMK_LRM_OP_DONE; if (safe_str_eq(cmd->action, "start") && rsc) { rsc->stonith_started = 1; } } if (recurring && rsc) { if (cmd->stonith_recurring_id) { g_source_remove(cmd->stonith_recurring_id); } cmd->stonith_recurring_id = g_timeout_add(cmd->interval, stonith_recurring_op_helper, cmd); } cmd_finalize(cmd, rsc); } static void lrmd_stonith_callback(stonith_t * stonith, stonith_callback_data_t *data) { stonith_action_complete(data->userdata, data->rc); } void stonith_connection_failed(void) { GHashTableIter iter; GList *cmd_list = NULL; GList *cmd_iter = NULL; lrmd_rsc_t *rsc = NULL; char *key = NULL; g_hash_table_iter_init(&iter, rsc_list); while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & rsc)) { if (safe_str_eq(rsc->class, "stonith")) { if (rsc->recurring_ops) { cmd_list = g_list_concat(cmd_list, rsc->recurring_ops); } if (rsc->pending_ops) { cmd_list = g_list_concat(cmd_list, rsc->pending_ops); } rsc->pending_ops = rsc->recurring_ops = NULL; } } if (!cmd_list) { return; } crm_err("STONITH connection failed, finalizing %d pending operations.", g_list_length(cmd_list)); for (cmd_iter = cmd_list; cmd_iter; cmd_iter = cmd_iter->next) { stonith_action_complete(cmd_iter->data, -ENOTCONN); } g_list_free(cmd_list); } static int lrmd_rsc_execute_stonith(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd) { int rc = 0; int do_monitor = 0; stonith_t *stonith_api = get_stonith_connection(); if (!stonith_api) { cmd->exec_rc = get_uniform_rc("stonith", cmd->action, -ENOTCONN); cmd->lrmd_op_status = PCMK_LRM_OP_ERROR; cmd_finalize(cmd, rsc); return -EUNATCH; } if (safe_str_eq(cmd->action, "start")) { char *key = NULL; char *value = NULL; stonith_key_value_t *device_params = NULL; if (cmd->params) { GHashTableIter iter; g_hash_table_iter_init(&iter, cmd->params); while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) { device_params = stonith_key_value_add(device_params, key, value); } } /* Stonith automatically registers devices from the CIB when changes occur, * but to avoid a possible race condition between stonith receiving the CIB update * and the lrmd requesting that resource, the lrmd still registers the device as well. * Stonith knows how to handle duplicate device registrations correctly. */ rc = stonith_api->cmds->register_device(stonith_api, st_opt_sync_call, cmd->rsc_id, rsc->provider, rsc->type, device_params); stonith_key_value_freeall(device_params, 1, 1); if (rc == 0) { do_monitor = 1; } } else if (safe_str_eq(cmd->action, "stop")) { rc = stonith_api->cmds->remove_device(stonith_api, st_opt_sync_call, cmd->rsc_id); rsc->stonith_started = 0; } else if (safe_str_eq(cmd->action, "monitor")) { if (cmd->interval) { do_monitor = 1; } else { rc = rsc->stonith_started ? 0 : -ENODEV; } } if (!do_monitor) { goto cleanup_stonith_exec; } rc = stonith_api->cmds->monitor(stonith_api, 0, cmd->rsc_id, cmd->timeout / 1000 ); rc = stonith_api->cmds->register_callback( stonith_api, rc, 0, 0, cmd, "lrmd_stonith_callback", lrmd_stonith_callback); /* don't cleanup yet, we will find out the result of the monitor later */ if (rc > 0) { rsc->active = cmd; return rc; } else if (rc == 0) { rc = -1; } cleanup_stonith_exec: stonith_action_complete(cmd, rc); return rc; } static const char * normalize_action_name(lrmd_rsc_t * rsc, const char *action) { if (safe_str_eq(action, "monitor") && (safe_str_eq(rsc->class, "lsb") || safe_str_eq(rsc->class, "service") || safe_str_eq(rsc->class, "systemd"))) { return "status"; } return action; } static void dup_attr(gpointer key, gpointer value, gpointer user_data) { g_hash_table_replace(user_data, strdup(key), strdup(value)); } static int lrmd_rsc_execute_service_lib(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd) { svc_action_t *action = NULL; GHashTable *params_copy = NULL; CRM_ASSERT(rsc); CRM_ASSERT(cmd); crm_trace("Creating action, resource:%s action:%s class:%s provider:%s agent:%s", rsc->rsc_id, cmd->action, rsc->class, rsc->provider, rsc->type); if (cmd->params) { params_copy = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); if (params_copy != NULL) { g_hash_table_foreach(cmd->params, dup_attr, params_copy); } } action = resources_action_create(rsc->rsc_id, rsc->class, rsc->provider, rsc->type, normalize_action_name(rsc, cmd->action), cmd->interval, cmd->timeout, params_copy); if (!action) { crm_err("Failed to create action, action:%s on resource %s", cmd->action, rsc->rsc_id); cmd->lrmd_op_status = PCMK_LRM_OP_ERROR; goto exec_done; } action->cb_data = cmd; /* 'cmd' may not be valid after this point * * Upstart and systemd both synchronously determine monitor/status * results and call action_complete (which may free 'cmd') if necessary */ if (services_action_async(action, action_complete)) { return TRUE; } cmd->exec_rc = action->rc; cmd->lrmd_op_status = PCMK_LRM_OP_ERROR; services_action_free(action); action = NULL; exec_done: cmd_finalize(cmd, rsc); return TRUE; } static gboolean lrmd_rsc_execute(lrmd_rsc_t * rsc) { lrmd_cmd_t *cmd = NULL; CRM_CHECK(rsc != NULL, return FALSE); if (rsc->active) { crm_trace("%s is still active", rsc->rsc_id); return TRUE; } if (rsc->pending_ops) { GList *first = rsc->pending_ops; cmd = first->data; if (cmd->delay_id) { crm_trace ("Command %s %s was asked to run too early, waiting for start_delay timeout of %dms", cmd->rsc_id, cmd->action, cmd->start_delay); return TRUE; } rsc->pending_ops = g_list_remove_link(rsc->pending_ops, first); g_list_free_1(first); #ifdef HAVE_SYS_TIMEB_H ftime(&cmd->t_run); } #endif if (!cmd) { crm_trace("Nothing further to do for %s", rsc->rsc_id); return TRUE; } rsc->active = cmd; /* only one op at a time for a rsc */ if (cmd->interval) { rsc->recurring_ops = g_list_append(rsc->recurring_ops, cmd); } log_execute(cmd); if (safe_str_eq(rsc->class, "stonith")) { lrmd_rsc_execute_stonith(rsc, cmd); } else { lrmd_rsc_execute_service_lib(rsc, cmd); } return TRUE; } static gboolean lrmd_rsc_dispatch(gpointer user_data) { return lrmd_rsc_execute(user_data); } void free_rsc(gpointer data) { GListPtr gIter = NULL; lrmd_rsc_t *rsc = data; int is_stonith = safe_str_eq(rsc->class, "stonith"); for (gIter = rsc->pending_ops; gIter != NULL; gIter = gIter->next) { lrmd_cmd_t *cmd = gIter->data; /* command was never executed */ cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED; cmd_finalize(cmd, NULL); } /* frees list, but not list elements. */ g_list_free(rsc->pending_ops); for (gIter = rsc->recurring_ops; gIter != NULL; gIter = gIter->next) { lrmd_cmd_t *cmd = gIter->data; if (is_stonith) { cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED; cmd_finalize(cmd, NULL); } else { /* This command is already handed off to service library, * let service library cancel it and tell us via the callback * when it is cancelled. The rsc can be safely destroyed * even if we are waiting for the cancel result */ services_action_cancel(rsc->rsc_id, cmd->action, cmd->interval); } } /* frees list, but not list elements. */ g_list_free(rsc->recurring_ops); free(rsc->rsc_id); free(rsc->class); free(rsc->provider); free(rsc->type); mainloop_destroy_trigger(rsc->work); free(rsc); } static int process_lrmd_signon(lrmd_client_t * client, uint32_t id, xmlNode * request) { xmlNode *reply = create_xml_node(NULL, "reply"); crm_xml_add(reply, F_LRMD_OPERATION, CRM_OP_REGISTER); crm_xml_add(reply, F_LRMD_CLIENTID, client->id); - crm_ipcs_send(client->channel, id, reply, FALSE); + lrmd_server_send_reply(client, id, reply); free_xml(reply); return pcmk_ok; } static int process_lrmd_rsc_register(lrmd_client_t * client, uint32_t id, xmlNode * request) { int rc = pcmk_ok; lrmd_rsc_t *rsc = build_rsc_from_xml(request); lrmd_rsc_t *dup = g_hash_table_lookup(rsc_list, rsc->rsc_id); if (dup && safe_str_eq(rsc->class, dup->class) && safe_str_eq(rsc->provider, dup->provider) && safe_str_eq(rsc->type, dup->type)) { crm_warn("Can't add, RSC '%s' already present in the rsc list (%d active resources)", rsc->rsc_id, g_hash_table_size(rsc_list)); free_rsc(rsc); return rc; } g_hash_table_replace(rsc_list, rsc->rsc_id, rsc); crm_info("Added '%s' to the rsc list (%d active resources)", rsc->rsc_id, g_hash_table_size(rsc_list)); return rc; } static void process_lrmd_get_rsc_info(lrmd_client_t * client, uint32_t id, xmlNode * request) { int rc = pcmk_ok; int send_rc = 0; int call_id = 0; xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR); const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID); xmlNode *reply = NULL; lrmd_rsc_t *rsc = NULL; crm_element_value_int(request, F_LRMD_CALLID, &call_id); if (!rsc_id) { rc = -ENODEV; goto get_rsc_done; } if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) { crm_info("Resource '%s' not found (%d active resources)", rsc_id, g_hash_table_size(rsc_list)); rc = -ENODEV; goto get_rsc_done; } get_rsc_done: reply = create_xml_node(NULL, T_LRMD_REPLY); crm_xml_add(reply, F_LRMD_ORIGIN, __FUNCTION__); crm_xml_add_int(reply, F_LRMD_RC, rc); crm_xml_add_int(reply, F_LRMD_CALLID, call_id); if (rsc) { crm_xml_add(reply, F_LRMD_RSC_ID, rsc->rsc_id); crm_xml_add(reply, F_LRMD_CLASS, rsc->class); crm_xml_add(reply, F_LRMD_PROVIDER, rsc->provider); crm_xml_add(reply, F_LRMD_TYPE, rsc->type); } - send_rc = crm_ipcs_send(client->channel, id, reply, FALSE); + send_rc = lrmd_server_send_reply(client, id, reply); if (send_rc < 0) { crm_warn("LRMD reply to %s failed: %d", client->name, send_rc); } free_xml(reply); } static int process_lrmd_rsc_unregister(lrmd_client_t * client, uint32_t id, xmlNode * request) { int rc = pcmk_ok; lrmd_rsc_t *rsc = NULL; xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR); const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID); if (!rsc_id) { return -ENODEV; } if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) { crm_info("Resource '%s' not found (%d active resources)", rsc_id, g_hash_table_size(rsc_list)); return pcmk_ok; } if (rsc->active) { /* let the caller know there are still active ops on this rsc to watch for */ crm_trace("Operation still in progress: %p", rsc->active); rc = -EINPROGRESS; } g_hash_table_remove(rsc_list, rsc_id); return rc; } static int process_lrmd_rsc_exec(lrmd_client_t * client, uint32_t id, xmlNode * request) { lrmd_rsc_t *rsc = NULL; lrmd_cmd_t *cmd = NULL; xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR); const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID); if (!rsc_id) { return -EINVAL; } if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) { crm_info("Resource '%s' not found (%d active resources)", rsc_id, g_hash_table_size(rsc_list)); return -ENODEV; } cmd = create_lrmd_cmd(request, client); schedule_lrmd_cmd(rsc, cmd); return cmd->call_id; } static int cancel_op(const char *rsc_id, const char *action, int interval) { GListPtr gIter = NULL; lrmd_rsc_t *rsc = g_hash_table_lookup(rsc_list, rsc_id); /* How to cancel an action. * 1. Check pending ops list, if it hasn't been handed off * to the service library or stonith recurring list remove * it there and that will stop it. * 2. If it isn't in the pending ops list, then its either a * recurring op in the stonith recurring list, or the service * library's recurring list. Stop it there * 3. If not found in any lists, then this operation has either * been executed already and is not a recurring operation, or * never existed. */ if (!rsc) { return -ENODEV; } for (gIter = rsc->pending_ops; gIter != NULL; gIter = gIter->next) { lrmd_cmd_t *cmd = gIter->data; if (safe_str_eq(cmd->action, action) && cmd->interval == interval) { cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED; cmd_finalize(cmd, rsc); return pcmk_ok; } } if (safe_str_eq(rsc->class, "stonith")) { /* The service library does not handle stonith operations. * We have to handle recurring stonith opereations ourselves. */ for (gIter = rsc->recurring_ops; gIter != NULL; gIter = gIter->next) { lrmd_cmd_t *cmd = gIter->data; if (safe_str_eq(cmd->action, action) && cmd->interval == interval) { cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED; if (rsc->active != cmd) { cmd_finalize(cmd, rsc); } return pcmk_ok; } } } else if (services_action_cancel(rsc_id, normalize_action_name(rsc, action), interval) == TRUE) { /* The service library will tell the action_complete callback function * this action was cancelled, which will destroy the cmd and remove * it from the recurring_op list. Do not do that in this function * if the service library says it cancelled it. */ return pcmk_ok; } return -EOPNOTSUPP; } static void cancel_all_recurring(lrmd_rsc_t *rsc, const char *client_id) { GList *cmd_list = NULL; GList *cmd_iter = NULL; /* Notice a copy of each list is created when concat is called. * This prevents odd behavior from occurring when the cmd_list * is iterated through later on. It is possible the cancel_op * function may end up modifying the recurring_ops and pending_ops * lists. If we did not copy those lists, our cmd_list iteration * could get messed up.*/ if (rsc->recurring_ops) { cmd_list = g_list_concat(cmd_list, g_list_copy(rsc->recurring_ops)); } if (rsc->pending_ops) { cmd_list = g_list_concat(cmd_list, g_list_copy(rsc->pending_ops)); } if (!cmd_list) { return; } for (cmd_iter = cmd_list; cmd_iter; cmd_iter = cmd_iter->next) { lrmd_cmd_t *cmd = cmd_iter->data; if (cmd->interval == 0) { continue; } if (client_id && safe_str_neq(cmd->client_id, client_id)) { continue; } cancel_op(rsc->rsc_id, cmd->action, cmd->interval); } /* frees only the copied list data, not the cmds */ g_list_free(cmd_list); } static int process_lrmd_rsc_cancel(lrmd_client_t * client, uint32_t id, xmlNode * request) { xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR); const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID); const char *action = crm_element_value(rsc_xml, F_LRMD_RSC_ACTION); int interval = 0; crm_element_value_int(rsc_xml, F_LRMD_RSC_INTERVAL, &interval); if (!rsc_id || !action) { return -EINVAL; } return cancel_op(rsc_id, action, interval); } void process_lrmd_message(lrmd_client_t * client, uint32_t id, xmlNode * request) { int rc = pcmk_ok; int call_id = 0; const char *op = crm_element_value(request, F_LRMD_OPERATION); int do_reply = 0; int do_notify = 0; int exit = 0; crm_trace("Processing %s operation from %s", op, client->id); crm_element_value_int(request, F_LRMD_CALLID, &call_id); if (crm_str_eq(op, CRM_OP_REGISTER, TRUE)) { rc = process_lrmd_signon(client, id, request); } else if (crm_str_eq(op, LRMD_OP_RSC_REG, TRUE)) { rc = process_lrmd_rsc_register(client, id, request); do_notify = 1; do_reply = 1; } else if (crm_str_eq(op, LRMD_OP_RSC_INFO, TRUE)) { process_lrmd_get_rsc_info(client, id, request); } else if (crm_str_eq(op, LRMD_OP_RSC_UNREG, TRUE)) { rc = process_lrmd_rsc_unregister(client, id, request); /* don't notify anyone about failed un-registers */ if (rc == pcmk_ok || rc == -EINPROGRESS) { do_notify = 1; } do_reply = 1; } else if (crm_str_eq(op, LRMD_OP_RSC_EXEC, TRUE)) { rc = process_lrmd_rsc_exec(client, id, request); do_reply = 1; } else if (crm_str_eq(op, LRMD_OP_RSC_CANCEL, TRUE)) { rc = process_lrmd_rsc_cancel(client, id, request); do_reply = 1; } else { rc = -EOPNOTSUPP; do_reply = 1; crm_err("Unknown %s from %s", op, client->name); crm_log_xml_warn(request, "UnknownOp"); } crm_debug("Processed %s operation from %s: rc=%d, reply=%d, notify=%d, exit=%d", op, client->id, rc, do_reply, do_notify, exit); if (do_reply) { send_reply(client, rc, id, call_id); } if (do_notify) { send_generic_notify(rc, request); } if (exit) { lrmd_shutdown(0); } } diff --git a/lrmd/lrmd_private.h b/lrmd/lrmd_private.h index fe2ee95d0b..fe7576561d 100644 --- a/lrmd/lrmd_private.h +++ b/lrmd/lrmd_private.h @@ -1,87 +1,114 @@ /* * Copyright (c) 2012 David Vossel * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef LRMD_PVT__H # define LRMD_PVT__H # include # include # include # include +#ifdef HAVE_GNUTLS_GNUTLS_H +# undef KEYFILE +# include +#endif GHashTable *rsc_list; GHashTable *client_list; typedef struct lrmd_rsc_s { char *rsc_id; char *class; char *provider; char *type; int call_opts; /* NEVER dereference this pointer, * It simply exists as a switch to let us know * when the currently active operation has completed */ void *active; /* Operations in this list * have not been executed yet. */ GList *pending_ops; /* Operations in this list are recurring operations * that have been handed off from the pending ops list. */ GList *recurring_ops; int stonith_started; crm_trigger_t *work; } lrmd_rsc_t; typedef struct lrmd_client_s { char *id; char *name; - + enum lrmd_client_type type; + +#ifdef HAVE_GNUTLS_GNUTLS_H + gnutls_session *session; + char *recv_buf; + gboolean tls_handshake_complete; + int remote_auth_timeout; + mainloop_io_t *remote; +#endif qb_ipcs_connection_t *channel; long long flags; } lrmd_client_t; + +#ifdef HAVE_GNUTLS_GNUTLS_H +/* in remote_tls.c */ +int lrmd_init_remote_tls_server(int port); +void lrmd_tls_server_destroy(void); + +/* Hidden in lrmd client lib */ +extern int lrmd_tls_send_msg(gnutls_session *session, xmlNode *msg, uint32_t id, const char *msg_type); +extern int lrmd_tls_set_key(gnutls_datum_t *key, const char *location); +#endif + +int lrmd_server_send_reply(lrmd_client_t *client, uint32_t id, xmlNode *reply); + +int lrmd_server_send_notify(lrmd_client_t *client, xmlNode *msg); + void process_lrmd_message(lrmd_client_t * client, uint32_t id, xmlNode * request); void free_rsc(gpointer data); void lrmd_shutdown(int nsig); void client_disconnect_cleanup(const char *client_id); /*! * \brief Don't worry about freeing this connection. It is * taken care of after mainloop exits by the main() function. */ stonith_t *get_stonith_connection(void); /*! * \brief This is a callback that tells the lrmd * the current stonith connection has gone away. This allows * us to timeout any pending stonith commands */ void stonith_connection_failed(void); #endif diff --git a/lrmd/main.c b/lrmd/main.c index 954177bb6b..f4db7cf05d 100644 --- a/lrmd/main.c +++ b/lrmd/main.c @@ -1,280 +1,354 @@ /* * Copyright (c) 2012 David Vossel * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * */ #include #include #include #include #include #include #include #include #include GMainLoop *mainloop = NULL; qb_ipcs_service_t *ipcs = NULL; stonith_t *stonith_api = NULL; -static int global_call_id = 0; +static gboolean enable_remote = FALSE; +static int remote_port = 0; +int lrmd_call_id = 0; static void stonith_connection_destroy_cb(stonith_t * st, stonith_event_t *e) { stonith_api->state = stonith_disconnected; crm_err("LRMD lost STONITH connection"); stonith_connection_failed(); } stonith_t * get_stonith_connection(void) { if (stonith_api && stonith_api->state == stonith_disconnected) { stonith_api_delete(stonith_api); stonith_api = NULL; } if (!stonith_api) { int rc = 0; int tries = 10; stonith_api = stonith_api_new(); do { rc = stonith_api->cmds->connect(stonith_api, "lrmd", NULL); if (rc == pcmk_ok) { stonith_api->cmds->register_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT, stonith_connection_destroy_cb); break; } sleep(1); tries--; } while (tries); if (rc) { crm_err("Unable to connect to stonith daemon to execute command. error: %s", pcmk_strerror(rc)); stonith_api_delete(stonith_api); stonith_api = NULL; } } return stonith_api; } static int32_t lrmd_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid) { struct qb_ipcs_connection_stats stats = { 0, }; qb_ipcs_connection_stats_get(c, &stats, 1); crm_info("Accepting client connection: %p pid=%d for uid=%d gid=%d", c, stats.client_pid, uid, gid); return 0; } static void lrmd_ipc_created(qb_ipcs_connection_t * c) { lrmd_client_t *new_client = NULL; new_client = calloc(1, sizeof(lrmd_client_t)); + new_client->type = LRMD_CLIENT_IPC; new_client->channel = c; new_client->id = crm_generate_uuid(); crm_trace("LRMD client connection established. %p id: %s", c, new_client->id); g_hash_table_insert(client_list, new_client->id, new_client); qb_ipcs_context_set(c, new_client); } static int32_t lrmd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size) { uint32_t id = 0; uint32_t flags = 0; xmlNode *request = crm_ipcs_recv(c, data, size, &id, &flags); lrmd_client_t *client = (lrmd_client_t *) qb_ipcs_context_get(c); CRM_CHECK(client != NULL, crm_err("Invalid client"); return FALSE); CRM_CHECK(client->id != NULL, crm_err("Invalid client: %p", client); return FALSE); CRM_CHECK(flags & crm_ipc_client_response, crm_err("Invalid client request: %p", client); return FALSE); - + if (!request) { return 0; } if (!client->name) { const char *value = crm_element_value(request, F_LRMD_CLIENTNAME); if (value == NULL) { client->name = crm_itoa(crm_ipcs_client_pid(c)); } else { client->name = strdup(value); } } - global_call_id++; - if (global_call_id < 1) { - global_call_id = 1; + lrmd_call_id++; + if (lrmd_call_id < 1) { + lrmd_call_id = 1; } crm_xml_add(request, F_LRMD_CLIENTID, client->id); crm_xml_add(request, F_LRMD_CLIENTNAME, client->name); - crm_xml_add_int(request, F_LRMD_CALLID, global_call_id); + crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id); process_lrmd_message(client, id, request); free_xml(request); return 0; } static int32_t lrmd_ipc_closed(qb_ipcs_connection_t * c) { lrmd_client_t *client = (lrmd_client_t *) qb_ipcs_context_get(c); int found = 0; if (!client) { crm_err("No client for ipc"); return 0; } if (client->id) { found = g_hash_table_remove(client_list, client->id); } if (!found) { crm_err("Asked to remove unknown client with id %d", client->id); } return 0; } static void lrmd_ipc_destroy(qb_ipcs_connection_t * c) { lrmd_client_t *client = (lrmd_client_t *) qb_ipcs_context_get(c); if (!client) { crm_err("No client for ipc"); return; } crm_info("LRMD client disconnecting %p - name: %s id: %s", c, client->name, client->id); client_disconnect_cleanup(client->id); qb_ipcs_context_set(c, NULL); free(client->name); free(client->id); free(client); } static struct qb_ipcs_service_handlers lrmd_ipc_callbacks = { .connection_accept = lrmd_ipc_accept, .connection_created = lrmd_ipc_created, .msg_process = lrmd_ipc_dispatch, .connection_closed = lrmd_ipc_closed, .connection_destroyed = lrmd_ipc_destroy }; +int lrmd_server_send_reply(lrmd_client_t *client, uint32_t id, xmlNode *reply) +{ + + crm_trace("sending reply to client (%s) with msg id %d", client->id, id); + switch (client->type) { + case LRMD_CLIENT_IPC: + return crm_ipcs_send(client->channel, id, reply, FALSE); + case LRMD_CLIENT_TLS: +#ifdef HAVE_GNUTLS_GNUTLS_H + return lrmd_tls_send_msg(client->session, reply, id, "reply"); +#endif + default: + crm_err("Unknown lrmd client type %d" , client->type); + } + return -1; +} + +int lrmd_server_send_notify(lrmd_client_t *client, xmlNode *msg) +{ + crm_trace("sending notify to client (%s)", client->id); + switch (client->type) { + case LRMD_CLIENT_IPC: + if (client->channel == NULL) { + crm_trace("Asked to send event to disconnected local client"); + return -1; + } + return crm_ipcs_send(client->channel, 0, msg, TRUE); + case LRMD_CLIENT_TLS: +#ifdef HAVE_GNUTLS_GNUTLS_H + if (client->session == NULL) { + crm_trace("Asked to send event to disconnected remote client"); + return -1; + } + return lrmd_tls_send_msg(client->session, msg, 0, "notify"); +#endif + default: + crm_err("Unknown lrmd client type %d" , client->type); + } + return -1; +} + void lrmd_shutdown(int nsig) { crm_info("Terminating with %d clients", g_hash_table_size(client_list)); if (ipcs) { mainloop_del_ipc_server(ipcs); } crm_exit(0); } /* *INDENT-OFF* */ static struct crm_option long_options[] = { /* Top-level Options */ - {"help", 0, 0, '?', "\tThis text"}, - {"version", 0, 0, '$', "\tVersion information" }, - {"verbose", 0, 0, 'V', "\tIncrease debug output"}, - - {"logfile", 1, 0, 'l', "\tSend logs to the additional named logfile"}, + {"help", 0, 0, '?', "\tThis text"}, + {"version", 0, 0, '$', "\tVersion information" }, + {"verbose", 0, 0, 'V', "\tIncrease debug output"}, + {"tls_enable", 0, 0, 't', "\tEnable TLS connection."}, + {"tls_port", 1, 0, 'p', "\tTLS port to listen to, defaults to 1984"}, + + {"logfile", 1, 0, 'l', "\tSend logs to the additional named logfile"}, {0, 0, 0, 0} }; /* *INDENT-ON* */ int main(int argc, char **argv) { int rc = 0; int flag = 0; int index = 0; crm_log_init("lrmd", LOG_INFO, TRUE, FALSE, argc, argv, FALSE); crm_set_options(NULL, "[options]", long_options, "Daemon for controlling services confirming to different standards"); while (1) { flag = crm_get_option(argc, argv, &index); if (flag == -1) { break; } switch (flag) { case 'l': crm_add_logfile(optarg); break; + case 't': + enable_remote = TRUE; + break; + case 'p': + remote_port = atoi(optarg); case 'V': set_crm_log_level(crm_log_level+1); break; case '?': case '$': crm_help(flag, EX_OK); break; default: crm_help('?', EX_USAGE); break; } } + if (enable_remote && !remote_port) { + remote_port = DEFAULT_REMOTE_PORT; + } + rsc_list = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_rsc); client_list = g_hash_table_new(crm_str_hash, g_str_equal); + ipcs = mainloop_add_ipc_server(CRM_SYSTEM_LRMD, QB_IPC_SHM, &lrmd_ipc_callbacks); if (ipcs == NULL) { crm_err("Failed to create IPC server: shutting down and inhibiting respawn"); crm_exit(100); } + if (enable_remote) { +#ifdef HAVE_GNUTLS_GNUTLS_H + if (lrmd_init_remote_tls_server(remote_port) < 0) { + crm_err("Failed to create TLS server: shutting down and inhibiting respawn"); + crm_exit(100); + } +#else + crm_err("GNUTLS not enabled in this build, can not establish remote server"); + crm_exit(100); +#endif + } + mainloop_add_signal(SIGTERM, lrmd_shutdown); mainloop = g_main_new(FALSE); crm_info("Starting"); g_main_run(mainloop); mainloop_del_ipc_server(ipcs); + if (enable_remote) { +#ifdef HAVE_GNUTLS_GNUTLS_H + lrmd_tls_server_destroy(); +#endif + } + g_hash_table_destroy(client_list); g_hash_table_destroy(rsc_list); if (stonith_api) { stonith_api->cmds->disconnect(stonith_api); stonith_api_delete(stonith_api); } return rc; } diff --git a/lrmd/tls_backend.c b/lrmd/tls_backend.c new file mode 100644 index 0000000000..edef5cc929 --- /dev/null +++ b/lrmd/tls_backend.c @@ -0,0 +1,350 @@ +/* + * Copyright (c) 2012 David Vossel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#ifdef HAVE_GNUTLS_GNUTLS_H +#define LRMD_REMOTE_AUTH_TIMEOUT 3000 +gnutls_psk_server_credentials_t psk_cred_s; +gnutls_dh_params_t dh_params; +static int ssock = 0; +extern int lrmd_call_id; + +static void +debug_log(int level, const char *str) +{ + fputs(str, stderr); +} + +static int +lrmd_remote_client_msg(gpointer data) +{ + int id = 0; + int rc = 0; + int disconnected = 0; + xmlNode *request = NULL; + lrmd_client_t *client = data; + + if (client->tls_handshake_complete == FALSE) { + int rc = 0; + + /* Muliple calls to handshake will be required, this callback + * will be invoked once the client sends more handshake data. */ + do { + rc = gnutls_handshake(*client->session); + + if (rc < 0 && rc != GNUTLS_E_AGAIN) { + crm_err("Remote lrmd tls handshake failed"); + return -1; + } + } while (rc == GNUTLS_E_INTERRUPTED); + + if (rc == 0) { + crm_debug("Remote lrmd tls handshake completed"); + client->tls_handshake_complete = TRUE; + if (client->remote_auth_timeout) { + g_source_remove(client->remote_auth_timeout); + } + client->remote_auth_timeout = 0; + } + return 0; + } + + rc = crm_recv_remote_ready(client->session, TRUE, 0); + if (rc == 0) { + /* no msg to read */ + return 0; + } else if (rc < 0) { + crm_info("Client disconnected during remote client read"); + return -1; + } + + crm_recv_remote_msg(client->session, &client->recv_buf, TRUE, -1, &disconnected); + + request = crm_parse_remote_buffer(&client->recv_buf); + while (request) { + crm_element_value_int(request, F_LRMD_REMOTE_MSG_ID, &id); + crm_trace("processing request from remote client with remote msg id %d", id); + if (!client->name) { + const char *value = crm_element_value(request, F_LRMD_CLIENTNAME); + if (value) { + client->name = strdup(value); + } + } + + lrmd_call_id++; + if (lrmd_call_id < 1) { + lrmd_call_id = 1; + } + + crm_xml_add(request, F_LRMD_CLIENTID, client->id); + crm_xml_add(request, F_LRMD_CLIENTNAME, client->name); + crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id); + + process_lrmd_message(client, id, request); + free_xml(request); + + /* process all the messages in the current buffer */ + request = crm_parse_remote_buffer(&client->recv_buf); + } + + if (disconnected) { + crm_info("Client disconnect detected in tls msg dispatcher."); + return -1; + } + + return 0; +} + +static void +lrmd_remote_client_destroy(gpointer user_data) +{ + lrmd_client_t *client = user_data; + + if (client == NULL) { + return; + } + + if (client->id) { + g_hash_table_remove(client_list, client->id); + } + + client_disconnect_cleanup(client->id); + + crm_notice("LRMD client disconnecting remote client - name: %s id: %s", + client->name ? client->name : "", + client->id); + + if (client->session) { + void *sock_ptr; + int csock; + sock_ptr = gnutls_transport_get_ptr(*client->session); + csock = GPOINTER_TO_INT(sock_ptr); + + gnutls_bye(*client->session, GNUTLS_SHUT_RDWR); + gnutls_deinit(*client->session); + gnutls_free(client->session); + close(csock); + } + + if (client->remote_auth_timeout) { + g_source_remove(client->remote_auth_timeout); + } + + free(client->recv_buf); + free(client->name); + free(client->id); + free(client); + + return; +} + +static gboolean +lrmd_auth_timeout_cb(gpointer data) +{ + lrmd_client_t *client = data; + + client->remote_auth_timeout = 0; + + if (client->tls_handshake_complete == TRUE) { + return FALSE; + } + + mainloop_del_fd(client->remote); + client->remote = NULL; + crm_err("Remote client authentication timed out"); + + return FALSE; +} + +static int +lrmd_remote_listen(gpointer data) +{ + int csock = 0; + int flag = 0; + unsigned laddr; + struct sockaddr_in addr; + gnutls_session *session = NULL; + lrmd_client_t *new_client = NULL; + + static struct mainloop_fd_callbacks lrmd_remote_fd_cb = + { + .dispatch = lrmd_remote_client_msg, + .destroy = lrmd_remote_client_destroy, + }; + + /* accept the connection */ + laddr = sizeof(addr); + csock = accept(ssock, (struct sockaddr *)&addr, &laddr); + crm_debug("New remote connection from %s", inet_ntoa(addr.sin_addr)); + + if (csock == -1) { + crm_err("accept socket failed"); + return TRUE; + } + + if ((flag = fcntl(csock, F_GETFL)) >= 0) { + if (fcntl(csock, F_SETFL, flag | O_NONBLOCK) < 0) { + crm_err( "fcntl() write failed"); + close(csock); + return TRUE; + } + } else { + crm_err( "fcntl() read failed"); + close(csock); + return TRUE; + } + + session = create_psk_tls_session(csock, GNUTLS_SERVER, psk_cred_s); + if (session == NULL) { + crm_err("TLS session creation failed"); + close(csock); + return TRUE; + } + + new_client = calloc(1, sizeof(lrmd_client_t)); + new_client->type = LRMD_CLIENT_TLS; + new_client->session = session; + new_client->id = crm_generate_uuid(); + new_client->remote_auth_timeout = g_timeout_add(LRMD_REMOTE_AUTH_TIMEOUT, lrmd_auth_timeout_cb, new_client); + crm_notice("LRMD client connection established. %p id: %s", new_client, new_client->id); + + new_client->remote = mainloop_add_fd("lrmd-remote-client", G_PRIORITY_DEFAULT, csock, new_client, &lrmd_remote_fd_cb); + g_hash_table_insert(client_list, new_client->id, new_client); + + return TRUE; +} + +static void +lrmd_remote_connection_destroy(gpointer user_data) +{ + crm_notice("Remote tls server disconnected"); + return; +} + +static int +lrmd_tls_server_key_cb(gnutls_session_t session, const char *username, gnutls_datum_t *key) +{ + int rc = 0; + + if (lrmd_tls_set_key(key, DEFAULT_REMOTE_KEY_LOCATION)) { + rc = lrmd_tls_set_key(key, ALT_REMOTE_KEY_LOCATION); + } + if (rc) { + crm_err("No lrmd remote key found"); + return -1; + } + + return rc; +} + +int +lrmd_init_remote_tls_server(int port) +{ + int rc; + struct sockaddr_in saddr; + int optval; + static struct mainloop_fd_callbacks remote_listen_fd_callbacks = + { + .dispatch = lrmd_remote_listen, + .destroy = lrmd_remote_connection_destroy, + }; + + crm_notice("Starting a tls listener on port %d.", port); + gnutls_global_init(); + gnutls_global_set_log_function(debug_log); + + gnutls_dh_params_init(&dh_params); + gnutls_dh_params_generate2(dh_params, 1024); + gnutls_psk_allocate_server_credentials(&psk_cred_s); + gnutls_psk_set_server_credentials_function(psk_cred_s, lrmd_tls_server_key_cb); + gnutls_psk_set_server_dh_params(psk_cred_s, dh_params); + + /* create server socket */ + ssock = socket(AF_INET, SOCK_STREAM, 0); + if (ssock == -1) { + crm_err("Can not create server socket."); + return -1; + } + + /* reuse address */ + optval = 1; + rc = setsockopt(ssock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); + if(rc < 0) { + crm_perror(LOG_INFO, "Couldn't allow the reuse of local addresses by our remote listener"); + } + + rc = -1; + + /* bind server socket */ + memset(&saddr, '\0', sizeof(saddr)); + saddr.sin_family = AF_INET; + saddr.sin_addr.s_addr = INADDR_ANY; + saddr.sin_port = htons(port); + if (bind(ssock, (struct sockaddr *)&saddr, sizeof(saddr)) == -1) { + crm_err("Can not bind server socket."); + goto init_remote_cleanup; + } + + if (listen(ssock, 10) == -1) { + crm_err("Can not start listen."); + goto init_remote_cleanup; + } + + mainloop_add_fd("lrmd-remote", G_PRIORITY_DEFAULT, ssock, NULL, &remote_listen_fd_callbacks); + + rc = ssock; +init_remote_cleanup: + if (rc < 0) { + close(ssock); + ssock = 0; + } + return rc; + +} + +void +lrmd_tls_server_destroy(void) +{ + if (psk_cred_s) { + gnutls_psk_free_server_credentials(psk_cred_s); + psk_cred_s = 0; + } + + if (ssock > 0) { + close(ssock); + ssock = 0; + } +} +#endif