diff --git a/extra/resources/docker-wrapper b/extra/resources/docker-wrapper index f5ac276cc9..e1962f7608 100755 --- a/extra/resources/docker-wrapper +++ b/extra/resources/docker-wrapper @@ -1,516 +1,516 @@ #!/bin/bash # # Copyright (c) 2015 David Vossel # All Rights Reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as # published by the Free Software Foundation. # # This program is distributed in the hope that it would be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # # Further, this software is distributed without any warranty that it is # free of the rightful claim of any third person regarding infringement # or the like. Any license provided herein, whether implied or # otherwise, applies only to this software file. Patent licenses, if # any, provided herein do not apply to combinations of this program with # other software, or any other product whatsoever. # # You should have received a copy of the GNU General Public License # along with this program; if not, write the Free Software Foundation, # Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. # ####################################################################### # Initialization: : ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs ####################################################################### CONF_PREFIX="pcmk_docker" meta_data() { cat < 1.0 Docker technology wrapper for pacemaker remote. docker wrapper Docker image to run resources within docker image Give resources within container access to cluster resources such as the CIB and the ability to manage cluster attributes. is privileged Add options to be appended to the 'docker run' command which is used when creating the container during the start action. This option allows users to do things such as setting a custom entry point and injecting environment variables into the newly created container. Note the '-d' option is supplied regardless of this value to force containers to run in the background. NOTE: Do not explicitly specify the --name argument in the run_opts. This agent will set --name using the resource's instance name run options Allow the container to be reused after stopping the container. By default containers are removed after stop. With the reuse option containers will persist after the container stops. reuse container END } ####################################################################### CLIENT="/usr/libexec/pacemaker/lrmd_internal_ctl" DOCKER_AGENT="/usr/lib/ocf/resource.d/heartbeat/docker" KEY_VAL_STR="" PROVIDER=$OCF_RESKEY_CRM_meta_provider CLASS=$OCF_RESKEY_CRM_meta_class TYPE=$OCF_RESKEY_CRM_meta_type CONTAINER=$OCF_RESKEY_CRM_meta_isolation_instance if [ -z "$CONTAINER" ]; then CONTAINER=$OCF_RESOURCE_INSTANCE fi RSC_STATE_DIR="${HA_RSCTMP}/docker-wrapper/${CONTAINER}-data/" RSC_STATE_FILE="$RSC_STATE_DIR/$OCF_RESOURCE_INSTANCE.state" CONNECTION_FAILURE=0 DOCKER_CLIENT="/usr/bin/docker" pcmk_docker_wrapper_usage() { cat < $RSC_STATE_FILE fi } clear_state_file() { if [ -f "$RSC_STATE_FILE" ]; then rm -f $RSC_STATE_FILE fi } clear_state_dir() { [ -d "$RSC_STATE_DIR" ] || return 0 rm -rf $RSC_STATE_DIR } num_active_resources() { local count [ -d "$RSC_STATE_DIR" ] || return 0 count="$(ls $RSC_STATE_DIR | wc -w)" if [ $? -ne 0 ] || [ -z "$count" ]; then return 0 fi return $count } random_port() { local port=$(python -c 'import socket; s=socket.socket(); s.bind(("localhost", 0)); print(s.getsockname()[1]); s.close()') if [ $? -eq 0 ] && [ -n "$port" ]; then echo "$port" fi } get_active_port() { PORT="$(docker port $CONTAINER 3121 | awk -F: '{ print $2 }')" } # separate docker args from ocf resource args. separate_args() { local env key value # write out arguments to key value string for ocf agent while read -r line; do key="$(echo $line | awk -F= '{print $1}' | sed 's/^OCF_RESKEY_//g')" val="$(echo $line | awk -F= '{print $2}')" KEY_VAL_STR="$KEY_VAL_STR -k '$key' -v '$val'" done < <(printenv | grep "^OCF.*" | grep -v "^OCF_RESKEY_${CONF_PREFIX}_.*") # sanitize args for DOCKER agent's consumption while read -r line; do env="$(echo $line | awk -F= '{print $1}')" val="$(echo $line | awk -F= '{print $2}')" key="$(echo "$env" | sed "s/^OCF_RESKEY_${CONF_PREFIX}/OCF_RESKEY/g")" export $key="$val" done < <(printenv | grep "^OCF_RESKEY_${CONF_PREFIX}_.*") if ocf_is_true $OCF_RESKEY_privileged ; then export OCF_RESKEY_run_cmd="/usr/sbin/pacemaker_remoted" # on start set random port to run_opts # write port to state file... or potentially get from ps? maybe docker info or inspect as well? else export OCF_RESKEY_run_cmd="/usr/libexec/pacemaker/lrmd" fi export OCF_RESKEY_name="$CONTAINER" } monitor_container() { local rc $DOCKER_AGENT monitor rc=$? if [ $rc -ne $OCF_SUCCESS ]; then clear_state_dir return $rc fi poke_remote rc=$? if [ $rc -ne $OCF_SUCCESS ]; then # container is up without an active daemon. this is bad ocf_log err "Container, $CONTAINER, is active without a responsive pacemaker_remote instance" CONNECTION_FAILURE=1 return $OCF_ERR_GENERIC fi CONNECTION_FAILURE=0 return $rc } pcmk_docker_wrapper_monitor() { local rc monitor_container rc=$? if [ $rc -ne $OCF_SUCCESS ]; then return $rc fi client_action "monitor" rc=$? if [ $rc -eq $OCF_SUCCESS ] || [ $rc -eq $OCF_RUNNING_MASTER ]; then write_state_file else clear_state_file fi return $rc } pcmk_docker_wrapper_generic_action() { local rc monitor_container rc=$? if [ $? -ne $OCF_SUCCESS ]; then return $rc fi client_action "$1" } client_action() { local action=$1 local agent_type="-T $TYPE -C $CLASS" local rc=0 if [ -n "$PROVIDER" ]; then agent_type="$agent_type -P $PROVIDER" fi if ocf_is_true $OCF_RESKEY_privileged ; then if [ -z "$PORT" ]; then get_active_port fi ocf_log info "$CLIENT -c 'exec' -S '127.0.0.1' -p '$PORT' -a '$action' -r '$OCF_RESOURCE_INSTANCE' -n '$CONTAINER' '$agent_type' $KEY_VAL_STR " eval $CLIENT -c 'exec' -S '127.0.0.1' -p '$PORT' -a '$action' -r '$OCF_RESOURCE_INSTANCE' -n '$CONTAINER' '$agent_type' $KEY_VAL_STR else ocf_log info "$CLIENT -c \"exec\" -a $action -r \"$OCF_RESOURCE_INSTANCE\" $agent_type $KEY_VAL_STR" - echo "$CLIENT -c \"exec\" -a $action -r \"$OCF_RESOURCE_INSTANCE\" $agent_type $KEY_VAL_STR " | nsenter --target $(docker inspect --format {{.State.Pid}} ${CONTAINER}) --mount --uts --ipc --net --pid 2>&1 + echo "$CLIENT -c \"exec\" -a $action -r \"$OCF_RESOURCE_INSTANCE\" $agent_type $KEY_VAL_STR " | nsenter --target $(docker inspect --format {{.State.Pid}} ${CONTAINER}) --mount --uts --ipc --net --pid fi rc=$? ocf_log debug "Client action $action with result $rc" return $rc } poke_remote() { # verifies daemon in container is active if ocf_is_true $OCF_RESKEY_privileged ; then get_active_port ocf_log info "Attempting to contect $CONTAINER on port $PORT" $CLIENT -c "poke" -S "127.0.0.1" -p $PORT -n $CONTAINER fi # no op for non privileged containers since we handed the # client monitor action as the monitor_cmd for the docker agent } start_container() { local rc monitor_container rc=$? if [ $rc -eq $OCF_SUCCESS ]; then return $rc fi export OCF_RESKEY_run_opts="-e PCMK_debug=yes -e PCMK_logfile=/var/log/pacemaker.log $OCF_RESKEY_run_opts" if ocf_is_true $OCF_RESKEY_privileged ; then if ! [ -f "/etc/pacemaker/authkey" ]; then # generate an authkey if it doesn't exist. mkdir -p /etc/pacemaker/ dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1 > /dev/null 2>&1 chmod 600 /etc/pacemaker/authkey fi PORT=$(random_port) if [ -z "$PORT" ]; then ocf_exit_reason "Unable to assign random port for pacemaker remote" return $OCF_ERR_GENERIC fi export OCF_RESKEY_run_opts="-p 127.0.0.1:${PORT}:3121 $OCF_RESKEY_run_opts" export OCF_RESKEY_run_opts="-v /etc/pacemaker/authkey:/etc/pacemaker/authkey $OCF_RESKEY_run_opts" ocf_log debug "using privileged mode: run_opts=$OCF_RESKEY_run_opts" else export OCF_RESKEY_monitor_cmd="$CLIENT -c poke" fi $DOCKER_AGENT start rc=$? if [ $rc -ne $OCF_SUCCESS ]; then docker ps > /dev/null 2>&1 if [ $? -ne 0 ]; then ocf_exit_reason "docker daemon is inactive." fi return $rc fi monitor_container } pcmk_docker_wrapper_start() { local rc start_container rc=$? if [ $rc -ne $OCF_SUCCESS ]; then return $rc fi client_action "start" rc=$? if [ $? -ne "$OCF_SUCCESS" ]; then ocf_exit_reason "Failed to start agent within container" return $rc fi pcmk_docker_wrapper_monitor return $? } stop_container() { local rc local count num_active_resources count=$? if [ $count -ne 0 ]; then ocf_log err "Failed to stop agent within container. Killing container $CONTAINER with $count active resources" fi $DOCKER_AGENT "stop" rc=$? if [ $rc -ne $OCF_SUCCESS ]; then ocf_exit_reason "Docker container failed to stop" return $rc fi clear_state_dir return $rc } stop_resource() { local rc client_action "stop" rc=$? if [ $? -ne "$OCF_SUCCESS" ]; then export OCF_RESKEY_force_stop="true" kill_now=1 else clear_state_file fi } pcmk_docker_wrapper_stop() { local rc local kill_now=0 local all_stopped=0 pcmk_docker_wrapper_monitor rc=$? if [ $rc -eq $OCF_NOT_RUNNING ]; then rc=$OCF_SUCCESS num_active_resources if [ $? -eq 0 ]; then # stop container if no more resources are running ocf_log info "Gracefully stopping container $CONTAINER because no resources are left running." stop_container rc=$? fi return $rc fi # if we can't talk to the remote daemon but the container is # active, we have to force kill the container. if [ $CONNECTION_FAILURE -eq 1 ]; then export OCF_RESKEY_force_kill="true" stop_container return $? fi # If we've gotten this far, the container is up, and we # need to gracefully stop a resource within the container. client_action "stop" rc=$? if [ $? -ne "$OCF_SUCCESS" ]; then export OCF_RESKEY_force_stop="true" # force kill the container if we fail to stop a resource. stop_container rc=$? else clear_state_file num_active_resources if [ $? -eq 0 ]; then # stop container if no more resources are running ocf_log info "Gracefully stopping container $CONTAINER because last resource has stopped" stop_container rc=$? fi fi return $rc } pcmk_docker_wrapper_validate() { check_binary docker if [ -z "$CLASS" ] || [ -z "$TYPE" ]; then ocf_exit_reason "Update pacemaker to a version that supports container wrappers." return $OCF_ERR_CONFIGURED fi if ! [ -f "$DOCKER_AGENT" ]; then ocf_exit_reason "Requires $DOCKER_AGENT to be installed. update the resource-agents package" return $OCF_ERR_INSTALLED fi $DOCKER_AGENT validate-all return $? } case $__OCF_ACTION in meta-data) meta_data exit $OCF_SUCCESS ;; usage|help) pcmk_docker_wrapper_usage exit $OCF_SUCCESS ;; esac separate_args pcmk_docker_wrapper_validate rc=$? if [ $rc -ne 0 ]; then case $__OCF_ACTION in stop) exit $OCF_SUCCESS;; monitor) exit $OCF_NOT_RUNNING;; *) exit $rc;; esac fi case $__OCF_ACTION in start) pcmk_docker_wrapper_start;; stop) pcmk_docker_wrapper_stop;; monitor|status) pcmk_docker_wrapper_monitor;; reload|promote|demote|notify) pcmk_docker_wrapper_generic_action $__OCF_ACTION;; validate-all) pcmk_docker_wrapper_validate;; *) pcmk_docker_wrapper_usage exit $OCF_ERR_UNIMPLEMENTED ;; esac rc=$? ocf_log debug "Docker-wrapper ${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc" exit $rc