diff --git a/extra/resources/docker-wrapper b/extra/resources/docker-wrapper
index f5ac276cc9..e1962f7608 100755
--- a/extra/resources/docker-wrapper
+++ b/extra/resources/docker-wrapper
@@ -1,516 +1,516 @@
#!/bin/bash
#
# Copyright (c) 2015 David Vossel <dvossel@redhat.com>
# All Rights Reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it would be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# Further, this software is distributed without any warranty that it is
# free of the rightful claim of any third person regarding infringement
# or the like. Any license provided herein, whether implied or
# otherwise, applies only to this software file. Patent licenses, if
# any, provided herein do not apply to combinations of this program with
# other software, or any other product whatsoever.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write the Free Software Foundation,
# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
#
#######################################################################
# Initialization:
: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
#######################################################################
CONF_PREFIX="pcmk_docker"
meta_data() {
cat <<END
<?xml version="1.0"?>
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
<resource-agent name="docker-wrapper" version="1.0">
<version>1.0</version>
<longdesc lang="en">
Docker technology wrapper for pacemaker remote.
</longdesc>
<shortdesc lang="en">docker wrapper</shortdesc>
<parameters>
<parameter name="${CONF_PREFIX}_image" required="1" >
<longdesc lang="en">
Docker image to run resources within
</longdesc>
<shortdesc lang="en">docker image</shortdesc>
<content type="string" default="" />
</parameter>
<parameter name="${CONF_PREFIX}_privileged" >
<longdesc lang="en">
Give resources within container access to cluster resources
such as the CIB and the ability to manage cluster attributes.
</longdesc>
<shortdesc lang="en">is privileged</shortdesc>
<content type="string" default="false" />
</parameter>
<parameter name="${CONF_PREFIX}_run_opts" required="0" unique="0">
<longdesc lang="en">
Add options to be appended to the 'docker run' command which is used
when creating the container during the start action. This option allows
users to do things such as setting a custom entry point and injecting
environment variables into the newly created container. Note the '-d'
option is supplied regardless of this value to force containers to run
in the background.
NOTE: Do not explicitly specify the --name argument in the run_opts. This
agent will set --name using the resource's instance name
</longdesc>
<shortdesc lang="en">run options</shortdesc>
<content type="string"/>
</parameter>
<parameter name="${CONF_PREFIX}_reuse" required="0" unique="0">
<longdesc lang="en">
Allow the container to be reused after stopping the container. By default
containers are removed after stop. With the reuse option containers
will persist after the container stops.
</longdesc>
<shortdesc lang="en">reuse container</shortdesc>
<content type="boolean"/>
</parameter>
</parameters>
<actions>
<action name="start" timeout="400" />
<action name="stop" timeout="120" />
<action name="monitor" timeout="60" interval="30" depth="0"/>
<action name="reload" timeout="120" />
<action name="promote" timeout="120s" />
<action name="demote" timeout="120s" />
<action name="notify" timeout="120" />
<action name="validate-all" timeout="30" />
<action name="meta-data" timeout="10" />
</actions>
</resource-agent>
END
}
#######################################################################
CLIENT="/usr/libexec/pacemaker/lrmd_internal_ctl"
DOCKER_AGENT="/usr/lib/ocf/resource.d/heartbeat/docker"
KEY_VAL_STR=""
PROVIDER=$OCF_RESKEY_CRM_meta_provider
CLASS=$OCF_RESKEY_CRM_meta_class
TYPE=$OCF_RESKEY_CRM_meta_type
CONTAINER=$OCF_RESKEY_CRM_meta_isolation_instance
if [ -z "$CONTAINER" ]; then
CONTAINER=$OCF_RESOURCE_INSTANCE
fi
RSC_STATE_DIR="${HA_RSCTMP}/docker-wrapper/${CONTAINER}-data/"
RSC_STATE_FILE="$RSC_STATE_DIR/$OCF_RESOURCE_INSTANCE.state"
CONNECTION_FAILURE=0
DOCKER_CLIENT="/usr/bin/docker"
pcmk_docker_wrapper_usage() {
cat <<END
usage: $0 {start|stop|monitor|reload|promote|demote|notify|validate-all|meta-data}
Expects to have a fully populated OCF RA-compliant environment set.
END
}
write_state_file()
{
if ! [ -f "$RSC_STATE_FILE" ]; then
[ -d "$RSC_STATE_DIR" ] || mkdir -p $RSC_STATE_DIR
echo "$OCF_RESOURCE_INSTANCE" > $RSC_STATE_FILE
fi
}
clear_state_file()
{
if [ -f "$RSC_STATE_FILE" ]; then
rm -f $RSC_STATE_FILE
fi
}
clear_state_dir()
{
[ -d "$RSC_STATE_DIR" ] || return 0
rm -rf $RSC_STATE_DIR
}
num_active_resources()
{
local count
[ -d "$RSC_STATE_DIR" ] || return 0
count="$(ls $RSC_STATE_DIR | wc -w)"
if [ $? -ne 0 ] || [ -z "$count" ]; then
return 0
fi
return $count
}
random_port()
{
local port=$(python -c 'import socket; s=socket.socket(); s.bind(("localhost", 0)); print(s.getsockname()[1]); s.close()')
if [ $? -eq 0 ] && [ -n "$port" ]; then
echo "$port"
fi
}
get_active_port()
{
PORT="$(docker port $CONTAINER 3121 | awk -F: '{ print $2 }')"
}
# separate docker args from ocf resource args.
separate_args()
{
local env key value
# write out arguments to key value string for ocf agent
while read -r line;
do
key="$(echo $line | awk -F= '{print $1}' | sed 's/^OCF_RESKEY_//g')"
val="$(echo $line | awk -F= '{print $2}')"
KEY_VAL_STR="$KEY_VAL_STR -k '$key' -v '$val'"
done < <(printenv | grep "^OCF.*" | grep -v "^OCF_RESKEY_${CONF_PREFIX}_.*")
# sanitize args for DOCKER agent's consumption
while read -r line;
do
env="$(echo $line | awk -F= '{print $1}')"
val="$(echo $line | awk -F= '{print $2}')"
key="$(echo "$env" | sed "s/^OCF_RESKEY_${CONF_PREFIX}/OCF_RESKEY/g")"
export $key="$val"
done < <(printenv | grep "^OCF_RESKEY_${CONF_PREFIX}_.*")
if ocf_is_true $OCF_RESKEY_privileged ; then
export OCF_RESKEY_run_cmd="/usr/sbin/pacemaker_remoted"
# on start set random port to run_opts
# write port to state file... or potentially get from ps? maybe docker info or inspect as well?
else
export OCF_RESKEY_run_cmd="/usr/libexec/pacemaker/lrmd"
fi
export OCF_RESKEY_name="$CONTAINER"
}
monitor_container()
{
local rc
$DOCKER_AGENT monitor
rc=$?
if [ $rc -ne $OCF_SUCCESS ]; then
clear_state_dir
return $rc
fi
poke_remote
rc=$?
if [ $rc -ne $OCF_SUCCESS ]; then
# container is up without an active daemon. this is bad
ocf_log err "Container, $CONTAINER, is active without a responsive pacemaker_remote instance"
CONNECTION_FAILURE=1
return $OCF_ERR_GENERIC
fi
CONNECTION_FAILURE=0
return $rc
}
pcmk_docker_wrapper_monitor() {
local rc
monitor_container
rc=$?
if [ $rc -ne $OCF_SUCCESS ]; then
return $rc
fi
client_action "monitor"
rc=$?
if [ $rc -eq $OCF_SUCCESS ] || [ $rc -eq $OCF_RUNNING_MASTER ]; then
write_state_file
else
clear_state_file
fi
return $rc
}
pcmk_docker_wrapper_generic_action()
{
local rc
monitor_container
rc=$?
if [ $? -ne $OCF_SUCCESS ]; then
return $rc
fi
client_action "$1"
}
client_action()
{
local action=$1
local agent_type="-T $TYPE -C $CLASS"
local rc=0
if [ -n "$PROVIDER" ]; then
agent_type="$agent_type -P $PROVIDER"
fi
if ocf_is_true $OCF_RESKEY_privileged ; then
if [ -z "$PORT" ]; then
get_active_port
fi
ocf_log info "$CLIENT -c 'exec' -S '127.0.0.1' -p '$PORT' -a '$action' -r '$OCF_RESOURCE_INSTANCE' -n '$CONTAINER' '$agent_type' $KEY_VAL_STR "
eval $CLIENT -c 'exec' -S '127.0.0.1' -p '$PORT' -a '$action' -r '$OCF_RESOURCE_INSTANCE' -n '$CONTAINER' '$agent_type' $KEY_VAL_STR
else
ocf_log info "$CLIENT -c \"exec\" -a $action -r \"$OCF_RESOURCE_INSTANCE\" $agent_type $KEY_VAL_STR"
- echo "$CLIENT -c \"exec\" -a $action -r \"$OCF_RESOURCE_INSTANCE\" $agent_type $KEY_VAL_STR " | nsenter --target $(docker inspect --format {{.State.Pid}} ${CONTAINER}) --mount --uts --ipc --net --pid 2>&1
+ echo "$CLIENT -c \"exec\" -a $action -r \"$OCF_RESOURCE_INSTANCE\" $agent_type $KEY_VAL_STR " | nsenter --target $(docker inspect --format {{.State.Pid}} ${CONTAINER}) --mount --uts --ipc --net --pid
fi
rc=$?
ocf_log debug "Client action $action with result $rc"
return $rc
}
poke_remote()
{
# verifies daemon in container is active
if ocf_is_true $OCF_RESKEY_privileged ; then
get_active_port
ocf_log info "Attempting to contect $CONTAINER on port $PORT"
$CLIENT -c "poke" -S "127.0.0.1" -p $PORT -n $CONTAINER
fi
# no op for non privileged containers since we handed the
# client monitor action as the monitor_cmd for the docker agent
}
start_container()
{
local rc
monitor_container
rc=$?
if [ $rc -eq $OCF_SUCCESS ]; then
return $rc
fi
export OCF_RESKEY_run_opts="-e PCMK_debug=yes -e PCMK_logfile=/var/log/pacemaker.log $OCF_RESKEY_run_opts"
if ocf_is_true $OCF_RESKEY_privileged ; then
if ! [ -f "/etc/pacemaker/authkey" ]; then
# generate an authkey if it doesn't exist.
mkdir -p /etc/pacemaker/
dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1 > /dev/null 2>&1
chmod 600 /etc/pacemaker/authkey
fi
PORT=$(random_port)
if [ -z "$PORT" ]; then
ocf_exit_reason "Unable to assign random port for pacemaker remote"
return $OCF_ERR_GENERIC
fi
export OCF_RESKEY_run_opts="-p 127.0.0.1:${PORT}:3121 $OCF_RESKEY_run_opts"
export OCF_RESKEY_run_opts="-v /etc/pacemaker/authkey:/etc/pacemaker/authkey $OCF_RESKEY_run_opts"
ocf_log debug "using privileged mode: run_opts=$OCF_RESKEY_run_opts"
else
export OCF_RESKEY_monitor_cmd="$CLIENT -c poke"
fi
$DOCKER_AGENT start
rc=$?
if [ $rc -ne $OCF_SUCCESS ]; then
docker ps > /dev/null 2>&1
if [ $? -ne 0 ]; then
ocf_exit_reason "docker daemon is inactive."
fi
return $rc
fi
monitor_container
}
pcmk_docker_wrapper_start() {
local rc
start_container
rc=$?
if [ $rc -ne $OCF_SUCCESS ]; then
return $rc
fi
client_action "start"
rc=$?
if [ $? -ne "$OCF_SUCCESS" ]; then
ocf_exit_reason "Failed to start agent within container"
return $rc
fi
pcmk_docker_wrapper_monitor
return $?
}
stop_container()
{
local rc
local count
num_active_resources
count=$?
if [ $count -ne 0 ]; then
ocf_log err "Failed to stop agent within container. Killing container $CONTAINER with $count active resources"
fi
$DOCKER_AGENT "stop"
rc=$?
if [ $rc -ne $OCF_SUCCESS ]; then
ocf_exit_reason "Docker container failed to stop"
return $rc
fi
clear_state_dir
return $rc
}
stop_resource()
{
local rc
client_action "stop"
rc=$?
if [ $? -ne "$OCF_SUCCESS" ]; then
export OCF_RESKEY_force_stop="true"
kill_now=1
else
clear_state_file
fi
}
pcmk_docker_wrapper_stop() {
local rc
local kill_now=0
local all_stopped=0
pcmk_docker_wrapper_monitor
rc=$?
if [ $rc -eq $OCF_NOT_RUNNING ]; then
rc=$OCF_SUCCESS
num_active_resources
if [ $? -eq 0 ]; then
# stop container if no more resources are running
ocf_log info "Gracefully stopping container $CONTAINER because no resources are left running."
stop_container
rc=$?
fi
return $rc
fi
# if we can't talk to the remote daemon but the container is
# active, we have to force kill the container.
if [ $CONNECTION_FAILURE -eq 1 ]; then
export OCF_RESKEY_force_kill="true"
stop_container
return $?
fi
# If we've gotten this far, the container is up, and we
# need to gracefully stop a resource within the container.
client_action "stop"
rc=$?
if [ $? -ne "$OCF_SUCCESS" ]; then
export OCF_RESKEY_force_stop="true"
# force kill the container if we fail to stop a resource.
stop_container
rc=$?
else
clear_state_file
num_active_resources
if [ $? -eq 0 ]; then
# stop container if no more resources are running
ocf_log info "Gracefully stopping container $CONTAINER because last resource has stopped"
stop_container
rc=$?
fi
fi
return $rc
}
pcmk_docker_wrapper_validate() {
check_binary docker
if [ -z "$CLASS" ] || [ -z "$TYPE" ]; then
ocf_exit_reason "Update pacemaker to a version that supports container wrappers."
return $OCF_ERR_CONFIGURED
fi
if ! [ -f "$DOCKER_AGENT" ]; then
ocf_exit_reason "Requires $DOCKER_AGENT to be installed. update the resource-agents package"
return $OCF_ERR_INSTALLED
fi
$DOCKER_AGENT validate-all
return $?
}
case $__OCF_ACTION in
meta-data) meta_data
exit $OCF_SUCCESS
;;
usage|help) pcmk_docker_wrapper_usage
exit $OCF_SUCCESS
;;
esac
separate_args
pcmk_docker_wrapper_validate
rc=$?
if [ $rc -ne 0 ]; then
case $__OCF_ACTION in
stop) exit $OCF_SUCCESS;;
monitor) exit $OCF_NOT_RUNNING;;
*) exit $rc;;
esac
fi
case $__OCF_ACTION in
start) pcmk_docker_wrapper_start;;
stop) pcmk_docker_wrapper_stop;;
monitor|status) pcmk_docker_wrapper_monitor;;
reload|promote|demote|notify)
pcmk_docker_wrapper_generic_action $__OCF_ACTION;;
validate-all) pcmk_docker_wrapper_validate;;
*) pcmk_docker_wrapper_usage
exit $OCF_ERR_UNIMPLEMENTED
;;
esac
rc=$?
ocf_log debug "Docker-wrapper ${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc"
exit $rc