diff --git a/extra/resources/Makefile.am b/extra/resources/Makefile.am index cc162e5e6e..955e233a1b 100644 --- a/extra/resources/Makefile.am +++ b/extra/resources/Makefile.am @@ -1,51 +1,56 @@ # Makefile.am for OCF RAs # # Author: Andrew Beekhof # Copyright (C) 2008 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # include $(top_srcdir)/Makefile.common EXTRA_DIST = $(ocf_SCRIPTS) + +containertechdir = @OCF_RA_DIR@/containers + ocfdir = @OCF_RA_DIR@/pacemaker ocf_SCRIPTS = ClusterMon \ controld \ Dummy \ HealthCPU \ HealthSMART \ o2cb \ ping \ pingd \ Stateful \ SysInfo \ SystemHealth \ remote +containertech_SCRIPTS = docker-wrapper + if BUILD_XML_HELP man7_MANS = $(ocf_SCRIPTS:%=ocf_pacemaker_%.7) DBOOK_OPTS = --stringparam command.prefix ocf_pacemaker_ --stringparam variable.prefix OCF_RESKEY_ --param man.vol 7 ocf_pacemaker_%.xml: % $(AM_V_GEN)OCF_FUNCTIONS=/dev/null OCF_ROOT=$(OCF_ROOT_DIR) sh $(abs_builddir)/$< meta-data > $@ endif clean-generic: rm -f $(man7_MANS) $(ocf_SCRIPTS:%=%.xml) *~ diff --git a/extra/resources/docker-wrapper b/extra/resources/docker-wrapper new file mode 100755 index 0000000000..c67022ff2b --- /dev/null +++ b/extra/resources/docker-wrapper @@ -0,0 +1,492 @@ +#!/bin/sh +# +# Copyright (c) 2015 David Vossel +# All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# + +####################################################################### +# Initialization: + +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +####################################################################### + +meta_data() { + cat < + + +1.0 + + +Docker technology wrapper for pacemaker remote. + +docker wrapper + + + + +Docker image to run resources within + +docker image + + + + + +Give resources within container access to cluster resources +such as the CIB and the ability to manage cluster attributes. + +is privileged + + + + + +Add options to be appended to the 'docker run' command which is used +when creating the container during the start action. This option allows +users to do things such as setting a custom entry point and injecting +environment variables into the newly created container. Note the '-d' +option is supplied regardless of this value to force containers to run +in the background. + +NOTE: Do not explicitly specify the --name argument in the run_opts. This +agent will set --name using the resource's instance name + + +run options + + + + + + + + + + + + + + +END +} + +####################################################################### + + +CLIENT="/usr/libexec/pacemaker/pacemaker_remote_ctl" +DOCKER_AGENT="/usr/lib/ocf/resource.d/heartbeat/docker" +KEY_VAL_STR="" +PROVIDER=$OCF_RESKEY_CRM_meta_provider +CLASS=$OCF_RESKEY_CRM_meta_class +TYPE=$OCF_RESKEY_CRM_meta_type + +CONTAINER=$OCF_RESKEY_CRM_meta_isolation_instance +if [ -z "$CONTAINER" ]; then + CONTAINER=$OCF_RESOURCE_INSTANCE +fi + +RSC_STATE_DIR="${HA_RSCTMP}/docker-wrapper/${CONTAINER}-data/" +RSC_STATE_FILE="$RSC_STATE_DIR/$OCF_RESOURCE_INSTANCE.state" +CONNECTION_FAILURE=0 + +pcmk_docker_wrapper_usage() { + cat < $RSC_STATE_FILE + fi +} + +clear_state_file() +{ + if [ -f "$RSC_STATE_FILE" ]; then + rm -f $RSC_STATE_FILE + fi +} + +clear_state_dir() +{ + [ -d "$RSC_STATE_DIR" ] || return 0 + + rm -rf $RSC_STATE_DIR +} + +num_active_resources() +{ + local count + + [ -d "$RSC_STATE_DIR" ] || return 0 + + count="$(ls $RSC_STATE_DIR | wc -w)" + if [ $? -ne 0 ] || [ -z "$count" ]; then + return 0 + fi + return $count +} + +random_port() +{ + local port=$(python -c 'import socket; s=socket.socket(); s.bind(("localhost", 0)); print(s.getsockname()[1]); s.close()') + if [ $? -eq 0 ] && [ -n "$port" ]; then + echo "$port" + fi +} + +get_active_port() +{ + PORT="$(docker port $CONTAINER 3121 | awk -F: '{ print $2 }')" +} + +# separate docker args from ocf resource args. +separate_args() +{ + local env key value + + # write out arguments to key value string for ocf agent + for item in $(printenv | grep "^OCF.*" | grep -v "^OCF_RESKEY_pcmk_docker_.*"); + do + key="$(echo $item | awk -F= '{print $1}')" + val="$(echo $item | awk -F= '{print $2}')" + KEY_VAL_STR="$KEY_VAL_STR -k \"$key\" -v \"$val\"" + done + + # sanitize args for DOCKER agent's consumption + for item in $(printenv | grep "^OCF_RESKEY_pcmk_docker_.*"); + do + env="$(echo $item | awk -F= '{print $1}')" + val="$(echo $item | awk -F= '{print $2}')" + key="$(echo "$env" | sed 's/^OCF_RESKEY_pcmk_docker/OCF_RESKEY/g')" + export ${key}=$(echo $val) + done + + if ocf_is_true $OCF_RESKEY_pcmk_docker_privileged ; then + export OCF_RESKEY_run_cmd="/usr/sbin/pacemaker_remoted" + # on start set random port to run_opts + # write port to state file... or potentially get from ps? maybe docker info or inspect as well? + + else + export OCF_RESKEY_run_cmd="/usr/libexec/pacemaker/lrmd" + fi + export OCF_RESKEY_name="$CONTAINER" +} + +monitor_container() +{ + local rc + + $DOCKER_AGENT monitor + rc=$? + if [ $rc -ne $OCF_SUCCESS ]; then + clear_state_dir + return $rc + fi + + poke_remote + rc=$? + if [ $rc -ne $OCF_SUCCESS ]; then + # container is up without an active daemon. this is bad + ocf_log err "Container, $CONTAINER, is active without a responsive pacemaker_remote instance" + CONNECTION_FAILURE=1 + return $OCF_ERR_GENERIC + fi + CONNECTION_FAILURE=0 + + return $rc +} + +pcmk_docker_wrapper_monitor() { + local rc + + monitor_container + rc=$? + if [ $rc -ne $OCF_SUCCESS ]; then + return $rc + fi + + client_action "monitor" + rc=$? + if [ $rc -eq $OCF_SUCCESS ]; then + write_state_file + elif [ $rc -eq $OCF_NOT_RUNNING ]; then + clear_state_file + fi + + return $rc +} + +client_action() +{ + local action=$1 + local agent_type="-T $TYPE -C $CLASS" + local rc=0 + + if [ -n "$PROVIDER" ]; then + agent_type="$agent_type -P $PROVIDER" + fi + + if ocf_is_true $OCF_RESKEY_pcmk_docker_privileged ; then + if [ -z "$PORT" ]; then + get_active_port + fi + ocf_log debug "$CLIENT -c \"exec\" -S \"127.0.0.1\" -p $PORT -a $action -r \"$OCF_RESOURCE_INSTANCE\" $agent_type $KEY_VAL_STR >/dev/null 2>&1" + $CLIENT -c "exec" -S "127.0.0.1" -p $PORT -a $action -r "$OCF_RESOURCE_INSTANCE" $agent_type $KEY_VAL_STR >/dev/null 2>&1 + else + echo "$CLIENT -c \"exec\" -a $action -r \"$OCF_RESOURCE_INSTANCE\" $agent_type $KEY_VAL_STR >/dev/null 2>&1" | nsenter --target $(docker inspect --format {{.State.Pid}} ${CONTAINER}) --mount --uts --ipc --net --pid 2>&1 + + + fi + rc=$? + + ocf_log debug "Client action $action with result $rc" + return $rc +} + +poke_remote() +{ + # verifies daemon in container is active + if ocf_is_true $OCF_RESKEY_pcmk_docker_privileged ; then + get_active_port + ocf_log debug "Attempting to contect $CONTAINER on port $PORT" + $CLIENT -c "poke" -S "127.0.0.1" -p $PORT >/dev/null 2>&1 + fi + # no op for non privileged containers since we handed the + # client monitor action as the monitor_cmd for the docker agent +} + +pcmk_docker_wrapper_reload() +{ + local rc + + monitor_container + rc=$? + if [ $? -ne $OCF_SUCCESS ]; then + return $rc + fi + + client_action "reload" +} + +start_container() +{ + local rc + + monitor_container + rc=$? + if [ $rc -eq $OCF_SUCCESS ]; then + return $rc + fi + + if ocf_is_true $OCF_RESKEY_pcmk_docker_privileged ; then + if ! [ -f "/etc/pacemaker/authkey" ]; then + # generate an authkey if it doesn't exist. + mkdir -p /etc/pacemaker/ + dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1 > /dev/null 2>&1 + fi + + PORT=$(random_port) + if [ -z "$PORT" ]; then + ocf_exit_reason "Unable to assign random port for pacemaker remote" + return $OCF_ERR_GENERIC + fi + export OCF_RESKEY_run_opts="-p 127.0.0.1:${PORT}:3121 $OCF_RESKEY_run_opts" + export OCF_RESKEY_run_opts="-v /etc/pacemaker/authkey:/etc/pacemaker/authkey $OCF_RESKEY_run_opts" + ocf_log debug "using privileged mode: run_opts=$OCF_RESKEY_run_opts" + else + export OCF_RESKEY_monitor_cmd="$CLIENT -c poke" + fi + + $DOCKER_AGENT start + rc=$? + if [ $rc -ne $OCF_SUCCESS ]; then + ocf_exit_reason "Docker container failed to start" + return $rc + fi + + monitor_container +} + +pcmk_docker_wrapper_start() { + local rc + + start_container + rc=$? + if [ $rc -ne $OCF_SUCCESS ]; then + return $rc + fi + + client_action "start" + rc=$? + if [ $? -ne "$OCF_SUCCESS" ]; then + ocf_exit_reason "Failed to start agent within container" + return $rc + fi + + pcmk_docker_wrapper_monitor + return $? +} + +stop_container() +{ + local rc + local count + + num_active_resources + count=$? + if [ $count -ne 0 ]; then + ocf_log err "Failed to stop agent within container. Killing container $CONTAINER with $count active resources" + fi + + $DOCKER_AGENT "stop" + rc=$? + if [ $rc -ne $OCF_SUCCESS ]; then + ocf_exit_reason "Docker container failed to stop" + return $rc + fi + clear_state_dir + return $rc +} + +stop_resource() +{ + local rc + + client_action "stop" + rc=$? + if [ $? -ne "$OCF_SUCCESS" ]; then + export OCF_RESKEY_force_stop="true" + kill_now=1 + else + clear_state_file + fi +} + +pcmk_docker_wrapper_stop() { + local rc + local kill_now=0 + local all_stopped=0 + + pcmk_docker_wrapper_monitor + rc=$? + if [ $rc -eq $OCF_NOT_RUNNING ]; then + rc=$OCF_SUCCESS + num_active_resources + if [ $? -eq 0 ]; then + # stop container if no more resources are running + ocf_log info "Gracefully stopping container $CONTAINER because no resources are left running." + stop_container + rc=$? + fi + return $rc + fi + + # if we can't talk to the remote daemon but the container is + # active, we have to force kill the container. + if [ $CONNECTION_FAILURE -eq 1 ]; then + export OCF_RESKEY_force_kill="true" + stop_container + return $? + fi + + + # If we've gotten this far, the container is up, and we + # need to gracefully stop a resource within the container. + client_action "stop" + rc=$? + if [ $? -ne "$OCF_SUCCESS" ]; then + export OCF_RESKEY_force_stop="true" + # force kill the container if we fail to stop a resource. + stop_container + rc=$? + else + clear_state_file + num_active_resources + if [ $? -eq 0 ]; then + # stop container if no more resources are running + ocf_log info "Gracefully stopping container $CONTAINER because last resource has stopped" + stop_container + rc=$? + fi + fi + + return $rc +} + +pcmk_docker_wrapper_validate() { + + if [ -z "$CLASS" ] || [ -z "$TYPE" ]; then + ocf_exit_reason "Update pacemaker to a version that supports container wrappers." + return $OCF_ERR_CONFIGURED + fi + + if ! [ -f "$DOCKER_AGENT" ]; then + ocf_exit_reason "Requires $DOCKER_AGENT to be installed. update the resource-agents package" + return $OCF_ERR_INSTALLED + fi + $DOCKER_AGENT validate-all + return $? +} + +case $__OCF_ACTION in +meta-data) meta_data + exit $OCF_SUCCESS + ;; +usage|help) pcmk_docker_wrapper_usage + exit $OCF_SUCCESS + ;; +esac + +separate_args +pcmk_docker_wrapper_validate +rc=$? +if [ $rc -ne 0 ]; then + case $__OCF_ACTION in + stop) exit $OCF_SUCCESS;; + monitor) exit $OCF_NOT_RUNNING;; + *) exit $rc;; + esac +fi + +case $__OCF_ACTION in + start) pcmk_docker_wrapper_start;; + stop) pcmk_docker_wrapper_stop;; + monitor) pcmk_docker_wrapper_monitor;; + reload) pcmk_docker_wrapper_reload;; + validate-all) pcmk_docker_wrapper_validate;; + *) pcmk_docker_wrapper_usage + exit $OCF_ERR_UNIMPLEMENTED + ;; +esac +rc=$? +ocf_log debug "Docker-wrapper ${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc" +exit $rc +