diff --git a/extra/ansible/docker/group_vars/all b/extra/ansible/docker/group_vars/all new file mode 100644 index 0000000000..935e88a8bf --- /dev/null +++ b/extra/ansible/docker/group_vars/all @@ -0,0 +1,5 @@ +max: 4 +prefix: ansible-pcmk +base_image: centos:centos7 +subnet: 172.17.200 +pacemaker_authkey: this_is_very_insecure \ No newline at end of file diff --git a/extra/ansible/docker/hosts b/extra/ansible/docker/hosts new file mode 100644 index 0000000000..5b0fb7103e --- /dev/null +++ b/extra/ansible/docker/hosts @@ -0,0 +1,7 @@ +[controllers] +oss-uk-1.clusterlabs.org + +[containers] +ansible-1 +ansible-2 +ansible-3 diff --git a/extra/ansible/docker/roles/docker-host/files/docker-enter b/extra/ansible/docker/roles/docker-host/files/docker-enter new file mode 100644 index 0000000000..04c48223f5 --- /dev/null +++ b/extra/ansible/docker/roles/docker-host/files/docker-enter @@ -0,0 +1,29 @@ +#! /bin/sh -e + +case "$1" in + -h|--help) + echo "Usage: docker-enter CONTAINER [COMMAND]" + exit 0 + ;; +esac + +if [ $(id -ru) -ne 0 ]; then + echo "You have to be root." + exit 1 +fi + +if [ $# -eq 0 ]; then + echo "Usage: docker-enter CONTAINER [COMMAND]" + exit 1 +fi + +container=$1; shift +PID=$(docker inspect --format {{.State.Pid}} "$container") + +if [ $# -ne 0 ]; then + nsenter --target $PID --mount --uts --ipc --net --pid -- $* + exit $? +fi + +nsenter --target $PID --mount --uts --ipc --net --pid +exit 0 diff --git a/extra/ansible/docker/roles/docker-host/files/fence_docker_cts b/extra/ansible/docker/roles/docker-host/files/fence_docker_cts new file mode 100644 index 0000000000..6d6f025145 --- /dev/null +++ b/extra/ansible/docker/roles/docker-host/files/fence_docker_cts @@ -0,0 +1,202 @@ +#!/bin/bash +# +# Copyright (c) 2014 David Vossel +# All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# +####################################################################### + +port="" +action="list" # Default fence action + +function usage() +{ +cat < + + + fence_docker_cts fences docker containers for testing purposes. + + + + + + Fencing Action + + + + + The name/id of docker container to control/check + + + + + + + + + + + + +EOF + exit 0; +} + +function docker_log() { + if ! [ "$action" = "list" ]; then + printf "$*\n" 1>&2 + fi +} + +# stdin option processing +if [ -z $1 ]; then + # If there are no command line args, look for options from stdin + while read line; do + for word in $(echo "$line"); do + case $word in + option=*|action=*) action=`echo $word | sed s/.*=//`;; + port=*) port=`echo $word | sed s/.*=//`;; + node=*) port=`echo $word | sed s/.*=//`;; + nodename=*) port=`echo $word | sed s/.*=//`;; + --);; + *) docker_log "Invalid command: $word";; + esac + done + done +fi + +# Command line option processing +while true ; do + if [ -z "$1" ]; then + break; + fi + case "$1" in + -o|--action|--option) action=$2; shift; shift;; + -n|--port) port=$2; shift; shift;; + -V|--version) echo "1.0.0"; exit 0;; + --help|-h) + usage; + exit 0;; + --) shift ; break ;; + *) docker_log "Unknown option: $1. See --help for details."; exit 1;; + esac +done + +action=`echo $action | tr 'A-Z' 'a-z'` +case $action in + hostlist|list) action=list;; + stat|status) action=status;; + restart|reboot|reset) action=reboot;; + poweron|on) action=start;; + poweroff|off) action=stop;; +esac + +function fence_done() +{ + if [ $1 -eq 0 ]; then + docker_log "Operation $action (port=$port) passed" + else + docker_log "Operation $action (port=$port) failed: $1" + fi + if [ -z "$returnfile" ]; then + rm -f $returnfile + fi + if [ -z "$helperscript" ]; then + rm -f $helperscript + fi + exit $1 +} + +case $action in + metadata) metadata;; +esac + +returnfile=$(mktemp /tmp/fence_docker_cts_returnfileXXXX) +returnstring="" +helper_script=$(mktemp /tmp/fence_docker_cts_helperXXXX) + +exec_action() +{ + echo "#!/bin/bash" > $helper_script + echo "sleep 10000" >> $helper_script + chmod 755 $helper_script + src="$(uname -n)" + + $helper_script "$src" "$action" "$returnfile" "$port" > /dev/null 2>&1 & + pid=$! + docker_log "waiting on pid $pid" + wait $pid > /dev/null 2>&1 + returnstring=$(cat $returnfile) + + if [ -z "$returnstring" ]; then + docker_log "fencing daemon did not respond" + fence_done 1 + fi + + if [ "$returnstring" == "fail" ]; then + docker_log "fencing daemon failed to execute action [$action on port $port]" + fence_done 1 + fi + + return 0 +} + +exec_action +case $action in + list) + cat $returnfile + fence_done 0 + ;; + + status) + # 0 if container is on + # 1 if container can not be contacted or unknown + # 2 if container is off + if [ "$returnstring" = "true" ]; then + fence_done 0 + else + fence_done 2 + fi + ;; + monitor|stop|start|reboot) : ;; + *) docker_log "Unknown action: $action"; fence_done 1;; +esac + +fence_done $? diff --git a/extra/ansible/docker/roles/docker-host/files/launch.sh b/extra/ansible/docker/roles/docker-host/files/launch.sh new file mode 100644 index 0000000000..66bebf48e1 --- /dev/null +++ b/extra/ansible/docker/roles/docker-host/files/launch.sh @@ -0,0 +1,4 @@ +#!/bin/bash +while true; do + sleep 1 +done diff --git a/extra/ansible/docker/roles/docker-host/files/pcmk_remote_start b/extra/ansible/docker/roles/docker-host/files/pcmk_remote_start new file mode 100644 index 0000000000..1bf032003a --- /dev/null +++ b/extra/ansible/docker/roles/docker-host/files/pcmk_remote_start @@ -0,0 +1,18 @@ +#!/bin/bash +/usr/sbin/ip_start +pid=$(pidof pacemaker_remoted) +if [ "$?" -ne 0 ]; then + mkdir -p /var/run + + export PCMK_debugfile=$pcmklogs + (pacemaker_remoted &) & > /dev/null 2>&1 + sleep 5 + + pid=$(pidof pacemaker_remoted) + if [ "$?" -ne 0 ]; then + echo "startup of pacemaker failed" + exit 1 + fi + echo "$pid" > /var/run/pacemaker_remoted.pid +fi +exit 0 diff --git a/extra/ansible/docker/roles/docker-host/files/pcmk_remote_stop b/extra/ansible/docker/roles/docker-host/files/pcmk_remote_stop new file mode 100644 index 0000000000..074cd598aa --- /dev/null +++ b/extra/ansible/docker/roles/docker-host/files/pcmk_remote_stop @@ -0,0 +1,36 @@ +#!/bin/bash +status() +{ + pid=$(pidof $1 2>/dev/null) + rtrn=$? + if [ $rtrn -ne 0 ]; then + echo "$1 is stopped" + else + echo "$1 (pid $pid) is running..." + fi + return $rtrn +} +stop() +{ + desc="Pacemaker Remote" + prog=$1 + shutdown_prog=$prog + + if status $shutdown_prog > /dev/null 2>&1; then + kill -TERM $(pidof $prog) > /dev/null 2>&1 + + while status $prog > /dev/null 2>&1; do + sleep 1 + echo -n "." + done + else + echo -n "$desc is already stopped" + fi + + rm -f /var/lock/subsystem/pacemaker + rm -f /var/run/${prog}.pid + killall -q -9 'crmd stonithd attrd cib lrmd pacemakerd pacemaker_remoted' +} + +stop "pacemaker_remoted" +exit 0 diff --git a/extra/ansible/docker/roles/docker-host/files/pcmk_start b/extra/ansible/docker/roles/docker-host/files/pcmk_start new file mode 100644 index 0000000000..d8b2ba8989 --- /dev/null +++ b/extra/ansible/docker/roles/docker-host/files/pcmk_start @@ -0,0 +1,23 @@ +#!/bin/bash + +/usr/sbin/ip_start +sed -i 's@to_syslog:.*yes@to_logfile: yes\nlogfile: /var/log/pacemaker.log@g' /etc/corosync/corosync.conf + +/usr/share/corosync/corosync start > /dev/null 2>&1 + +pid=$(pidof pacemakerd) +if [ "$?" -ne 0 ]; then + mkdir -p /var/run + + export PCMK_debugfile=$pcmklogs + (pacemakerd &) & > /dev/null 2>&1 + sleep 5 + + pid=$(pidof pacemakerd) + if [ "$?" -ne 0 ]; then + echo "startup of pacemaker failed" + exit 1 + fi + echo "$pid" > /var/run/pacemakerd.pid +fi +exit 0 diff --git a/extra/ansible/docker/roles/docker-host/files/pcmk_stop b/extra/ansible/docker/roles/docker-host/files/pcmk_stop new file mode 100644 index 0000000000..a8f395ad47 --- /dev/null +++ b/extra/ansible/docker/roles/docker-host/files/pcmk_stop @@ -0,0 +1,45 @@ +#!/bin/bash +status() +{ + pid=$(pidof $1 2>/dev/null) + rtrn=$? + if [ $rtrn -ne 0 ]; then + echo "$1 is stopped" + else + echo "$1 (pid $pid) is running..." + fi + return $rtrn +} +stop() +{ + desc="Pacemaker Cluster Manager" + prog=$1 + shutdown_prog=$prog + + if ! status $prog > /dev/null 2>&1; then + shutdown_prog="crmd" + fi + + cname=$(crm_node --name) + crm_attribute -N $cname -n standby -v true -l reboot + + if status $shutdown_prog > /dev/null 2>&1; then + kill -TERM $(pidof $prog) > /dev/null 2>&1 + + while status $prog > /dev/null 2>&1; do + sleep 1 + echo -n "." + done + else + echo -n "$desc is already stopped" + fi + + rm -f /var/lock/subsystem/pacemaker + rm -f /var/run/${prog}.pid + killall -q -9 'crmd stonithd attrd cib lrmd pacemakerd pacemaker_remoted' +} + +stop "pacemakerd" +/usr/share/corosync/corosync stop > /dev/null 2>&1 +killall -q -9 'corosync' +exit 0 diff --git a/extra/ansible/docker/roles/docker-host/tasks/main.yml b/extra/ansible/docker/roles/docker-host/tasks/main.yml new file mode 100644 index 0000000000..ce69adf8d5 --- /dev/null +++ b/extra/ansible/docker/roles/docker-host/tasks/main.yml @@ -0,0 +1,77 @@ +--- +#local_action: command /usr/bin/take_out_of_pool {{ inventory_hostname }} +- name: Update docker + yum: pkg=docker state=latest +- name: Start docker + service: name=docker state=started enabled=yes +- name: Install helper + copy: src=docker-enter dest=/usr/sbin/ mode=0755 +- name: Download image + shell: docker pull {{ base_image }} +- name: Cleanup kill + shell: docker kill $(docker ps -a | grep {{ prefix }} | awk '{print $1}') || echo "Nothing to kill" +- name: Cleanup remove + shell: docker rm $(docker ps -a | grep {{ prefix }} | awk '{print $1}') || echo "Nothing to remove" +- name: Cleanup docker skeleton + file: path={{ prefix }} state=absent +- name: Create docker skeleton + file: path={{ prefix }}/{{ item }} state=directory recurse=yes + with_items: + - rpms + - repos + - bin_files + - launch_scripts +- name: Create IP helper + template: src=ip_start.j2 dest={{ prefix }}/bin_files/ip_start mode=0755 +- name: Copy helper scripts + copy: src={{ item }} dest={{ prefix }}/bin_files/{{ item }} mode=0755 + with_items: + - pcmk_stop + - pcmk_start + - pcmk_remote_stop + - pcmk_remote_start + - fence_docker_cts +- name: Copy launch script + copy: src=launch.sh dest={{ prefix }}/launch_scripts/launch.sh mode=0755 +- name: Copy authorized keys + shell: cp /root/.ssh/authorized_keys {{ prefix }} +- name: Create docker file + template: src=Dockerfile.j2 dest={{ prefix }}/Dockerfile +- name: Making image + shell: docker build -t {{ prefix }} {{ prefix }} +- name: Launch images + shell: docker run -d -i -t -P -h {{ prefix }}-{{ item }} --name={{ prefix }}-{{ item }} -p 2200{{ item }}:22 $(docker images | grep {{ prefix }}.*latest | awk '{print $3}') /bin/bash + with_sequence: count={{ max }} +- name: Calculate IPs + shell: for n in $(seq {{ max }} ); do echo {{ subnet }}.${n}; done | tr '\n' ' ' + register: node_ips +- name: Start the IP + shell: docker-enter {{ prefix }}-{{ item }} ip_start + with_sequence: count={{ max }} +- name: Configure cluster + shell: docker-enter {{ prefix }}-{{ item }} pcs cluster setup --local --name {{ prefix }} {{ node_ips.stdout }} + with_sequence: count={{ max }} +- name: Start the cluster + shell: docker-enter {{ prefix }}-{{ item }} pcmk_start + with_sequence: count={{ max }} +- name: Set cluster options + shell: docker-enter {{ prefix }}-1 pcs property set stonith-enabled=false +- name: Configure VIP + shell: docker-enter {{ prefix }}-1 pcs resource create ClusterIP ocf:heartbeat:IPaddr2 ip={{ subnet }}.100 cidr_netmask=32 op monitor interval=30s +- name: Configure + shell: docker-enter {{ prefix }}-1 pcs resource defaults resource-stickiness=100 +- name: Configure + shell: docker-enter {{ prefix }}-1 pcs resource create WebSite apache configfile=/etc/httpd/conf/httpd.conf statusurl="http://localhost/server-status" op monitor interval=1min +- name: Configure + shell: docker-enter {{ prefix }}-1 pcs constraint colocation add WebSite with ClusterIP INFINITY +- name: Configure + shell: docker-enter {{ prefix }}-1 pcs constraint order ClusterIP then WebSite +- name: Configure + shell: docker-enter {{ prefix }}-1 pcs constraint location WebSite prefers {{ prefix }}-1=50 +# TODO: Enable fencing +# TODO: Make this a full LAMP stack similar to https://github.com/ansible/ansible-examples/tree/master/lamp_simple +# TODO: Create a Pacemaker module? + +# run_once: true +# delegate_to: web01.example.org + diff --git a/extra/ansible/docker/roles/docker-host/templates/Dockerfile.j2 b/extra/ansible/docker/roles/docker-host/templates/Dockerfile.j2 new file mode 100644 index 0000000000..1d5717546f --- /dev/null +++ b/extra/ansible/docker/roles/docker-host/templates/Dockerfile.j2 @@ -0,0 +1,16 @@ +FROM {{ base_image }} +ADD /repos /etc/yum.repos.d/ +#ADD /rpms /root/ +#RUN yum install -y /root/*.rpm +ADD /launch_scripts /root/ +ADD /bin_files /usr/sbin/ + +RUN mkdir -p /root/.ssh; chmod 700 /root/.ssh +ADD authorized_keys /root/.ssh/ + +RUN yum install -y openssh-server net-tools pacemaker pacemaker-cts resource-agents pcs corosync which fence-agents-common sysvinit-tools +RUN mkdir -p /etc/pacemaker/ +RUN echo {{ pacemaker_authkey }} > /etc/pacemaker/authkey +RUN /usr/sbin/sshd + +ENTRYPOINT ["/root/launch.sh"] diff --git a/extra/ansible/docker/roles/docker-host/templates/ip_start.j2 b/extra/ansible/docker/roles/docker-host/templates/ip_start.j2 new file mode 100755 index 0000000000..edbd3927e8 --- /dev/null +++ b/extra/ansible/docker/roles/docker-host/templates/ip_start.j2 @@ -0,0 +1,3 @@ +offset=$(hostname | sed s/.*-//) +export OCF_ROOT=/usr/lib/ocf/ OCF_RESKEY_ip={{ subnet }}.${offset} OCF_RESKEY_cidr_netmask=32 +/usr/lib/ocf/resource.d/heartbeat/IPaddr2 start diff --git a/extra/ansible/docker/site.yml b/extra/ansible/docker/site.yml new file mode 100644 index 0000000000..0cc65e47a4 --- /dev/null +++ b/extra/ansible/docker/site.yml @@ -0,0 +1,12 @@ +--- +# See /etc/ansible/hosts or -i hosts +- hosts: controllers + remote_user: root + roles: + - docker-host + +#- hosts: containers +# gather_facts: no +# remote_user: root +# roles: +# - docker-container