diff --git a/Makefile.am b/Makefile.am index d65b162..03a74ef 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,164 +1,166 @@ # Copyright (c) 2009 Red Hat, Inc. # # Authors: Andrew Beekhof # Steven Dake (sdake@redhat.com) # # This software licensed under BSD license, the text of which follows: # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # - Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # - Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # - Neither the name of the MontaVista Software, Inc. nor the names of its # contributors may be used to endorse or promote products derived from this # software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. SPEC = $(PACKAGE_NAME).spec TARFILE = $(PACKAGE_NAME)-$(VERSION).tar.gz EXTRA_DIST = autogen.sh conf/booth.conf.example \ $(bootharbitrator_SCRIPTS) $(boothsite_SCRIPTS) \ $(boothnoarch_SCRIPTS) AUTOMAKE_OPTIONS = foreign MAINTAINERCLEANFILES = Makefile.in aclocal.m4 configure depcomp \ config.guess config.sub missing install-sh \ autoheader automake autoconf test_lense.sh dist_doc_DATA = AUTHORS README COPYING README.upgrade-from-v0.1 -notrans_dist_man8_MANS = docs/boothd.8 +notrans_dist_man8_MANS = docs/boothd.8 docs/booth-keygen.8 boothconfdir = ${BOOTHSYSCONFDIR} boothconf_DATA = conf/booth.conf.example boothsitedir = /usr/lib/ocf/resource.d/pacemaker boothsite_SCRIPTS = script/ocf/booth-site boothocfdir = /usr/lib/ocf/resource.d/booth boothocf_SCRIPTS = script/ocf/sharedrsc bootharbitratordir = ${INITDDIR} bootharbitrator_SCRIPTS = script/lsb/booth-arbitrator boothnoarchdir = $(datadir)/$(PACKAGE_NAME) boothnoarch_SCRIPTS = script/service-runnable +sbin_SCRIPTS = script/booth-keygen + TESTS = test/runtests.py SUBDIRS = src docs coverity: cov-build --dir=cov make cov-analyze --dir cov --concurrency --wait-for-license cov-format-errors --dir cov install-exec-local: $(INSTALL) -d $(DESTDIR)/${boothconfdir} $(INSTALL) -d $(DESTDIR)/${bootharbitratordir} $(INSTALL) -d $(DESTDIR)/${boothsitedir} $(INSTALL) -d $(DESTDIR)/${boothocfdir} $(INSTALL) -d $(DESTDIR)/${SOCKETDIR} install-exec-hook: ln -sf ${sbindir}/boothd $(DESTDIR)/${sbindir}/booth uninstall-local: rmdir $(DESTDIR)/${boothconfdir} || :; rmdir $(DESTDIR)/${bootharbitratordir} || :; rmdir $(DESTDIR)/${boothsitedir} || :; rmdir $(DESTDIR)/${SOCKETDIR} || :; test: check lint: for dir in src; do make -C $$dir lint; done dist-clean-local: rm -f autoconf automake autoheader dist-hook: echo $(VERSION) > $(distdir)/.tarball-version ## make rpm/srpm section. $(SPEC): $(SPEC).in rm -f $@-t $@ date="$(shell LC_ALL=C date "+%a %b %d %Y")" && \ if [ -f .tarball-version ]; then \ gitver="$(shell cat .tarball-version)" && \ rpmver=$$gitver && \ alphatag="" && \ dirty="" && \ numcomm="0"; \ else \ gitver="$(shell git describe --tags --abbrev=4 --match='v*' HEAD 2>/dev/null)" && \ rpmver=`echo $$gitver | sed -e "s/^v//" -e "s/-.*//g"` && \ alphatag=`echo $$gitver | sed -e "s/.*-//" -e "s/^g//"` && \ vtag=`echo $$gitver | sed -e "s/-.*//g"` && \ numcomm=`git rev-list $$vtag..HEAD | wc -l` && \ git update-index --refresh > /dev/null 2>&1 || true && \ dirty=`git diff-index --name-only HEAD 2>/dev/null`; \ fi && \ if [ -n "$$dirty" ]; then dirty="dirty"; else dirty=""; fi && \ if [ "$$numcomm" = "0" ]; then \ sed \ -e "s#@version@#$$rpmver#g" \ -e "s#%glo.*alpha.*##g" \ -e "s#%glo.*numcomm.*##g" \ -e "s#@dirty@#$$dirty#g" \ -e "s#@date@#$$date#g" \ $< > $@-t; \ else \ sed \ -e "s#@version@#$$rpmver#g" \ -e "s#@alphatag@#$$alphatag#g" \ -e "s#@numcomm@#$$numcomm#g" \ -e "s#@dirty@#$$dirty#g" \ -e "s#@date@#$$date#g" \ $< > $@-t; \ fi; \ if [ -z "$$dirty" ]; then sed -i -e "s#%glo.*dirty.*##g" $@-t; fi chmod a-w $@-t mv $@-t $@ $(TARFILE): $(MAKE) dist RPMBUILDOPTS = --define "_sourcedir $(abs_builddir)" \ --define "_specdir $(abs_builddir)" \ --define "_builddir $(abs_builddir)" \ --define "_srcrpmdir $(abs_builddir)" \ --define "_rpmdir $(abs_builddir)" srpm: clean autoreconf -if $(MAKE) $(SPEC) $(TARFILE) rpmbuild $(WITH_LIST) $(RPMBUILDOPTS) --nodeps -bs $(SPEC) rpm: clean autoreconf -if $(MAKE) $(SPEC) $(TARFILE) rpmbuild $(WITH_LIST) $(RPMBUILDOPTS) -ba $(SPEC) diff --git a/README b/README index 5d8c38c..e3d5156 100644 --- a/README +++ b/README @@ -1,195 +1,191 @@ The Booth Cluster Ticket Manager ============= Booth manages tickets which authorize cluster sites located in geographically dispersed locations to run resources. It facilitates support of geographically distributed clustering in Pacemaker. Booth is based on the Raft consensus algorithm. Though the implementation is not complete (there is no log) and there are a few additions and modifications, booth guarantees that a ticket is always available at just one site as long as it has exclusive control of the tickets. The git repository is available at github: github can also track issues or bug reports. Description of a booth cluster ============================== Booth cluster is a collection of cooperative servers communicating using the booth protocol. The purpose of the booth cluster is to manage cluster tickets. The booth cluster consists of at least three servers. A booth server can be either a site or an arbitrator. Arbitrators take part in elections and so help resolve ties, but cannot hold tickets. The basic unit in the booth cluster is a ticket. Every non-granted ticket is in the initial state on all servers. For granted tickets, the server holding the ticket is the leader and other servers are followers. The leader issues heartbeats and ticket updates to the followers. The followers are required to obey the leader. Booth startup ------------ On startup, the booth process first loads tickets, if available, from the CIB. Afterwards, it broadcasts a query to get tickets' status from other servers. In-memory copies are updated from the replies if they contain newer ticket data. If the server discovers that itself is the ticket leader, it tries to establish its authority again by broadcasting heartbeat. If it succeeds, it continues as the leader for this ticket. The other booth servers become followers. This procedure is possible only immediately after the booth startup. It also serves as a configuration reload. Grant and revoke operations ------------ A ticket first has to be granted using the 'booth client grant' command. Obviously, it is not possible to grant a ticket which is currently granted. Ticket revoke is the operation which is the opposite of grant. An administrative revoke may be started at any server, but the operation itself happens only at the leader. If the leader is unreachable, the ticket cannot be revoked. The user will need to wait until the ticket expires. A ticket grant may be delayed if not all sites are reachable. The delay is the ticket expiry time extended by acquire-after, if set. This is to ensure that the unreachable site relinquished the ticket it may have been holding and stopped the corresponding cluster resources. If the user is absolutely sure that the unreachable site does not hold the ticket, the delay may be skipped by using the '-F' option of the 'booth grant' command. If in effect, the grant delay time is shown in the 'booth list' command output. Ticket management and server operation ------------ A granted ticket is managed by the booth servers so that its availability is maximized without breaking the basic guarantee that the ticket is granted to one site only. The server where the ticket is granted is the leader, the other servers are followers. The leader occasionally sends heartbeats, once every half ticket expiry under normal circumstances. If a follower doesn't hear from the leader longer than the ticket expiry time, it will consider the ticket lost, and try to acquire it by starting new elections. A server starts elections by broadcasting the REQ_VOTE RPC. Other servers reply with the VOTE_FOR RPC, in which they record its vote. Normally, the sender of the first REQ_VOTE gets the vote of the receiver. Whichever server gets a majority of votes wins the elections. On ties, elections are restarted. To decrease chance of elections ending in a tie, a server waits for a short random period before sending out the REQ_VOTE packets. Everything else being equal, the server which sends REQ_VOTE first gets elected. Elections are described in more detail in the raft paper at . Ticket renewal (or update) is a two-step process. Before actually writing the ticket to the CIB, the server holding the ticket first tries to establish that it still has the majority for that ticket. That is done by broadcasting a heartbeat. If the server receives enough acknowledgements, it then stores the ticket to the CIB and broadcasts the UPDATE RPC with updated ticket expiry time so that the followers can update local ticket copies. Ticket renewals are configurable and by default set to half ticket expire time. Before ticket renewal, the leader runs an external program if such program is set in 'before-acquire-handler'. The external program should ensure that the cluster managed service which is protected by this ticket can run at this site. If that program fails, the leader relinquishes the ticket. It announces its intention to step down by broadcasting an unsolicited VOTE_FOR with an empty vote. On receiving such RPC other servers start new elections to elect a new leader. Split brain ------------ On split brains two possible issues arise: leader in minority and follower disconnected from the leader. Let's take a look at the first one. The leader in minority eventually expires the ticket because it cannot receieve majority of acknowledgements in reply to its heartbeats. The other partition runs elections (at about the same time, as they find the ticket lost after its expiry) and, if it can get the majority, the elections winner becomes a new leader for the ticket. After split brain gets resolved, the old leader will become follower as soon as it receives heartbeat from the new leader. Note the timing: the old leader releases the ticket at around the same time as when new elections in the other partition are held. This is because the leader ensures that the ticket expire time is always the same on all servers in the booth cluster. The second situation, where a follower is disconnected from the leader, is a bit more difficult to handle. After the ticket expiry time, the follower will consider the ticket lost and start new elections. The elections repeatedly get restarted until the split brain is resolved. Then, the rest of the cluster send rejects in reply to REQ_VOTE RPC because the ticket is still valid and therefore couldn't have been lost. They know that because the reason for elections is included with every REQ_VOTE. Short intermittent split brains are handled well because the leader keeps resending heartbeats until it gets replies from all servers serving sites. Authentication ============== In order to prevent malicious parties from affecting booth operation, booth server can authenticate both clients (connecting over TCP) and other booth servers (connecting over UDP). The authentication is based on SHA1 HMAC (Keyed-Hashing Message Authentication) and shared key. The HMAC implementation is provided by the libgcrypt or mhash library. Message encryption is not included as the information exchanged between various booth parties does not seem to justify that. Every message (packet) contains a hash code computed from the combination of payload and the secret key. Whoever has the secret key can then verify that the message is authentic. The shared key is used by both the booth client and the booth server, hence it needs to be copied to all nodes at each site and all arbitrators. Of course, a secure channel is required for key transfer. It is recommended to use csync2 or ssh. Timestamps are included and verified to fend against replay attacks. Certain time skew, 10 minutes by default, is tolerated. Packets either not older than that or with a timestamp more recent than the previous one from the same peer are accepted. The time skew can be configured in the booth configuration file. -There is no way to configure booth to use a hash different from -SHA1. That should not be a problem in practice as SHA1 provides -sufficient security (the lesser MD5 would too). - # vim: set ft=asciidoc : diff --git a/booth.spec b/booth.spec index 09f1f68..7d54924 100644 --- a/booth.spec +++ b/booth.spec @@ -1,190 +1,192 @@ %if 0%{?suse_version} %global booth_docdir %{_defaultdocdir}/%{name} %else # newer fedora distros have _pkgdocdir, rely on that when # available %{!?_pkgdocdir: %global _pkgdocdir %%{_docdir}/%{name}-%{version}} # Directory where we install documentation %global booth_docdir %{_pkgdocdir} %endif %global test_path %{_datadir}/booth/tests %if 0%{?suse_version} %define _libexecdir %{_libdir} %endif %define with_extra_warnings 0 %define with_debugging 0 %define without_fatal_warnings 1 %define _fwdefdir /etc/sysconfig/SuSEfirewall2.d/services %if 0%{?fedora} || 0%{?centos} || 0%{?rhel} %define pkg_group System Environment/Daemons %else %define pkg_group Productivity/Clustering/HA %endif Name: booth Url: https://github.com/ClusterLabs/booth Summary: Ticket Manager for Multi-site Clusters License: GPL-2.0+ Group: %{pkg_group} Version: 0.2.0 Release: 0 Source: booth.tar.bz2 Source1: %name-rpmlintrc BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRequires: asciidoc BuildRequires: autoconf BuildRequires: automake BuildRequires: glib2-devel BuildRequires: libgcrypt-devel %if 0%{?fedora} || 0%{?centos} || 0%{?rhel} BuildRequires: cluster-glue-libs-devel BuildRequires: pacemaker-libs-devel %else BuildRequires: libglue-devel BuildRequires: libpacemaker-devel %endif BuildRequires: libxml2-devel BuildRequires: pkgconfig %if 0%{?fedora} || 0%{?centos} || 0%{?rhel} Requires: pacemaker >= 1.1.8 Requires: cluster-glue-libs >= 1.0.6 %else Requires: pacemaker-ticket-support >= 2.0 %endif %description Booth manages tickets which authorize cluster sites located in geographically dispersed locations to run resources. It facilitates support of geographically distributed clustering in Pacemaker. %prep %setup -q -n %{name} %build ./autogen.sh %configure \ --with-initddir=%{_initrddir} \ --docdir=%{booth_docdir} make #except check #%check #make check %install make DESTDIR=$RPM_BUILD_ROOT install docdir=%{booth_docdir} mkdir -p %{buildroot}/%{_mandir}/man8/ gzip < docs/boothd.8 > %{buildroot}/%{_mandir}/man8/booth.8.gz ln %{buildroot}/%{_mandir}/man8/booth.8.gz %{buildroot}/%{_mandir}/man8/boothd.8.gz %if %{defined _unitdir} # systemd mkdir -p %{buildroot}/%{_unitdir} cp -a conf/booth@.service %{buildroot}/%{_unitdir}/booth@.service ln -s /usr/sbin/service %{buildroot}%{_sbindir}/rcbooth-arbitrator %else # sysV init ln -s ../../%{_initddir}/booth-arbitrator %{buildroot}%{_sbindir}/rcbooth-arbitrator %endif #install test-parts mkdir -p %{buildroot}/%{test_path} cp -a unit-tests/ script/unit-test.py test conf %{buildroot}/%{test_path}/ chmod +x %{buildroot}/%{test_path}/test/booth_path chmod +x %{buildroot}/%{test_path}/test/live_test.sh mkdir -p %{buildroot}/%{test_path}/src/ ln -s %{_sbindir}/boothd %{buildroot}/%{test_path}/src/ rm -f %{buildroot}/%{test_path}/test/*.pyc %if 0%{?suse_version} #SUSE firewall rule mkdir -p $RPM_BUILD_ROOT/%{_fwdefdir} install -m 644 %{S:2} $RPM_BUILD_ROOT/%{_fwdefdir}/booth %endif %check %if 0%{?run_build_tests} echo "%%run_build_tests set to %run_build_tests; including tests" make check %else echo "%%run_build_tests set to %run_build_tests; skipping tests" %endif %clean rm -rf %{buildroot} %files %defattr(-,root,root,-) %{_sbindir}/booth %{_sbindir}/boothd +%{_sbindir}/booth-keygen %{_mandir}/man8/booth.8.gz %{_mandir}/man8/boothd.8.gz +%{_mandir}/man8/booth-keygen.8.gz %dir /usr/lib/ocf %dir /usr/lib/ocf/resource.d %dir /usr/lib/ocf/resource.d/pacemaker %dir %{_sysconfdir}/booth %{_sbindir}/rcbooth-arbitrator /usr/lib/ocf/resource.d/pacemaker/booth-site %config %{_sysconfdir}/booth/booth.conf.example %if 0%{?suse_version} %config %{_fwdefdir}/booth %endif %if %{defined _unitdir} %{_unitdir}/booth@.service %exclude %{_initddir}/booth-arbitrator %else %{_initddir}/booth-arbitrator %endif %dir %{_datadir}/booth %{_datadir}/booth/service-runnable %doc AUTHORS README COPYING %doc README.upgrade-from-v0.1 # this should be preun, but... %pre # new installation? test -x %{_sbindir}/booth || exit 0 # stop the arbitrator if it's the previous paxos version 1.0 if [ "`booth version | awk '{print $2}'`" = "1.0" ]; then echo "booth v0.1 found" if grep -qs 'ticket.*;' /etc/booth/booth.conf; then echo "Convert the booth configuration in /etc/booth/booth.conf!" fi if ps -o pid,cmd -e | grep -qs "[b]oothd arbitrator"; then rcbooth-arbitrator stop fi fi exit 0 %package test Summary: Test scripts for Booth Group: %{pkg_group} Requires: booth Requires: python %description test This package contains automated tests for Booth, the Cluster Ticket Manager for Pacemaker. %files test %defattr(-,root,root) %doc README-testing %{test_path} %dir /usr/lib/ocf %dir /usr/lib/ocf/resource.d %dir /usr/lib/ocf/resource.d/booth /usr/lib/ocf/resource.d/booth/sharedrsc %changelog diff --git a/docs/Makefile.am b/docs/Makefile.am index d5792dc..3e9e7bc 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -1,40 +1,40 @@ # # docs: booth manual pages # # Copyright (C) 2014 Dejan Muhamedagic # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # MAINTAINERCLEANFILES = Makefile.in -asciiman = boothd.8.txt +asciiman = boothd.8.txt booth-keygen.8.txt doc_DATA = $(generated_docs) generated_docs = generated_mans = if BUILD_ASCIIDOC generated_docs += $(ascii:%.txt=%.html) $(asciiman:%.txt=%.html) generated_mans += $(asciiman:%.8.txt=%.8) $(generated_mans): $(asciiman) man8_MANS = $(generated_mans) endif %.html: %.txt $(ASCIIDOC) --unsafe --backend=xhtml11 $< %.8: %.8.txt a2x -f manpage $< clean-local: -rm -rf $(generated_docs) $(generated_mans) diff --git a/docs/booth-keygen.8.txt b/docs/booth-keygen.8.txt new file mode 100644 index 0000000..0cf6785 --- /dev/null +++ b/docs/booth-keygen.8.txt @@ -0,0 +1,52 @@ +BOOTH-KEYGEN(8) +=============== +:doctype: manpage + + +NAME +---- +booth-keygen - generate authentication key + + +SYNOPSIS +-------- +*booth-keygen* ['-h'] ['auth-file'] + + +DESCRIPTION +----------- +This program generates an authentication key suitable for 'booth' +using '/dev/urandom' as source. + + +PARAMETERS +---------- + +*'auth-file'*:: + The file to contain the generated key. Defaults to + '/etc/booth/authkey'. Use absolute paths. + + +OPTIONS +------- +*-h*, *--help*:: + Print usage. + + +EXIT STATUS +----------- +*0*:: + Success. + +*!= 0*:: + File already exists or some other error. + + +COPYING +------- + +Copyright (C) 2015 Dejan Muhamedagic + +Free use of this software is granted under the terms of the GNU +General Public License (GPL). + diff --git a/script/booth-keygen b/script/booth-keygen new file mode 100644 index 0000000..54f079d --- /dev/null +++ b/script/booth-keygen @@ -0,0 +1,63 @@ +#!/bin/sh +# +# Generate authentication key for booth +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# + +DFLT_AUTHFILE=/etc/booth/authkey +KEYSIZE=64 +# /dev/urandom should be good enough +RND_SRC=/dev/urandom + +usage() { + cat<&2 +} +fatal() { + error $* + exit 1 +} + +case "$1" in +"-h"|"--help"|"-?") usage;; +/*|"") : ;; +*) fatal "please use absolute path for the key file" ;; +esac + +keyf=${1:-$DFLT_AUTHFILE} + +if test -f $keyf; then + fatal "file $keyf already exists" +fi + +umask 077 +dd if=$RND_SRC of=$keyf bs=$KEYSIZE count=1 status=none +rc=$? +if [ $rc -ne 0 ]; then + exit $rc +fi + +chown root:root $keyf