diff --git a/cts/lxc_autogen.sh.in b/cts/lxc_autogen.sh.in
index d06ba2ddb5..da4b92e98b 100755
--- a/cts/lxc_autogen.sh.in
+++ b/cts/lxc_autogen.sh.in
@@ -1,424 +1,526 @@
#!/bin/bash
containers="2"
download=0
share_configs=0
# different than default libvirt network in case this is run nested in a KVM instance
addr="192.168.123.1"
restore=0
restore_pcmk=0
restore_all=0
generate=0
key_gen=0
cib=0
anywhere=0
add_master=0
verify=0
working_dir="@CRM_CONFIG_CTS@/lxc"
-curdir=$(pwd)
run_dirs="/run /var/run /usr/var/run"
+SSH_CMD_OPTS="
+ -o StrictHostKeyChecking=no
+ -o ConnectTimeout=30
+ -o BatchMode=yes
+ -l root
+ -T
+"
+# must be on one line b/c used inside quotes
+SSH_RSYNC_OPTS="-o UserKnownHostsFile=/dev/null -o BatchMode=yes -o StrictHostKeyChecking=no"
+
function helptext() {
echo "lxc_autogen.sh - A tool for generating libvirt lxc containers for testing purposes."
echo ""
echo "Usage: lxc-autogen [options]"
echo ""
echo "Options:"
echo "-g, --generate Generate libvirt lxc environment in the directory this script is run from."
- echo "-k, --key-gen Generate local pacemaker remote key only."
+ echo "-k, --key-gen Generate pacemaker remote key only."
echo "-r, --restore-libvirt Restore the default network, and libvirt config to before this script ran."
echo "-p, --restore-cib Remove cib entries this script generated."
echo "-R, --restore-all Restore both libvirt and cib plus clean working directory. This will leave libvirt xml files though so rsc can be stopped properly."
echo ""
echo "-A, --allow-anywhere Allow the containers to live anywhere in the cluster"
echo "-a, --add-cib Add remote-node entries for each lxc instance into the cib"
echo "-m, --add-master Add master resource shared between remote-nodes"
echo "-d, --download-agent Download and install the latest VirtualDomain agent."
- echo "-s, --share-configs Copy container configs to all other known cluster nodes, (crm_node -l)"
+ echo "-s, --share-configs Synchronize on all known cluster nodes"
echo "-c, --containers Specify the number of containers to generate, defaults to $containers. Used with -g"
echo "-n, --network What network to override default libvirt network to. Example: -n 192.168.123.1. Used with -g"
echo "-v, --verify Verify environment is capable of running lxc"
echo ""
exit $1
}
while true ; do
case "$1" in
--help|-h|-\?) helptext 0;;
-c|--containers) containers="$2"; shift; shift;;
-d|--download-agent) download=1; shift;;
-s|--share-configs) share_configs=1; shift;;
-n|--network) addr="$2"; shift; shift;;
-r|--restore-libvirt) restore=1; shift;;
-p|--restore-cib) restore_pcmk=1; shift;;
-R|--restore-all)
restore_all=1
restore=1
restore_pcmk=1
shift;;
- -g|--generate) generate=1; shift;;
+ -g|--generate) generate=1; key_gen=1; shift;;
-k|--key-gen) key_gen=1; shift;;
-a|--add-cib) cib=1; shift;;
-A|--allow-anywhere) anywhere=1; shift;;
-m|--add-master) add_master=1; shift;;
-v|--verify) verify=1; shift;;
"") break;;
*) helptext 1;;
esac
done
if [ $verify -eq 1 ]; then
# verify virsh tool is available and that
# we can connect to lxc driver.
virsh -c lxc:/// list --all > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo "Could not connect 'virsh -c lxc:///' check that libvirt lxc driver is installed"
# yum install -y libvirt-daemon-driver-lxc libvirt-daemon-lxc libvirt-login-shell
exit 1
fi
cat /etc/selinux/config | grep -e "SELINUX.*=.*permissive" -e "SELINUX.*=.*enforcing" > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo "/etc/selinux/config must have SELINUX set to permissive or enforcing mode."
exit 1
fi
ps x > /tmp/lxc-autogen-libvirt-test.txt
grep "libvirtd" /tmp/lxc-autogen-libvirt-test.txt
if [ $? -ne 0 ]; then
rm -f /tmp/lxc-autogen-libvirt-test.txt
echo "libvirtd isn't up."
exit 1
fi
rm -f /tmp/lxc-autogen-libvirt-test.txt
which rsync > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo "rsync is required"
fi
which pacemaker_remoted > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo "pacemaker_remoted is required"
fi
fi
#strip last digits off addr
addr=$(echo $addr | awk -F. '{print $1"."$2"."$3}')
+this_node()
+{
+ crm_node -n
+}
+
+other_nodes()
+{
+ crm_node -l | awk "\$2 != \"$(this_node)\" {print \$2}"
+}
+
+make_directory()
+{
+ # argument must be full path
+ DIR="$1"
+
+ mkdir -p "$DIR"
+ if [ $share_configs -eq 1 ]; then
+ for node in $(other_nodes); do
+ ssh $SSH_CMD_OPTS $node mkdir -p "$DIR"
+ done
+ fi
+}
+
+sync_file()
+{
+ TARGET="$1"
+
+ if [ $share_configs -eq 1 ]; then
+ for node in $(other_nodes); do
+ rsync -ave "ssh $SSH_RSYNC_OPTS" "$TARGET" "${node}:${TARGET}"
+ done
+ fi
+}
+
+download_agent()
+{
+ wget https://raw.github.com/ClusterLabs/resource-agents/master/heartbeat/VirtualDomain
+ chmod 755 VirtualDomain
+ mv -f VirtualDomain /usr/lib/ocf/resource.d/heartbeat/VirtualDomain
+ sync_file /usr/lib/ocf/resource.d/heartbeat/VirtualDomain
+}
+
set_network()
{
rm -f cur_network.xml
cat << END >> cur_network.xml
<network>
<name>default</name>
<uuid>41ebdb84-7134-1111-a136-91f0f1119225</uuid>
<forward mode='nat'/>
<bridge name='virbr0' stp='on' delay='0' />
<mac address='52:54:00:A8:12:35'/>
<ip address='$addr.1' netmask='255.255.255.0'>
<dhcp>
<range start='$addr.2' end='$addr.254' />
</dhcp>
</ip>
</network>
END
+ sync_file ${working_dir}/cur_network.xml
+}
+
+distribute_configs()
+{
+ for node in $(other_nodes); do
+ rsync -ave "ssh $SSH_RSYNC_OPTS" ${working_dir}/lxc*.xml ${node}:${working_dir}
+ rsync -ave "ssh $SSH_RSYNC_OPTS" ${working_dir}/lxc*-filesystem ${node}:${working_dir}
+ done
+}
+
+start_network()
+{
+ NODE="$1"
+ ssh $SSH_CMD_OPTS $NODE <<-EOF
+ cd $working_dir
virsh net-info default >/dev/null 2>&1
- if [ $? -eq 0 ]; then
+ if [ \$? -eq 0 ]; then
if [ ! -f restore_default.xml ]; then
virsh net-dumpxml default > restore_default.xml
fi
virsh net-destroy default
virsh net-undefine default
fi
virsh net-define cur_network.xml
virsh net-start default
virsh net-autostart default
+EOF
+}
+
+start_network_all()
+{
+ start_network $(this_node)
+ if [ $share_configs -eq 1 ]; then
+ for node in $(other_nodes); do
+ start_network $node
+ done
+ fi
+}
+
+add_hosts_entry()
+{
+ IP="$1"
+ HNAME="$2"
+
+ echo $IP $HNAME >>/etc/hosts
+ if [ $share_configs -eq 1 ]; then
+ for node in $(other_nodes); do
+ ssh $SSH_CMD_OPTS $node "echo $IP $HNAME >>/etc/hosts"
+ done
+ fi
}
generate_key()
{
- #generate pacemaker remote key
- ls /etc/pacemaker/authkey > /dev/null 2>&1
- if [ $? != 0 ]; then
- mkdir -p /etc/pacemaker
- dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1
+ if [ ! -e /etc/pacemaker/authkey ]; then
+ make_directory /etc/pacemaker
+ dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1
+ sync_file /etc/pacemaker/authkey
fi
}
generate()
{
set_network
# Generate libvirt domains in xml
for (( c=1; c <= $containers; c++ ))
do
- rm -rf lxc$c-filesystem
+ # Clean any previous definition
+ rm -rf lxc$c.xml lxc$c-filesystem
+
+ # Create a basic filesystem with run directories
for dir in $run_dirs; do
mkdir -p lxc$c-filesystem/$dir
done
- rm -f lxc$c.xml
+ # Create libvirt definition
suffix=$((10 + $c))
prefix=$(echo $addr | awk -F. '{print $1"."$2}')
subnet=$(echo $addr | awk -F. '{print $3}')
while [ $suffix -gt 255 ]; do
subnet=$(($subnet + 1))
suffix=$(($subnet - 255))
done
+ cip=$prefix.$subnet.$suffix
cat << END >> lxc$c.xml
<domain type='lxc'>
<name>lxc$c</name>
<memory unit='KiB'>200704</memory>
<os>
<type>exe</type>
<init>$working_dir/lxc$c-filesystem/launch-helper</init>
</os>
<devices>
<console type='pty'/>
<filesystem type='ram'>
<source usage='150528'/>
<target dir='/dev/shm'/>
</filesystem>
END
for dir in $run_dirs; do
cat << END >> lxc$c.xml
<filesystem type='mount'>
- <source dir='$working_dir/lxc$c-filesystem/$dir'/>
+ <source dir='$working_dir/lxc$c-filesystem${dir}'/>
<target dir='$dir'/>
</filesystem>
END
done
cat << END >> lxc$c.xml
<interface type='network'>
<mac address='52:54:$(($RANDOM % 9))$(($RANDOM % 9)):$(($RANDOM % 9))$(($RANDOM % 9)):$(($RANDOM % 9))$(($RANDOM % 9)):$(($RANDOM % 9))$(($RANDOM % 9))'/>
<source network='default'/>
</interface>
</devices>
</domain>
END
+
+ # Create CIB definition
rm -f container$c.cib
cat << END >> container$c.cib
<primitive class="ocf" id="container$c" provider="heartbeat" type="VirtualDomain">
<instance_attributes id="container$c-instance_attributes">
<nvpair id="container$c-instance_attributes-force_stop" name="force_stop" value="true"/>
<nvpair id="container$c-instance_attributes-hypervisor" name="hypervisor" value="lxc:///"/>
<nvpair id="container$c-instance_attributes-config" name="config" value="$working_dir/lxc$c.xml"/>
</instance_attributes>
<utilization id="container$c-utilization">
<nvpair id="container$c-utilization-cpu" name="cpu" value="1"/>
<nvpair id="container$c-utilization-hv_memory" name="hv_memory" value="100"/>
</utilization>
<meta_attributes id="container$c-meta_attributes">
<nvpair id="container$c-meta_attributes-remote-node" name="remote-node" value="lxc$c"/>
</meta_attributes>
</primitive>
END
+ # Create container init
rm -f lxc$c-filesystem/launch-helper
cat << END >> lxc$c-filesystem/launch-helper
#!/bin/bash
-ip -f inet addr add $prefix.$subnet.$suffix/24 dev eth0
+ip -f inet addr add $cip/24 dev eth0
ip link set eth0 up
ip route add default via $addr.1
hostname lxc$c
df > $working_dir/lxc$c-filesystem/disk_usage.txt
export PCMK_debugfile=/var/log/pacemaker_remote_lxc$c.log
/usr/sbin/pacemaker_remoted
END
chmod 711 lxc$c-filesystem/launch-helper
- cat << END >> /etc/hosts
-$prefix.$subnet.$suffix lxc$c
-END
+ add_hosts_entry $cip lxc$c
done
+ # Create CIB fragment for a master-slave resource
rm -f lxc-ms.cib
cat << END >> lxc-ms.cib
<master id="lxc-ms-master">
<primitive class="ocf" id="lxc-ms" provider="pacemaker" type="Stateful">
<instance_attributes id="lxc-ms-instance_attributes"/>
<operations>
<op id="lxc-ms-monitor-interval-10s" interval="10s" name="monitor"/>
</operations>
</primitive>
<meta_attributes id="lxc-ms-meta_attributes">
<nvpair id="lxc-ms-meta_attributes-master-max" name="master-max" value="1"/>
<nvpair id="lxc-ms-meta_attributes-clone-max" name="clone-max" value="$containers"/>
</meta_attributes>
</master>
END
}
apply_cib_master()
{
cibadmin -Q > cur.cib
export CIB_file=cur.cib
cibadmin -o resources -Mc -x lxc-ms.cib
- for tmp in $(ls lxc*.xml); do
- tmp=$(echo $tmp | sed -e 's/\.xml//g')
+ for tmp in $(ls lxc*.xml | sed -e 's/\.xml//g'); do
echo "<rsc_location id=\"lxc-ms-location-${tmp}\" node=\"${tmp}\" rsc=\"lxc-ms-master\" score=\"INFINITY\"/>" > tmp_constraint
cibadmin -o constraints -Mc -x tmp_constraint
done
# Make sure the version changes even if the content doesn't
cibadmin -B
unset CIB_file
cibadmin --replace -o configuration --xml-file cur.cib
rm -f cur.cib
}
apply_cib_entries()
{
- node=$(crm_node -n)
-
cibadmin -Q > cur.cib
export CIB_file=cur.cib
for tmp in $(ls container*.cib); do
cibadmin -o resources -Mc -x $tmp
remote_node=$(cat ${tmp} | grep remote-node | sed -n -e 's/^.*value=\"\(.*\)\".*/\1/p')
if [ $anywhere -eq 0 ]; then
tmp=$(echo $tmp | sed -e 's/\.cib//g')
- crm_resource -M -r $tmp -H $node
+ crm_resource -M -r $tmp -H $(this_node)
fi
echo "<rsc_location id=\"lxc-ping-location-${remote_node}\" node=\"${remote_node}\" rsc=\"Connectivity\" score=\"-INFINITY\"/>" > tmp_constraint
# it's fine if applying this constraint fails. it's just to help with cts
# when the connectivity resources are in use. those resources fail the remote-nodes.
cibadmin -o constraints -Mc -x tmp_constraint > /dev/null 2>&1
for rsc in $(crm_resource -l | grep rsc_ ); do
echo "<rsc_location id=\"lxc-${rsc}-location-${remote_node}\" node=\"${remote_node}\" rsc=\"${rsc}\" score=\"-INFINITY\"/>" > tmp_constraint
cibadmin -o constraints -Mc -x tmp_constraint > /dev/null 2>&1
done
rm -f tmp_constraint
done
# Make sure the version changes even if the content doesn't
cibadmin -B
unset CIB_file
cibadmin --replace -o configuration --xml-file cur.cib
rm -f cur.cib
}
restore_cib()
{
- node=$(crm_node -n)
cibadmin -Q > cur.cib
export CIB_file=cur.cib
- for tmp in $(ls lxc*.xml); do
- tmp=$(echo $tmp | sed -e 's/\.xml//g')
+ for tmp in $(ls lxc*.xml | sed -e 's/\.xml//g'); do
echo "<rsc_location id=\"lxc-ms-location-${tmp}\" node=\"${tmp}\" rsc=\"lxc-ms-master\" score=\"INFINITY\"/>" > tmp_constraint
cibadmin -o constraints -D -x tmp_constraint
echo "<rsc_location id=\"lxc-ping-location-${tmp}\" node=\"${tmp}\" rsc=\"Connectivity\" score=\"-INFINITY\"/>" > tmp_constraint
cibadmin -o constraints -D -x tmp_constraint
for rsc in $(crm_resource -l | grep rsc_ ); do
echo "<rsc_location id=\"lxc-${rsc}-location-${tmp}\" node=\"${tmp}\" rsc=\"${rsc}\" score=\"-INFINITY\"/>" > tmp_constraint
cibadmin -o constraints -D -x tmp_constraint
done
rm -f tmp_constraint
done
cibadmin -o resources -D -x lxc-ms.cib
for tmp in $(ls container*.cib); do
tmp=$(echo $tmp | sed -e 's/\.cib//g')
- crm_resource -U -r $tmp -H $node
+ crm_resource -U -r $tmp -H $(this_node)
crm_resource -D -r $tmp -t primitive
done
# Make sure the version changes even if the content doesn't
cibadmin -B
unset CIB_file
cibadmin --replace -o configuration --xml-file cur.cib
rm -f cur.cib
# Allow the cluster to stabilize before continuing
crm_resource --wait
# Purge nodes from caches and CIB status section
- for tmp in $(ls lxc*.xml); do
- tmp=$(echo $tmp | sed -e 's/\.xml//g')
+ for tmp in $(ls lxc*.xml | sed -e 's/\.xml//g'); do
crm_node --force --remove $tmp
done
}
-restore_libvirt()
+restore_network()
{
- for tmp in $(ls lxc*.xml); do
- tmp=$(echo $tmp | sed -e 's/\.xml//g')
- virsh -c lxc:/// destroy $tmp > /dev/null 2>&1
- virsh -c lxc:/// undefine $tmp > /dev/null 2>&1
-
- sed -i.bak "/...\....\....\..* ${tmp}/d" /etc/hosts
- echo "$tmp destroyed"
+ NODE="$1"
+
+ ssh $SSH_CMD_OPTS $NODE <<-EOF
+ cd $working_dir
+ for tmp in \$(ls lxc*.xml | sed -e 's/\.xml//g'); do
+ virsh -c lxc:/// destroy \$tmp >/dev/null 2>&1
+ virsh -c lxc:/// undefine \$tmp >/dev/null 2>&1
+ sed -i.bak "/...\....\....\..* \${tmp}/d" /etc/hosts
done
-
- virsh net-destroy default > /dev/null 2>&1
- virsh net-undefine default > /dev/null 2>&1
+ virsh net-destroy default >/dev/null 2>&1
+ virsh net-undefine default >/dev/null 2>&1
if [ -f restore_default.xml ]; then
virsh net-define restore_default.xml
virsh net-start default
- if [ $? -eq 0 ]; then
- echo "default network restored"
- fi
+ rm restore_default.xml
fi
- rm -f restore_default.xml > /dev/null 2>&1
+EOF
+ echo "Containers destroyed and default network restored on $NODE"
}
-distribute_configs()
+restore_libvirt()
+{
+ restore_network $(this_node)
+ if [ $share_configs -eq 1 ]; then
+ for node in $(other_nodes); do
+ restore_network $node
+ done
+ fi
+}
+
+restore_files()
{
- nodes=`crm_node -l | awk '{print $2}'`
- for node in $nodes; do
- ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o BatchMode=yes -l root $node mkdir -p /$working_dir
- rsync -ave 'ssh -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o StrictHostKeyChecking=no' $working_dir/lxc*.xml $node:/$working_dir
- rsync -ave 'ssh -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o StrictHostKeyChecking=no' $working_dir/lxc*-filesystem $node:/$working_dir
- done
+ ls | grep -v "lxc.\.xml" | xargs rm -rf
+ if [ $share_configs -eq 1 ]; then
+ for node in $(other_nodes); do
+ ssh $SSH_CMD_OPTS $node rm -rf \
+ $working_dir/lxc*-filesystem \
+ $working_dir/cur_network.xml
+ done
+ fi
}
-mkdir -p $working_dir
+make_directory $working_dir
cd $working_dir
+# Generate files as requested
if [ $download -eq 1 ]; then
- wget https://raw.github.com/ClusterLabs/resource-agents/master/heartbeat/VirtualDomain
- chmod 755 VirtualDomain
- mv -f VirtualDomain /usr/lib/ocf/resource.d/heartbeat/VirtualDomain
-fi
-if [ $restore_pcmk -eq 1 ]; then
- restore_cib
-fi
-if [ $restore -eq 1 ]; then
- restore_libvirt
+ download_agent
fi
if [ $key_gen -eq 1 ]; then
generate_key
fi
if [ $generate -eq 1 ]; then
- if [ $key_gen -eq 0 ]; then
- generate_key
- fi
generate
fi
+if [ $share_configs -eq 1 ]; then
+ distribute_configs
+fi
+if [ $generate -eq 1 ]; then
+ start_network_all
+fi
+
+# Update cluster as requested
if [ $cib -eq 1 ]; then
apply_cib_entries
fi
if [ $add_master -eq 1 ]; then
apply_cib_master
fi
-if [ $share_configs -eq 1 ]; then
- distribute_configs
+
+# Restore original state as requested
+if [ $restore_pcmk -eq 1 ]; then
+ restore_cib
+fi
+if [ $restore -eq 1 ]; then
+ restore_libvirt
fi
if [ $restore_all -eq 1 ]; then
- ls | grep -v "lxc.\.xml" | xargs rm -rf
+ restore_files
fi
-
-cd $curdir