From 00fac7322e34e284ab02c50e37c7bcec57b83d4f Mon Sep 17 00:00:00 2001 From: Denis Lambolez Date: Mon, 18 Dec 2017 23:31:52 +0100 Subject: [PATCH] First version with full cluster installation --- .vagrant/machines/admin/libvirt/vagrant_cwd | 1 - .vagrant/machines/osd1/libvirt/vagrant_cwd | 1 - .vagrant/machines/osd2/libvirt/vagrant_cwd | 1 - Vagrantfile | 10 +-- scripts/ceph-install.sh | 87 +++++++++++++------ scripts/cephtest-utils.sh | 3 +- scripts/post-destroy.sh | 39 --------- scripts/provision-admin.sh | 51 +++++++++++ scripts/provision.sh | 93 +++++++++------------ 9 files changed, 156 insertions(+), 130 deletions(-) delete mode 100644 .vagrant/machines/admin/libvirt/vagrant_cwd delete mode 100644 .vagrant/machines/osd1/libvirt/vagrant_cwd delete mode 100644 .vagrant/machines/osd2/libvirt/vagrant_cwd delete mode 100755 scripts/post-destroy.sh create mode 100644 scripts/provision-admin.sh diff --git a/.vagrant/machines/admin/libvirt/vagrant_cwd b/.vagrant/machines/admin/libvirt/vagrant_cwd deleted file mode 100644 index 5e1374f..0000000 --- a/.vagrant/machines/admin/libvirt/vagrant_cwd +++ /dev/null @@ -1 +0,0 @@ -/virt/vagrant/cephtest \ No newline at end of file diff --git a/.vagrant/machines/osd1/libvirt/vagrant_cwd b/.vagrant/machines/osd1/libvirt/vagrant_cwd deleted file mode 100644 index 5e1374f..0000000 --- a/.vagrant/machines/osd1/libvirt/vagrant_cwd +++ /dev/null @@ -1 +0,0 @@ -/virt/vagrant/cephtest \ No newline at end of file diff --git a/.vagrant/machines/osd2/libvirt/vagrant_cwd b/.vagrant/machines/osd2/libvirt/vagrant_cwd deleted file mode 100644 index 5e1374f..0000000 --- a/.vagrant/machines/osd2/libvirt/vagrant_cwd +++ /dev/null @@ -1 +0,0 @@ -/virt/vagrant/cephtest \ No newline at end of file diff --git a/Vagrantfile b/Vagrantfile index 83b187c..e362d77 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -10,13 +10,8 @@ Vagrant.configure("2") do |config| run "pre-up.sh" end - # Trigger the Vagrant post-destroy script aftre destroying the last VM only - # config.trigger.after :destroy, :vm => ["node-admin"], :append_to_path => ["#{vagrant_root}/scripts"] do - run "post-destroy.sh" - # end - # Shell provisionner for all VMs - config.vm.provision "shell", path: "scripts/provision.sh" + config.vm.provision "ceph-preflight", type: "shell", path: "scripts/provision.sh" # All VMs are based on the same box config.vm.box = "bento/ubuntu-16.04" @@ -37,7 +32,7 @@ Vagrant.configure("2") do |config| # Standard configuration for all VMs config.vm.provider :libvirt do |libvirt| - libvirt.memory = 1024 + libvirt.memory = 2048 libvirt.volume_cache = "writeback" libvirt.graphics_type = "spice" libvirt.video_type = "qxl" @@ -46,6 +41,7 @@ Vagrant.configure("2") do |config| # admin VM config.vm.define "node-admin", primary: true do |admin| admin.vm.hostname = "node-admin" + admin.vm.provision "ceph-install", type: "shell", keep_color: true, path: "scripts/provision-admin.sh" end # osd1 VM with private cluster network diff --git a/scripts/ceph-install.sh b/scripts/ceph-install.sh index 1a0136e..78b64f0 100755 --- a/scripts/ceph-install.sh +++ b/scripts/ceph-install.sh @@ -3,8 +3,8 @@ # Execute ceph distributed storage installation steps from the admin node via ceph-deploy # # Written by : Denis Lambolez -# Release : 1.0 -# Creation date : 04 December 2017 +# Release : 2.0 +# Creation date : 18 December 2017 # Description : Bash script # This script has been designed and written on Ubuntu 16.04 plateform. # It must be executed in ceph-admin context, on admin node @@ -15,46 +15,85 @@ # HISTORY : # Release | Date | Authors | Description # --------------+---------------+--------------- +------------------------------------------ +# 2.0 | 12.18.17 | Denis Lambolez | Make it a standalone provisioner # 1.0 | 12.04.17 | Denis Lambolez | Creation # | | | # | | | -# | | | # ========================================================================================= #set -xv # Version -VERSION=ceph-install-1.0-120417 +VERSION=ceph-install-2.0-121817 # This script is executed in guest context source "/vagrant/scripts/cephtest-utils.sh" +# Make sure only root can run the script +if [[ $(whoami) != $CEPH_ADMIN_USER ]]; then + echo "This script must be run as $CEPH_ADMIN_USER" >&2 + exit 1 +fi + # Network (dynamically defined by Vagrant) IP_ADDRESS=$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1) -PUBLIC_NETWORK=$(echo $IP_ADDRESS | awk -F '.' '{print $1"."$2"."$3".0"}') - -# Make sure only CEPH_ADMIN_USER can run the script -if [[ $(whoami) != $CEPH_ADMIN_USER ]]; then - echo "This script must be run as $CEPH_ADMIN_USER" 1>&2 - exit 1 -fi - -# Make sure this script is run from the admin node -if [[ $(hostname -s) != $ADMIN_NODE ]]; then - echo "This script must be run from $ADMIN_NODE" 1>&2 - exit 1 -fi +PUBLIC_NETWORK=$(echo $IP_ADDRESS | awk -F '.' '{print $1"."$2"."$3".0/24"}') # Cluster configuration directory mkdir -p "$GUEST_USER_DIR/ceph-cluster" cd "$GUEST_USER_DIR/ceph-cluster" +# Create cluster +ceph-deploy new $ADMIN_NODE + # Initialize cluster configuration -ceph-deploy new node-admin -echo "public network = $PUBLIC_NETWORK" >> ceph.conf -echo "cluster network = $CLUSTER_NETWORK" >> ceph.conf -echo "" >> ceph.conf -echo "osd pool default size = 2" >> ceph.conf -echo "osd pool default min size = 1" >> ceph.conf +cat << CLUSTERCONFIG >> ceph.conf +public network = $PUBLIC_NETWORK +cluster network = $CLUSTER_NETWORK + +osd pool default size = 2 +osd pool default min size = 1 +osd pool default pg num = 256 +osd pool default pgp num = 256 +CLUSTERCONFIG + +for NODE in $OSD_NODES; do +cat << CLUSTERCONFIG >> ceph.conf + +[mds.$NODE] +mds standby replay = true +mds standby for rank = 0 +CLUSTERCONFIG +done # Install ceph on all nodes -ceph-deploy install --release luminous node-admin node-osd1 node-osd2 \ No newline at end of file +ceph-deploy install --release luminous $NODES + +# Create initial monitor +ceph-deploy --overwrite-conf mon create-initial + +# Deploy configuration file and client keys +ceph-deploy admin $NODES + +# Add monitor on osd nodes +ceph-deploy mon add $OSD_NODES + +# Create manager on all nodes +#sudo mkdir -p /var/lib/ceph/mgr/ceph-$ADMIN_NODE +#sudo chown ceph:ceph /var/lib/ceph/mgr/ceph-$ADMIN_NODE +ceph-deploy mgr create $NODES + +# Create metadata server on osd nodes +ceph-deploy mds create $OSD_NODES + +# For each osd node, gather keys from admin node and create OSDs +for NODE in $OSD_NODES; do + ssh $NODE ceph-deploy gatherkeys $ADMIN_NODE + ssh $NODE sudo cp /home/$CEPH_ADMIN_USER/ceph.bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring + ssh $NODE sudo chown ceph:ceph /var/lib/ceph/bootstrap-osd/ceph.keyring + ssh $NODE sudo ceph-volume lvm create --filestore --data /dev/vdb --journal /dev/vda1 + ssh $NODE sudo ceph-volume lvm create --filestore --data /dev/vdc --journal /dev/vda2 +done; + +# wait 10 seconds and get cluster status +sleep 10 +sudo ceph -s diff --git a/scripts/cephtest-utils.sh b/scripts/cephtest-utils.sh index dbfa987..7811363 100755 --- a/scripts/cephtest-utils.sh +++ b/scripts/cephtest-utils.sh @@ -26,7 +26,6 @@ VERSION=cephtest-utils-1.0-120417 # Ceph user CEPH_ADMIN_USER="ceph-admin" -CEPH_ADMIN_EXEC="sudo -i -u $CEPH_ADMIN_USER" # Nodes ADMIN_NODE="node-admin" @@ -34,7 +33,7 @@ OSD_NODES="node-osd1 node-osd2" NODES="$ADMIN_NODE $OSD_NODES" # Networks -CLUSTER_NETWORK="172.28.128.0" +CLUSTER_NETWORK="172.28.128.0/24" # Guest name GUEST_NAME=$(hostname -s) diff --git a/scripts/post-destroy.sh b/scripts/post-destroy.sh deleted file mode 100755 index 431a9ee..0000000 --- a/scripts/post-destroy.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# ======================================================================================== -# Execute post-destroy cleaning -# -# Written by : Denis Lambolez -# Release : 1.0 -# Creation date : 16 December 2017 -# Description : Bash script -# This script has been designed and written on Ubuntu 16.04 plateform. -# It must be executed in vagrant context -# Usage : ./post-destroy.sh -# ---------------------------------------------------------------------------------------- -# ======================================================================================== -# -# HISTORY : -# Release | Date | Authors | Description -# --------------+---------------+--------------- +------------------------------------------ -# 1.0 | 12.16.17 | Denis Lambolez | Creation -# | | | -# ========================================================================================= -#set -xev - -# Version -VERSION=post-destroy-1.0-121617 - -# This script is executed in host context -source "$(dirname "$(readlink -f "$0")")/cephtest-utils.sh" - -# clean-up networks to start with fresh configuration -for NETWORK in vagrant-libvirt vagrant-private-dhcp; do - virsh net-list --all 2> /dev/null | grep $NETWORK | grep active - if [[ $? -eq 0 ]]; then - virsh net-destroy $NETWORK 2> /dev/null - fi - virsh net-list --all 2> /dev/null | grep $NETWORK - if [[ $? -eq 0 ]]; then - virsh net-undefine $NETWORK 2> /dev/null - fi -done diff --git a/scripts/provision-admin.sh b/scripts/provision-admin.sh new file mode 100644 index 0000000..77f9ceb --- /dev/null +++ b/scripts/provision-admin.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# ======================================================================================== +# Launch ceph installtion when cluster provisioning is done +# +# Written by : Denis Lambolez +# Release : 1.0 +# Creation date : 18 December 2017 +# Description : Bash script +# This script has been designed and written on Ubuntu 16.04 plateform. +# It must be executed in vagrant context, on admin node +# Usage : ./provision-admin.sh +# ---------------------------------------------------------------------------------------- +# ======================================================================================== +# +# HISTORY : +# Release | Date | Authors | Description +# --------------+---------------+--------------- +------------------------------------------ +# 1.0 | 12.18.17 | Denis Lambolez | Creation +# | | | +# | | | +# ========================================================================================= +#set -xv + +# Version +VERSION=provision-admin-1.0-121817 + +# This script is executed in guest context +source "/vagrant/scripts/cephtest-utils.sh" + +# Make sure this script is run from the admin node +if [[ $(hostname -s) != $ADMIN_NODE ]]; then + echo "This script must be run from $ADMIN_NODE" 1>&2 + exit 1 +fi + +# Wait for all nodes to be ready +TIMER_MAX=300 +for NODE in $NODES; do + TIMER=0 + until [[ -r "$GUEST_VAGRANT_SIGNAL_DIR/$NODE-PROVISION" ]]; do + sleep 1 + TIMER=$(($TIMER + 1)) + if [[ $TIMER -gt $TIMER_MAX ]]; then + echo "Waited too long for $NODE!" >&2 + exit 1 + fi + done +done + +# Launch ceph-installation +sudo -i -u $CEPH_ADMIN_USER $GUEST_VAGRANT_SCRIPT_DIR/ceph-install.sh \ No newline at end of file diff --git a/scripts/provision.sh b/scripts/provision.sh index 44f1443..afc525c 100755 --- a/scripts/provision.sh +++ b/scripts/provision.sh @@ -36,46 +36,23 @@ if [[ $EUID -ne 0 ]]; then exit 1 fi -# Create user ceph-admin if not existing -cat /etc/passwd | grep $CEPH_ADMIN_USER -if [[ $? -ne 0 ]]; then - useradd -m -s /bin/bash $CEPH_ADMIN_USER -fi - -# Make ceph-admin passwordless sudoer -echo "$CEPH_ADMIN_USER ALL = (root) NOPASSWD:ALL" | tee "/etc/sudoers.d/$CEPH_ADMIN_USER" -chmod 0440 "/etc/sudoers.d/$CEPH_ADMIN_USER" - -# Copy ceph-admin ssh keys and ssh config from Vagrant synced folder (muste be created by vagrant-preflight script) -$CEPH_ADMIN_EXEC mkdir -p "$GUEST_USER_SSH_DIR" -$CEPH_ADMIN_EXEC chmod 700 "$GUEST_USER_SSH_DIR" -for FILE in id_rsa id_rsa.pub config; do - $CEPH_ADMIN_EXEC rm -f "$GUEST_USER_SSH_DIR/$FILE" - $CEPH_ADMIN_EXEC cp "$GUEST_VAGRANT_SSH_DIR/$CEPH_ADMIN_USER-$FILE" "$GUEST_USER_SSH_DIR/$FILE" - $CEPH_ADMIN_EXEC chmod 644 "$GUEST_USER_SSH_DIR/$FILE" -done -$CEPH_ADMIN_EXEC chmod 600 "$GUEST_USER_SSH_DIR/id_rsa" -# Copy ceph-admin public key in authorized_keys -$CEPH_ADMIN_EXEC rm -f "$GUEST_USER_SSH_DIR/authorized_keys" -$CEPH_ADMIN_EXEC cp "$GUEST_VAGRANT_SSH_DIR/$CEPH_ADMIN_USER-id_rsa.pub" "$GUEST_USER_SSH_DIR/authorized_keys" -$CEPH_ADMIN_EXEC chmod 644 "$GUEST_USER_SSH_DIR/authorized_keys" - -# Make debconf non interactive and set the right local +# Make debconf non interactive export DEBIAN_FRONTEND=noninteractive +# Make sure we have the french locale +locale-gen fr_FR.UTF-8 -# Install ceph repository +# Install ceph repository (luminous version) wget -q -O- 'https://download.ceph.com/keys/release.asc' | apt-key add - -echo deb https://download.ceph.com/debian/ $(lsb_release -sc) main | tee /etc/apt/sources.list.d/ceph.list +echo deb https://download.ceph.com/debian-luminous/ $(lsb_release -sc) main | tee /etc/apt/sources.list.d/ceph.list apt-get update # Install chrony for time synchronization, gdisk for GPT partitioning, # vnstat for network stats, htop for system monitor and ceph-deploy apt-get -y install chrony gdisk vnstat htop ceph-deploy -# Modify /etc/hosts to allow ceph-deploy to resolve the guests -IP_ADDRESS=$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1) -# Need to replace the loopback address by the real address -sed -i "s/127.0.0.1\t$GUEST_NAME\t$GUEST_NAME/$IP_ADDRESS\t$GUEST_NAME\t$GUEST_NAME/g" /etc/hosts +# Full update +#apt-get -y dist-upgrade +#apt-get -y autoremove # Create partitions on journal disk for osd nodes only for NODE in $OSD_NODES; do @@ -88,16 +65,17 @@ for NODE in $OSD_NODES; do fi done -# Full update -#apt-get -y dist-upgrade -#apt-get -y autoremove +# Modify /etc/hosts to allow ceph-deploy to resolve the guests +IP_ADDRESS=$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1) +# Need to replace the loopback address by the real address +sed -i "s/127.0.0.1\t$GUEST_NAME\t$GUEST_NAME/$IP_ADDRESS\t$GUEST_NAME\t$GUEST_NAME/g" /etc/hosts +echo >> /etc/hosts # Signal that IP is ready echo -e "$IP_ADDRESS\t$GUEST_NAME" > "$GUEST_VAGRANT_SIGNAL_DIR/$GUEST_NAME-IP" # Wait for all nodes IP and update /etc/hosts TIMER_MAX=300 -echo >> /etc/hosts for NODE in $NODES; do if [[ $NODE != $GUEST_NAME ]]; then TIMER=0 @@ -116,25 +94,30 @@ for NODE in $NODES; do fi done +# Create user ceph-admin if not existing +cat /etc/passwd | grep $CEPH_ADMIN_USER || useradd -m -s /bin/bash $CEPH_ADMIN_USER + +# Make ceph-admin passwordless sudoer +echo "$CEPH_ADMIN_USER ALL = (root) NOPASSWD:ALL" | tee "/etc/sudoers.d/$CEPH_ADMIN_USER" +chmod 0440 "/etc/sudoers.d/$CEPH_ADMIN_USER" + +# Copy ceph-admin ssh keys and ssh config from Vagrant folder +# Keys must be created by pre-up script +# Executed in ceph admin context +sudo -i -u $CEPH_ADMIN_USER <&2 - exit 1 - fi - done -done - -# Install ceph in ceph-admin context -$CEPH_ADMIN_EXEC $GUEST_VAGRANT_SCRIPT_DIR/ceph-install.sh