First version with full cluster installation

.vagrant/machines/admin/libvirt/vagrant_cwd

@@ -1 +0,0 @@
-/virt/vagrant/cephtest

.vagrant/machines/osd1/libvirt/vagrant_cwd

@@ -1 +0,0 @@
-/virt/vagrant/cephtest

.vagrant/machines/osd2/libvirt/vagrant_cwd

@@ -1 +0,0 @@
-/virt/vagrant/cephtest

Vagrantfile

@@ -10,13 +10,8 @@ Vagrant.configure("2") do |config|
 		run "pre-up.sh"
 	end
 
-	# Trigger the Vagrant post-destroy script aftre destroying the last VM only
-	# config.trigger.after :destroy, :vm => ["node-admin"], :append_to_path => ["#{vagrant_root}/scripts"] do
-		run "post-destroy.sh"
-	# end
-
 	# Shell provisionner for all VMs
-	config.vm.provision "shell", path: "scripts/provision.sh"
+	config.vm.provision "ceph-preflight", type: "shell", path: "scripts/provision.sh"
 	
 	# All VMs are based on the same box
 	config.vm.box = "bento/ubuntu-16.04"
@@ -37,7 +32,7 @@ Vagrant.configure("2") do |config|
 
 	# Standard configuration for all VMs
 	config.vm.provider :libvirt do |libvirt|
-		libvirt.memory = 1024
+		libvirt.memory = 2048
 		libvirt.volume_cache = "writeback"
 		libvirt.graphics_type = "spice"
 		libvirt.video_type = "qxl"
@@ -46,6 +41,7 @@ Vagrant.configure("2") do |config|
 	# admin VM
 	config.vm.define "node-admin", primary: true do |admin|
 		admin.vm.hostname = "node-admin"
+		admin.vm.provision "ceph-install", type: "shell", keep_color: true, path: "scripts/provision-admin.sh"
 	end
 
 	# osd1 VM with private cluster network

scripts/ceph-install.sh

@@ -3,8 +3,8 @@
 # Execute ceph distributed storage installation steps from the admin node via ceph-deploy
 #
 # Written by    : Denis Lambolez
-# Release       : 1.0
+# Release       : 2.0
-# Creation date : 04 December 2017
+# Creation date : 18 December 2017
 # Description   : Bash script
 #                 This script has been designed and written on Ubuntu 16.04 plateform.
 #                 It must be executed in ceph-admin context, on admin node 
@@ -15,46 +15,85 @@
 # HISTORY :
 #     Release   |     Date      |    Authors     |       Description
 # --------------+---------------+--------------- +------------------------------------------
+#       2.0     |    12.18.17   | Denis Lambolez | Make it a standalone provisioner
 #       1.0     |    12.04.17   | Denis Lambolez | Creation
 #               |               |                | 
 #               |               |                | 
-#               |               |                | 
 # =========================================================================================
 #set -xv
 
 # Version
-VERSION=ceph-install-1.0-120417
+VERSION=ceph-install-2.0-121817
 
 # This script is executed in guest context
 source "/vagrant/scripts/cephtest-utils.sh"
 
+# Make sure only root can run the script
+if [[ $(whoami) != $CEPH_ADMIN_USER ]]; then
+   echo "This script must be run as $CEPH_ADMIN_USER" >&2
+   exit 1
+fi
+
 # Network (dynamically defined by Vagrant)
 IP_ADDRESS=$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)
-PUBLIC_NETWORK=$(echo $IP_ADDRESS | awk -F '.' '{print $1"."$2"."$3".0"}')
+PUBLIC_NETWORK=$(echo $IP_ADDRESS | awk -F '.' '{print $1"."$2"."$3".0/24"}')
-
-# Make sure only CEPH_ADMIN_USER can run the script
-if [[ $(whoami) != $CEPH_ADMIN_USER ]]; then
-   echo "This script must be run as $CEPH_ADMIN_USER" 1>&2
-   exit 1
-fi
-
-# Make sure this script is run from the admin node
-if [[ $(hostname -s) != $ADMIN_NODE ]]; then
-   echo "This script must be run from $ADMIN_NODE" 1>&2
-   exit 1
-fi
 
 # Cluster configuration directory
 mkdir -p "$GUEST_USER_DIR/ceph-cluster"
 cd "$GUEST_USER_DIR/ceph-cluster"
 
+# Create cluster
+ceph-deploy new $ADMIN_NODE
+
 # Initialize cluster configuration
-ceph-deploy new node-admin
+cat << CLUSTERCONFIG >> ceph.conf
-echo "public network = $PUBLIC_NETWORK" >> ceph.conf
+public network = $PUBLIC_NETWORK
-echo "cluster network = $CLUSTER_NETWORK" >> ceph.conf
+cluster network = $CLUSTER_NETWORK
-echo "" >> ceph.conf
+
-echo "osd pool default size = 2" >> ceph.conf
+osd pool default size = 2
-echo "osd pool default min size = 1" >> ceph.conf
+osd pool default min size = 1
+osd pool default pg num = 256 
+osd pool default pgp num = 256
+CLUSTERCONFIG
+
+for NODE in $OSD_NODES; do
+cat << CLUSTERCONFIG >> ceph.conf
+
+[mds.$NODE]
+mds standby replay = true
+mds standby for rank = 0
+CLUSTERCONFIG
+done
 
 # Install ceph on all nodes
-ceph-deploy install --release luminous node-admin node-osd1 node-osd2
+ceph-deploy install --release luminous $NODES
+
+# Create initial monitor
+ceph-deploy --overwrite-conf mon create-initial
+
+# Deploy configuration file and client keys
+ceph-deploy admin $NODES
+
+# Add monitor on osd nodes
+ceph-deploy mon add $OSD_NODES
+
+# Create manager on all nodes
+#sudo mkdir -p /var/lib/ceph/mgr/ceph-$ADMIN_NODE
+#sudo chown ceph:ceph /var/lib/ceph/mgr/ceph-$ADMIN_NODE
+ceph-deploy mgr create $NODES
+
+# Create metadata server on osd nodes
+ceph-deploy mds create $OSD_NODES
+
+# For each osd node, gather keys from admin node and create OSDs
+for NODE in $OSD_NODES; do
+    ssh $NODE ceph-deploy gatherkeys $ADMIN_NODE
+    ssh $NODE sudo cp /home/$CEPH_ADMIN_USER/ceph.bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
+    ssh $NODE sudo chown ceph:ceph /var/lib/ceph/bootstrap-osd/ceph.keyring
+    ssh $NODE sudo ceph-volume lvm create --filestore --data /dev/vdb --journal /dev/vda1
+    ssh $NODE sudo ceph-volume lvm create --filestore --data /dev/vdc --journal /dev/vda2
+done;
+
+# wait 10 seconds and get cluster status
+sleep 10
+sudo ceph -s 

scripts/cephtest-utils.sh

@@ -26,7 +26,6 @@ VERSION=cephtest-utils-1.0-120417
 
 # Ceph user
 CEPH_ADMIN_USER="ceph-admin"
-CEPH_ADMIN_EXEC="sudo -i -u $CEPH_ADMIN_USER"
 
 # Nodes
 ADMIN_NODE="node-admin"
@@ -34,7 +33,7 @@ OSD_NODES="node-osd1 node-osd2"
 NODES="$ADMIN_NODE $OSD_NODES"
 
 # Networks
-CLUSTER_NETWORK="172.28.128.0"
+CLUSTER_NETWORK="172.28.128.0/24"
 
 # Guest name
 GUEST_NAME=$(hostname -s)

scripts/post-destroy.sh

@@ -1,39 +0,0 @@
-#!/bin/bash
-# ========================================================================================
-# Execute post-destroy cleaning
-#
-# Written by    : Denis Lambolez
-# Release       : 1.0
-# Creation date : 16 December 2017
-# Description   : Bash script
-#                 This script has been designed and written on Ubuntu 16.04 plateform.
-#                 It must be executed in vagrant context
-# Usage         : ./post-destroy.sh 
-# ----------------------------------------------------------------------------------------
-# ========================================================================================
-#
-# HISTORY :
-#     Release   |     Date      |    Authors     |       Description
-# --------------+---------------+--------------- +------------------------------------------
-#       1.0     |    12.16.17   | Denis Lambolez | Creation
-#               |               |                | 
-# =========================================================================================
-#set -xev
-
-# Version
-VERSION=post-destroy-1.0-121617
-
-# This script is executed in host context
-source "$(dirname "$(readlink -f "$0")")/cephtest-utils.sh"
-
-# clean-up networks to start with fresh configuration
-for NETWORK in vagrant-libvirt vagrant-private-dhcp; do
-    virsh net-list --all 2> /dev/null | grep $NETWORK | grep active
-    if [[ $? -eq 0 ]]; then
-        virsh net-destroy $NETWORK  2> /dev/null
-    fi
-    virsh net-list --all  2> /dev/null | grep $NETWORK
-    if [[ $? -eq 0 ]]; then
-        virsh net-undefine $NETWORK  2> /dev/null
-    fi
-done

scripts/provision-admin.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# ========================================================================================
+# Launch ceph installtion when cluster provisioning is done
+#
+# Written by    : Denis Lambolez
+# Release       : 1.0
+# Creation date : 18 December 2017
+# Description   : Bash script
+#                 This script has been designed and written on Ubuntu 16.04 plateform.
+#                 It must be executed in vagrant context, on admin node 
+# Usage         : ./provision-admin.sh
+# ----------------------------------------------------------------------------------------
+# ========================================================================================
+#
+# HISTORY :
+#     Release   |     Date      |    Authors     |       Description
+# --------------+---------------+--------------- +------------------------------------------
+#       1.0     |    12.18.17   | Denis Lambolez | Creation
+#               |               |                | 
+#               |               |                | 
+# =========================================================================================
+#set -xv
+
+# Version
+VERSION=provision-admin-1.0-121817
+
+# This script is executed in guest context
+source "/vagrant/scripts/cephtest-utils.sh"
+
+# Make sure this script is run from the admin node
+if [[ $(hostname -s) != $ADMIN_NODE ]]; then
+   echo "This script must be run from $ADMIN_NODE" 1>&2
+   exit 1
+fi
+
+# Wait for all nodes to be ready
+TIMER_MAX=300
+for NODE in $NODES; do
+    TIMER=0
+    until [[ -r "$GUEST_VAGRANT_SIGNAL_DIR/$NODE-PROVISION" ]]; do
+        sleep 1
+        TIMER=$(($TIMER + 1))
+        if [[ $TIMER -gt $TIMER_MAX ]]; then
+            echo "Waited too long for $NODE!" >&2
+            exit 1
+        fi
+    done
+done
+
+# Launch ceph-installation
+sudo -i -u $CEPH_ADMIN_USER $GUEST_VAGRANT_SCRIPT_DIR/ceph-install.sh

scripts/provision.sh

@@ -36,46 +36,23 @@ if [[ $EUID -ne 0 ]]; then
    exit 1
 fi
 
-# Create user ceph-admin if not existing
+# Make debconf non interactive
-cat /etc/passwd | grep $CEPH_ADMIN_USER
-if [[ $? -ne 0 ]]; then
-    useradd -m -s /bin/bash $CEPH_ADMIN_USER
-fi
-
-# Make ceph-admin passwordless sudoer
-echo "$CEPH_ADMIN_USER ALL = (root) NOPASSWD:ALL" | tee "/etc/sudoers.d/$CEPH_ADMIN_USER"
-chmod 0440 "/etc/sudoers.d/$CEPH_ADMIN_USER"
-
-# Copy ceph-admin ssh keys and ssh config from Vagrant synced folder (muste be created by vagrant-preflight script)
-$CEPH_ADMIN_EXEC mkdir -p "$GUEST_USER_SSH_DIR"
-$CEPH_ADMIN_EXEC chmod 700 "$GUEST_USER_SSH_DIR"
-for FILE in id_rsa id_rsa.pub config; do
-    $CEPH_ADMIN_EXEC rm -f "$GUEST_USER_SSH_DIR/$FILE"
-    $CEPH_ADMIN_EXEC cp "$GUEST_VAGRANT_SSH_DIR/$CEPH_ADMIN_USER-$FILE" "$GUEST_USER_SSH_DIR/$FILE"
-    $CEPH_ADMIN_EXEC chmod 644 "$GUEST_USER_SSH_DIR/$FILE"
-done  
-$CEPH_ADMIN_EXEC chmod 600 "$GUEST_USER_SSH_DIR/id_rsa"
-# Copy ceph-admin public key in authorized_keys 
-$CEPH_ADMIN_EXEC rm -f "$GUEST_USER_SSH_DIR/authorized_keys"
-$CEPH_ADMIN_EXEC cp "$GUEST_VAGRANT_SSH_DIR/$CEPH_ADMIN_USER-id_rsa.pub" "$GUEST_USER_SSH_DIR/authorized_keys"
-$CEPH_ADMIN_EXEC chmod 644 "$GUEST_USER_SSH_DIR/authorized_keys"
-
-# Make debconf non interactive and set the right local
 export DEBIAN_FRONTEND=noninteractive
+# Make sure we have the french locale
+locale-gen fr_FR.UTF-8
 
-# Install ceph repository 
+# Install ceph repository (luminous version)
 wget -q -O- 'https://download.ceph.com/keys/release.asc' | apt-key add -
-echo deb https://download.ceph.com/debian/ $(lsb_release -sc) main | tee /etc/apt/sources.list.d/ceph.list
+echo deb https://download.ceph.com/debian-luminous/ $(lsb_release -sc) main | tee /etc/apt/sources.list.d/ceph.list
 apt-get update
 
 # Install chrony for time synchronization, gdisk for GPT partitioning, 
 # vnstat for network stats, htop for system monitor and ceph-deploy
 apt-get -y install chrony gdisk vnstat htop ceph-deploy
 
-# Modify /etc/hosts to allow ceph-deploy to resolve the guests
+# Full update
-IP_ADDRESS=$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)
+#apt-get -y dist-upgrade
-# Need to replace the loopback address by the real address
+#apt-get -y autoremove
-sed -i "s/127.0.0.1\t$GUEST_NAME\t$GUEST_NAME/$IP_ADDRESS\t$GUEST_NAME\t$GUEST_NAME/g" /etc/hosts
 
 # Create partitions on journal disk for osd nodes only
 for NODE in $OSD_NODES; do
@@ -88,16 +65,17 @@ for NODE in $OSD_NODES; do
     fi
 done
 
-# Full update
+# Modify /etc/hosts to allow ceph-deploy to resolve the guests
-#apt-get -y dist-upgrade
+IP_ADDRESS=$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)
-#apt-get -y autoremove
+# Need to replace the loopback address by the real address
+sed -i "s/127.0.0.1\t$GUEST_NAME\t$GUEST_NAME/$IP_ADDRESS\t$GUEST_NAME\t$GUEST_NAME/g" /etc/hosts
+echo >> /etc/hosts
 
 # Signal that IP is ready
 echo -e "$IP_ADDRESS\t$GUEST_NAME" > "$GUEST_VAGRANT_SIGNAL_DIR/$GUEST_NAME-IP"
 
 # Wait for all nodes IP and update /etc/hosts
 TIMER_MAX=300
-echo >> /etc/hosts
 for NODE in $NODES; do
     if [[ $NODE != $GUEST_NAME ]]; then
         TIMER=0
@@ -116,25 +94,30 @@ for NODE in $NODES; do
     fi
 done
 
+# Create user ceph-admin if not existing
+cat /etc/passwd | grep $CEPH_ADMIN_USER || useradd -m -s /bin/bash $CEPH_ADMIN_USER
+
+# Make ceph-admin passwordless sudoer
+echo "$CEPH_ADMIN_USER ALL = (root) NOPASSWD:ALL" | tee "/etc/sudoers.d/$CEPH_ADMIN_USER"
+chmod 0440 "/etc/sudoers.d/$CEPH_ADMIN_USER"
+
+# Copy ceph-admin ssh keys and ssh config from Vagrant folder
+# Keys must be created by pre-up script
+# Executed in ceph admin context
+sudo -i -u $CEPH_ADMIN_USER <<CEPHADMINBLOCK
+    echo "Switch to $CEPH_ADMIN_USER context"
+    mkdir -p $GUEST_USER_SSH_DIR
+    chmod 700 $GUEST_USER_SSH_DIR
+    cd $GUEST_USER_SSH_DIR
+    rm -f id_rsa id_rsa.pub config authorized_keys
+    cp $GUEST_VAGRANT_SSH_DIR/$CEPH_ADMIN_USER-id_rsa id_rsa
+    cp $GUEST_VAGRANT_SSH_DIR/$CEPH_ADMIN_USER-id_rsa.pub id_rsa.pub
+    cp $GUEST_VAGRANT_SSH_DIR/$CEPH_ADMIN_USER-id_rsa.pub authorized_keys
+    cp $GUEST_VAGRANT_SSH_DIR/$CEPH_ADMIN_USER-config config
+    chmod 644 id_rsa.pub config authorized_keys
+    chmod 600 id_rsa
+CEPHADMINBLOCK
+echo "Switch to $(whoami) context"
+
 # Signal that provision is done
 echo "$(date --rfc-3339=ns) - Done!"  | tee "$GUEST_VAGRANT_SIGNAL_DIR/$GUEST_NAME-PROVISION"
-
-# Continue provisioning on admin node only
-[[ $GUEST_NAME != $ADMIN_NODE ]] && exit 0
-
-# Wait for all nodes to be ready
-TIMER_MAX=300
-for NODE in $NODES; do
-    TIMER=0
-    until [[ -r "$GUEST_VAGRANT_SIGNAL_DIR/$NODE-PROVISION" ]]; do
-        sleep 1
-        TIMER=$(($TIMER + 1))
-        if [[ $TIMER -gt $TIMER_MAX ]]; then
-            echo "Waited too long for $NODE!" >&2
-            exit 1
-        fi
-    done
-done
- 
-# Install ceph in ceph-admin context
-$CEPH_ADMIN_EXEC $GUEST_VAGRANT_SCRIPT_DIR/ceph-install.sh