First version with full cluster installation
This commit is contained in:
Denis Lambolez
parent
14893e7346
commit
00fac7322e
@ -1 +0,0 @@
|
||||
/virt/vagrant/cephtest
|
||||
@ -1 +0,0 @@
|
||||
/virt/vagrant/cephtest
|
||||
@ -1 +0,0 @@
|
||||
/virt/vagrant/cephtest
|
||||
10
Vagrantfile
vendored
10
Vagrantfile
vendored
@ -10,13 +10,8 @@ Vagrant.configure("2") do |config|
|
||||
run "pre-up.sh"
|
||||
end
|
||||
|
||||
# Trigger the Vagrant post-destroy script aftre destroying the last VM only
|
||||
# config.trigger.after :destroy, :vm => ["node-admin"], :append_to_path => ["#{vagrant_root}/scripts"] do
|
||||
run "post-destroy.sh"
|
||||
# end
|
||||
|
||||
# Shell provisionner for all VMs
|
||||
config.vm.provision "shell", path: "scripts/provision.sh"
|
||||
config.vm.provision "ceph-preflight", type: "shell", path: "scripts/provision.sh"
|
||||
|
||||
# All VMs are based on the same box
|
||||
config.vm.box = "bento/ubuntu-16.04"
|
||||
@ -37,7 +32,7 @@ Vagrant.configure("2") do |config|
|
||||
|
||||
# Standard configuration for all VMs
|
||||
config.vm.provider :libvirt do |libvirt|
|
||||
libvirt.memory = 1024
|
||||
libvirt.memory = 2048
|
||||
libvirt.volume_cache = "writeback"
|
||||
libvirt.graphics_type = "spice"
|
||||
libvirt.video_type = "qxl"
|
||||
@ -46,6 +41,7 @@ Vagrant.configure("2") do |config|
|
||||
# admin VM
|
||||
config.vm.define "node-admin", primary: true do |admin|
|
||||
admin.vm.hostname = "node-admin"
|
||||
admin.vm.provision "ceph-install", type: "shell", keep_color: true, path: "scripts/provision-admin.sh"
|
||||
end
|
||||
|
||||
# osd1 VM with private cluster network
|
||||
|
||||
@ -3,8 +3,8 @@
|
||||
# Execute ceph distributed storage installation steps from the admin node via ceph-deploy
|
||||
#
|
||||
# Written by : Denis Lambolez
|
||||
# Release : 1.0
|
||||
# Creation date : 04 December 2017
|
||||
# Release : 2.0
|
||||
# Creation date : 18 December 2017
|
||||
# Description : Bash script
|
||||
# This script has been designed and written on Ubuntu 16.04 plateform.
|
||||
# It must be executed in ceph-admin context, on admin node
|
||||
@ -15,46 +15,85 @@
|
||||
# HISTORY :
|
||||
# Release | Date | Authors | Description
|
||||
# --------------+---------------+--------------- +------------------------------------------
|
||||
# 2.0 | 12.18.17 | Denis Lambolez | Make it a standalone provisioner
|
||||
# 1.0 | 12.04.17 | Denis Lambolez | Creation
|
||||
# | | |
|
||||
# | | |
|
||||
# | | |
|
||||
# =========================================================================================
|
||||
#set -xv
|
||||
|
||||
# Version
|
||||
VERSION=ceph-install-1.0-120417
|
||||
VERSION=ceph-install-2.0-121817
|
||||
|
||||
# This script is executed in guest context
|
||||
source "/vagrant/scripts/cephtest-utils.sh"
|
||||
|
||||
# Make sure only root can run the script
|
||||
if [[ $(whoami) != $CEPH_ADMIN_USER ]]; then
|
||||
echo "This script must be run as $CEPH_ADMIN_USER" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Network (dynamically defined by Vagrant)
|
||||
IP_ADDRESS=$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)
|
||||
PUBLIC_NETWORK=$(echo $IP_ADDRESS | awk -F '.' '{print $1"."$2"."$3".0"}')
|
||||
|
||||
# Make sure only CEPH_ADMIN_USER can run the script
|
||||
if [[ $(whoami) != $CEPH_ADMIN_USER ]]; then
|
||||
echo "This script must be run as $CEPH_ADMIN_USER" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Make sure this script is run from the admin node
|
||||
if [[ $(hostname -s) != $ADMIN_NODE ]]; then
|
||||
echo "This script must be run from $ADMIN_NODE" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
PUBLIC_NETWORK=$(echo $IP_ADDRESS | awk -F '.' '{print $1"."$2"."$3".0/24"}')
|
||||
|
||||
# Cluster configuration directory
|
||||
mkdir -p "$GUEST_USER_DIR/ceph-cluster"
|
||||
cd "$GUEST_USER_DIR/ceph-cluster"
|
||||
|
||||
# Create cluster
|
||||
ceph-deploy new $ADMIN_NODE
|
||||
|
||||
# Initialize cluster configuration
|
||||
ceph-deploy new node-admin
|
||||
echo "public network = $PUBLIC_NETWORK" >> ceph.conf
|
||||
echo "cluster network = $CLUSTER_NETWORK" >> ceph.conf
|
||||
echo "" >> ceph.conf
|
||||
echo "osd pool default size = 2" >> ceph.conf
|
||||
echo "osd pool default min size = 1" >> ceph.conf
|
||||
cat << CLUSTERCONFIG >> ceph.conf
|
||||
public network = $PUBLIC_NETWORK
|
||||
cluster network = $CLUSTER_NETWORK
|
||||
|
||||
osd pool default size = 2
|
||||
osd pool default min size = 1
|
||||
osd pool default pg num = 256
|
||||
osd pool default pgp num = 256
|
||||
CLUSTERCONFIG
|
||||
|
||||
for NODE in $OSD_NODES; do
|
||||
cat << CLUSTERCONFIG >> ceph.conf
|
||||
|
||||
[mds.$NODE]
|
||||
mds standby replay = true
|
||||
mds standby for rank = 0
|
||||
CLUSTERCONFIG
|
||||
done
|
||||
|
||||
# Install ceph on all nodes
|
||||
ceph-deploy install --release luminous node-admin node-osd1 node-osd2
|
||||
ceph-deploy install --release luminous $NODES
|
||||
|
||||
# Create initial monitor
|
||||
ceph-deploy --overwrite-conf mon create-initial
|
||||
|
||||
# Deploy configuration file and client keys
|
||||
ceph-deploy admin $NODES
|
||||
|
||||
# Add monitor on osd nodes
|
||||
ceph-deploy mon add $OSD_NODES
|
||||
|
||||
# Create manager on all nodes
|
||||
#sudo mkdir -p /var/lib/ceph/mgr/ceph-$ADMIN_NODE
|
||||
#sudo chown ceph:ceph /var/lib/ceph/mgr/ceph-$ADMIN_NODE
|
||||
ceph-deploy mgr create $NODES
|
||||
|
||||
# Create metadata server on osd nodes
|
||||
ceph-deploy mds create $OSD_NODES
|
||||
|
||||
# For each osd node, gather keys from admin node and create OSDs
|
||||
for NODE in $OSD_NODES; do
|
||||
ssh $NODE ceph-deploy gatherkeys $ADMIN_NODE
|
||||
ssh $NODE sudo cp /home/$CEPH_ADMIN_USER/ceph.bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
|
||||
ssh $NODE sudo chown ceph:ceph /var/lib/ceph/bootstrap-osd/ceph.keyring
|
||||
ssh $NODE sudo ceph-volume lvm create --filestore --data /dev/vdb --journal /dev/vda1
|
||||
ssh $NODE sudo ceph-volume lvm create --filestore --data /dev/vdc --journal /dev/vda2
|
||||
done;
|
||||
|
||||
# wait 10 seconds and get cluster status
|
||||
sleep 10
|
||||
sudo ceph -s
|
||||
|
||||
@ -26,7 +26,6 @@ VERSION=cephtest-utils-1.0-120417
|
||||
|
||||
# Ceph user
|
||||
CEPH_ADMIN_USER="ceph-admin"
|
||||
CEPH_ADMIN_EXEC="sudo -i -u $CEPH_ADMIN_USER"
|
||||
|
||||
# Nodes
|
||||
ADMIN_NODE="node-admin"
|
||||
@ -34,7 +33,7 @@ OSD_NODES="node-osd1 node-osd2"
|
||||
NODES="$ADMIN_NODE $OSD_NODES"
|
||||
|
||||
# Networks
|
||||
CLUSTER_NETWORK="172.28.128.0"
|
||||
CLUSTER_NETWORK="172.28.128.0/24"
|
||||
|
||||
# Guest name
|
||||
GUEST_NAME=$(hostname -s)
|
||||
|
||||
@ -1,39 +0,0 @@
|
||||
#!/bin/bash
|
||||
# ========================================================================================
|
||||
# Execute post-destroy cleaning
|
||||
#
|
||||
# Written by : Denis Lambolez
|
||||
# Release : 1.0
|
||||
# Creation date : 16 December 2017
|
||||
# Description : Bash script
|
||||
# This script has been designed and written on Ubuntu 16.04 plateform.
|
||||
# It must be executed in vagrant context
|
||||
# Usage : ./post-destroy.sh
|
||||
# ----------------------------------------------------------------------------------------
|
||||
# ========================================================================================
|
||||
#
|
||||
# HISTORY :
|
||||
# Release | Date | Authors | Description
|
||||
# --------------+---------------+--------------- +------------------------------------------
|
||||
# 1.0 | 12.16.17 | Denis Lambolez | Creation
|
||||
# | | |
|
||||
# =========================================================================================
|
||||
#set -xev
|
||||
|
||||
# Version
|
||||
VERSION=post-destroy-1.0-121617
|
||||
|
||||
# This script is executed in host context
|
||||
source "$(dirname "$(readlink -f "$0")")/cephtest-utils.sh"
|
||||
|
||||
# clean-up networks to start with fresh configuration
|
||||
for NETWORK in vagrant-libvirt vagrant-private-dhcp; do
|
||||
virsh net-list --all 2> /dev/null | grep $NETWORK | grep active
|
||||
if [[ $? -eq 0 ]]; then
|
||||
virsh net-destroy $NETWORK 2> /dev/null
|
||||
fi
|
||||
virsh net-list --all 2> /dev/null | grep $NETWORK
|
||||
if [[ $? -eq 0 ]]; then
|
||||
virsh net-undefine $NETWORK 2> /dev/null
|
||||
fi
|
||||
done
|
||||
51
scripts/provision-admin.sh
Normal file
51
scripts/provision-admin.sh
Normal file
@ -0,0 +1,51 @@
|
||||
#!/bin/bash
|
||||
# ========================================================================================
|
||||
# Launch ceph installtion when cluster provisioning is done
|
||||
#
|
||||
# Written by : Denis Lambolez
|
||||
# Release : 1.0
|
||||
# Creation date : 18 December 2017
|
||||
# Description : Bash script
|
||||
# This script has been designed and written on Ubuntu 16.04 plateform.
|
||||
# It must be executed in vagrant context, on admin node
|
||||
# Usage : ./provision-admin.sh
|
||||
# ----------------------------------------------------------------------------------------
|
||||
# ========================================================================================
|
||||
#
|
||||
# HISTORY :
|
||||
# Release | Date | Authors | Description
|
||||
# --------------+---------------+--------------- +------------------------------------------
|
||||
# 1.0 | 12.18.17 | Denis Lambolez | Creation
|
||||
# | | |
|
||||
# | | |
|
||||
# =========================================================================================
|
||||
#set -xv
|
||||
|
||||
# Version
|
||||
VERSION=provision-admin-1.0-121817
|
||||
|
||||
# This script is executed in guest context
|
||||
source "/vagrant/scripts/cephtest-utils.sh"
|
||||
|
||||
# Make sure this script is run from the admin node
|
||||
if [[ $(hostname -s) != $ADMIN_NODE ]]; then
|
||||
echo "This script must be run from $ADMIN_NODE" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Wait for all nodes to be ready
|
||||
TIMER_MAX=300
|
||||
for NODE in $NODES; do
|
||||
TIMER=0
|
||||
until [[ -r "$GUEST_VAGRANT_SIGNAL_DIR/$NODE-PROVISION" ]]; do
|
||||
sleep 1
|
||||
TIMER=$(($TIMER + 1))
|
||||
if [[ $TIMER -gt $TIMER_MAX ]]; then
|
||||
echo "Waited too long for $NODE!" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# Launch ceph-installation
|
||||
sudo -i -u $CEPH_ADMIN_USER $GUEST_VAGRANT_SCRIPT_DIR/ceph-install.sh
|
||||
@ -36,46 +36,23 @@ if [[ $EUID -ne 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create user ceph-admin if not existing
|
||||
cat /etc/passwd | grep $CEPH_ADMIN_USER
|
||||
if [[ $? -ne 0 ]]; then
|
||||
useradd -m -s /bin/bash $CEPH_ADMIN_USER
|
||||
fi
|
||||
|
||||
# Make ceph-admin passwordless sudoer
|
||||
echo "$CEPH_ADMIN_USER ALL = (root) NOPASSWD:ALL" | tee "/etc/sudoers.d/$CEPH_ADMIN_USER"
|
||||
chmod 0440 "/etc/sudoers.d/$CEPH_ADMIN_USER"
|
||||
|
||||
# Copy ceph-admin ssh keys and ssh config from Vagrant synced folder (muste be created by vagrant-preflight script)
|
||||
$CEPH_ADMIN_EXEC mkdir -p "$GUEST_USER_SSH_DIR"
|
||||
$CEPH_ADMIN_EXEC chmod 700 "$GUEST_USER_SSH_DIR"
|
||||
for FILE in id_rsa id_rsa.pub config; do
|
||||
$CEPH_ADMIN_EXEC rm -f "$GUEST_USER_SSH_DIR/$FILE"
|
||||
$CEPH_ADMIN_EXEC cp "$GUEST_VAGRANT_SSH_DIR/$CEPH_ADMIN_USER-$FILE" "$GUEST_USER_SSH_DIR/$FILE"
|
||||
$CEPH_ADMIN_EXEC chmod 644 "$GUEST_USER_SSH_DIR/$FILE"
|
||||
done
|
||||
$CEPH_ADMIN_EXEC chmod 600 "$GUEST_USER_SSH_DIR/id_rsa"
|
||||
# Copy ceph-admin public key in authorized_keys
|
||||
$CEPH_ADMIN_EXEC rm -f "$GUEST_USER_SSH_DIR/authorized_keys"
|
||||
$CEPH_ADMIN_EXEC cp "$GUEST_VAGRANT_SSH_DIR/$CEPH_ADMIN_USER-id_rsa.pub" "$GUEST_USER_SSH_DIR/authorized_keys"
|
||||
$CEPH_ADMIN_EXEC chmod 644 "$GUEST_USER_SSH_DIR/authorized_keys"
|
||||
|
||||
# Make debconf non interactive and set the right local
|
||||
# Make debconf non interactive
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
# Make sure we have the french locale
|
||||
locale-gen fr_FR.UTF-8
|
||||
|
||||
# Install ceph repository
|
||||
# Install ceph repository (luminous version)
|
||||
wget -q -O- 'https://download.ceph.com/keys/release.asc' | apt-key add -
|
||||
echo deb https://download.ceph.com/debian/ $(lsb_release -sc) main | tee /etc/apt/sources.list.d/ceph.list
|
||||
echo deb https://download.ceph.com/debian-luminous/ $(lsb_release -sc) main | tee /etc/apt/sources.list.d/ceph.list
|
||||
apt-get update
|
||||
|
||||
# Install chrony for time synchronization, gdisk for GPT partitioning,
|
||||
# vnstat for network stats, htop for system monitor and ceph-deploy
|
||||
apt-get -y install chrony gdisk vnstat htop ceph-deploy
|
||||
|
||||
# Modify /etc/hosts to allow ceph-deploy to resolve the guests
|
||||
IP_ADDRESS=$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)
|
||||
# Need to replace the loopback address by the real address
|
||||
sed -i "s/127.0.0.1\t$GUEST_NAME\t$GUEST_NAME/$IP_ADDRESS\t$GUEST_NAME\t$GUEST_NAME/g" /etc/hosts
|
||||
# Full update
|
||||
#apt-get -y dist-upgrade
|
||||
#apt-get -y autoremove
|
||||
|
||||
# Create partitions on journal disk for osd nodes only
|
||||
for NODE in $OSD_NODES; do
|
||||
@ -88,16 +65,17 @@ for NODE in $OSD_NODES; do
|
||||
fi
|
||||
done
|
||||
|
||||
# Full update
|
||||
#apt-get -y dist-upgrade
|
||||
#apt-get -y autoremove
|
||||
# Modify /etc/hosts to allow ceph-deploy to resolve the guests
|
||||
IP_ADDRESS=$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)
|
||||
# Need to replace the loopback address by the real address
|
||||
sed -i "s/127.0.0.1\t$GUEST_NAME\t$GUEST_NAME/$IP_ADDRESS\t$GUEST_NAME\t$GUEST_NAME/g" /etc/hosts
|
||||
echo >> /etc/hosts
|
||||
|
||||
# Signal that IP is ready
|
||||
echo -e "$IP_ADDRESS\t$GUEST_NAME" > "$GUEST_VAGRANT_SIGNAL_DIR/$GUEST_NAME-IP"
|
||||
|
||||
# Wait for all nodes IP and update /etc/hosts
|
||||
TIMER_MAX=300
|
||||
echo >> /etc/hosts
|
||||
for NODE in $NODES; do
|
||||
if [[ $NODE != $GUEST_NAME ]]; then
|
||||
TIMER=0
|
||||
@ -116,25 +94,30 @@ for NODE in $NODES; do
|
||||
fi
|
||||
done
|
||||
|
||||
# Create user ceph-admin if not existing
|
||||
cat /etc/passwd | grep $CEPH_ADMIN_USER || useradd -m -s /bin/bash $CEPH_ADMIN_USER
|
||||
|
||||
# Make ceph-admin passwordless sudoer
|
||||
echo "$CEPH_ADMIN_USER ALL = (root) NOPASSWD:ALL" | tee "/etc/sudoers.d/$CEPH_ADMIN_USER"
|
||||
chmod 0440 "/etc/sudoers.d/$CEPH_ADMIN_USER"
|
||||
|
||||
# Copy ceph-admin ssh keys and ssh config from Vagrant folder
|
||||
# Keys must be created by pre-up script
|
||||
# Executed in ceph admin context
|
||||
sudo -i -u $CEPH_ADMIN_USER <<CEPHADMINBLOCK
|
||||
echo "Switch to $CEPH_ADMIN_USER context"
|
||||
mkdir -p $GUEST_USER_SSH_DIR
|
||||
chmod 700 $GUEST_USER_SSH_DIR
|
||||
cd $GUEST_USER_SSH_DIR
|
||||
rm -f id_rsa id_rsa.pub config authorized_keys
|
||||
cp $GUEST_VAGRANT_SSH_DIR/$CEPH_ADMIN_USER-id_rsa id_rsa
|
||||
cp $GUEST_VAGRANT_SSH_DIR/$CEPH_ADMIN_USER-id_rsa.pub id_rsa.pub
|
||||
cp $GUEST_VAGRANT_SSH_DIR/$CEPH_ADMIN_USER-id_rsa.pub authorized_keys
|
||||
cp $GUEST_VAGRANT_SSH_DIR/$CEPH_ADMIN_USER-config config
|
||||
chmod 644 id_rsa.pub config authorized_keys
|
||||
chmod 600 id_rsa
|
||||
CEPHADMINBLOCK
|
||||
echo "Switch to $(whoami) context"
|
||||
|
||||
# Signal that provision is done
|
||||
echo "$(date --rfc-3339=ns) - Done!" | tee "$GUEST_VAGRANT_SIGNAL_DIR/$GUEST_NAME-PROVISION"
|
||||
|
||||
# Continue provisioning on admin node only
|
||||
[[ $GUEST_NAME != $ADMIN_NODE ]] && exit 0
|
||||
|
||||
# Wait for all nodes to be ready
|
||||
TIMER_MAX=300
|
||||
for NODE in $NODES; do
|
||||
TIMER=0
|
||||
until [[ -r "$GUEST_VAGRANT_SIGNAL_DIR/$NODE-PROVISION" ]]; do
|
||||
sleep 1
|
||||
TIMER=$(($TIMER + 1))
|
||||
if [[ $TIMER -gt $TIMER_MAX ]]; then
|
||||
echo "Waited too long for $NODE!" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# Install ceph in ceph-admin context
|
||||
$CEPH_ADMIN_EXEC $GUEST_VAGRANT_SCRIPT_DIR/ceph-install.sh
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user