From 7c2f8b188e6acc8089efb76357ea2f66c83aa2c7 Mon Sep 17 00:00:00 2001 From: Cory Hawkless Date: Tue, 23 Mar 2021 20:35:29 +1030 Subject: [PATCH] Working v1 --- Vagrantfile | 40 +- log.log | 3231 +++++++++++++++++++++++++++++-------- scripts/ceph-install.sh | 5 +- scripts/cephtest-utils.sh | 2 +- 4 files changed, 2600 insertions(+), 678 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 8f9d0d7..f3781a5 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -4,7 +4,7 @@ Vagrant.configure("2") do |config| # Get the VagrantFile directory vagrant_root = File.dirname(__FILE__) - + # Trigger the Vagrant pre-up script before uping the first VM only @@ -34,6 +34,8 @@ Vagrant.configure("2") do |config| libvirt.volume_cache = "writeback" libvirt.graphics_type = "spice" libvirt.video_type = "qxl" + libvirt.storage_pool_name="cephlab" +# libvirt.snapshot_pool_name="cephlab" end # admin VM @@ -59,15 +61,15 @@ Vagrant.configure("2") do |config| :libvirt__network_name => "cluster-network" osd1.vm.provider :libvirt do |libvirt| libvirt.storage :file, - :size => "20G", + :size => "10G", :type => "raw", :cache => "writeback" libvirt.storage :file, - :size => "40G", + :size => "10G", :type => "raw", :cache => "writeback" libvirt.storage :file, - :size => "40G", + :size => "10G", :type => "raw", :cache => "writeback" end @@ -83,17 +85,41 @@ Vagrant.configure("2") do |config| :libvirt__network_name => "cluster-network" osd2.vm.provider :libvirt do |libvirt| libvirt.storage :file, - :size => "20G", + :size => "10G", :type => "raw", :cache => "writeback" libvirt.storage :file, - :size => "40G", + :size => "10G", :type => "raw", :cache => "writeback" libvirt.storage :file, - :size => "40G", + :size => "10G", :type => "raw", :cache => "writeback" end end + + # osd3 VM with private cluster network + # 3 additional disks: 1 for journals and 2 for osd + config.vm.define "node-osd3" do |osd3| + osd3.vm.hostname = "node-osd3" + osd3.vm.network :private_network, + :type => "dhcp", + :libvirt__network_name => "cluster-network" + osd3.vm.provider :libvirt do |libvirt| + libvirt.storage :file, + :size => "10G", + :type => "raw", + :cache => "writeback" + libvirt.storage :file, + :size => "10G", + :type => "raw", + :cache => "writeback" + libvirt.storage :file, + :size => "10G", + :type => "raw", + :cache => "writeback" + end + end + end diff --git a/log.log b/log.log index 628a8d1..40696c8 100644 --- a/log.log +++ b/log.log @@ -1,28 +1,47 @@ Bringing machine 'node-admin' up with 'libvirt' provider... Bringing machine 'node-osd1' up with 'libvirt' provider... Bringing machine 'node-osd2' up with 'libvirt' provider... +Bringing machine 'node-osd3' up with 'libvirt' provider... ==> node-admin: Running action triggers before up ... ==> node-admin: Running trigger... node-admin: Running local script: scripts/pre-up.sh +==> node-osd3: Checking if box 'generic/ubuntu1804' version '3.2.10' is up to date... node-admin: node-admin: {cephtest} {pre-up} Create SSH keys and config for ceph admin user -==> node-osd1: Checking if box 'generic/ubuntu1804' version '3.0.12' is up to date... -==> node-osd2: Checking if box 'generic/ubuntu1804' version '3.0.12' is up to date... - node-admin: Host node-admin - node-admin: Hostname node-admin - node-admin: User ceph-admin - node-admin: StrictHostKeyChecking no - node-admin: Host node-osd1 - node-admin: Hostname node-osd1 - node-admin: User ceph-admin - node-admin: StrictHostKeyChecking no - node-admin: Host node-osd2 - node-admin: Hostname node-osd2 - node-admin: User ceph-admin - node-admin: StrictHostKeyChecking no - node-admin: - node-admin: {cephtest} {pre-up} Clean up IP and PROVISION signals -==> node-admin: Checking if box 'generic/ubuntu1804' version '3.0.12' is up to date... +==> node-osd2: Checking if box 'generic/ubuntu1804' version '3.2.10' is up to date... +==> node-osd1: Checking if box 'generic/ubuntu1804' version '3.2.10' is up to date... +==> node-osd3: Creating image (snapshot of base box volume). +==> node-osd3: Creating domain with the following settings... +==> node-osd3: -- Name: cephtest_node-osd3 +==> node-osd3: -- Domain type: kvm +==> node-osd3: -- Cpus: 2 +==> node-osd3: -- Feature: acpi +==> node-osd3: -- Feature: apic +==> node-osd3: -- Feature: pae +==> node-osd3: -- Memory: 2048M +==> node-osd3: -- Management MAC: +==> node-osd3: -- Loader: +==> node-osd3: -- Nvram: +==> node-osd3: -- Base box: generic/ubuntu1804 +==> node-osd3: -- Storage pool: default +==> node-osd3: -- Image: /var/lib/libvirt/images/cephtest_node-osd3.img (128G) +==> node-osd3: -- Volume Cache: writeback +==> node-osd3: -- Kernel: +==> node-osd3: -- Initrd: +==> node-osd3: -- Graphics Type: spice +==> node-osd3: -- Graphics Port: -1 +==> node-osd3: -- Graphics IP: 127.0.0.1 +==> node-osd3: -- Graphics Password: Not defined +==> node-osd3: -- Video Type: qxl +==> node-osd3: -- Video VRAM: 256 +==> node-osd3: -- Sound Type: +==> node-osd3: -- Keymap: en-us +==> node-osd3: -- TPM Path: +==> node-osd3: -- Disks: vdb(raw,10G), vdc(raw,10G), vdd(raw,10G) +==> node-osd3: -- Disk(vdb): /var/lib/libvirt/images/cephtest_node-osd3-vdb.raw +==> node-osd3: -- Disk(vdc): /var/lib/libvirt/images/cephtest_node-osd3-vdc.raw +==> node-osd3: -- Disk(vdd): /var/lib/libvirt/images/cephtest_node-osd3-vdd.raw +==> node-osd3: -- INPUT: type=mouse, bus=ps2 ==> node-osd2: Creating image (snapshot of base box volume). ==> node-osd2: Creating domain with the following settings... ==> node-osd2: -- Name: cephtest_node-osd2 @@ -37,7 +56,7 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... ==> node-osd2: -- Nvram: ==> node-osd2: -- Base box: generic/ubuntu1804 ==> node-osd2: -- Storage pool: default -==> node-osd2: -- Image: /var/lib/libvirt/images/cephtest_node-osd2.img (32G) +==> node-osd2: -- Image: /var/lib/libvirt/images/cephtest_node-osd2.img (128G) ==> node-osd2: -- Volume Cache: writeback ==> node-osd2: -- Kernel: ==> node-osd2: -- Initrd: @@ -50,44 +69,11 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... ==> node-osd2: -- Sound Type: ==> node-osd2: -- Keymap: en-us ==> node-osd2: -- TPM Path: -==> node-osd2: -- Disks: vdb(raw,20G), vdc(raw,40G), vdd(raw,40G) +==> node-osd2: -- Disks: vdb(raw,10G), vdc(raw,10G), vdd(raw,10G) ==> node-osd2: -- Disk(vdb): /var/lib/libvirt/images/cephtest_node-osd2-vdb.raw ==> node-osd2: -- Disk(vdc): /var/lib/libvirt/images/cephtest_node-osd2-vdc.raw ==> node-osd2: -- Disk(vdd): /var/lib/libvirt/images/cephtest_node-osd2-vdd.raw ==> node-osd2: -- INPUT: type=mouse, bus=ps2 -==> node-admin: Creating image (snapshot of base box volume). -==> node-admin: Creating domain with the following settings... -==> node-admin: -- Name: cephtest_node-admin -==> node-admin: -- Domain type: kvm -==> node-admin: -- Cpus: 2 -==> node-admin: -- Feature: acpi -==> node-admin: -- Feature: apic -==> node-admin: -- Feature: pae -==> node-admin: -- Memory: 2048M -==> node-admin: -- Management MAC: -==> node-admin: -- Loader: -==> node-admin: -- Nvram: -==> node-admin: -- Base box: generic/ubuntu1804 -==> node-admin: -- Storage pool: default -==> node-admin: -- Image: /var/lib/libvirt/images/cephtest_node-admin.img (32G) -==> node-admin: -- Volume Cache: writeback -==> node-admin: -- Kernel: -==> node-admin: -- Initrd: -==> node-admin: -- Graphics Type: spice -==> node-admin: -- Graphics Port: -1 -==> node-admin: -- Graphics IP: 127.0.0.1 -==> node-admin: -- Graphics Password: Not defined -==> node-admin: -- Video Type: qxl -==> node-admin: -- Video VRAM: 256 -==> node-admin: -- Sound Type: -==> node-admin: -- Keymap: en-us -==> node-admin: -- TPM Path: -==> node-admin: -- INPUT: type=mouse, bus=ps2 -==> node-osd2: Pruning invalid NFS exports. Administrator privileges will be required... -==> node-admin: Pruning invalid NFS exports. Administrator privileges will be required... -==> node-osd1: A newer version of the box 'generic/ubuntu1804' for provider 'libvirt' is -==> node-osd1: available! You currently have version '3.0.12'. The latest is version -==> node-osd1: '3.2.10'. Run `vagrant box update` to update. ==> node-osd1: Creating image (snapshot of base box volume). ==> node-osd1: Creating domain with the following settings... ==> node-osd1: -- Name: cephtest_node-osd1 @@ -102,7 +88,7 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... ==> node-osd1: -- Nvram: ==> node-osd1: -- Base box: generic/ubuntu1804 ==> node-osd1: -- Storage pool: default -==> node-osd1: -- Image: /var/lib/libvirt/images/cephtest_node-osd1.img (32G) +==> node-osd1: -- Image: /var/lib/libvirt/images/cephtest_node-osd1.img (128G) ==> node-osd1: -- Volume Cache: writeback ==> node-osd1: -- Kernel: ==> node-osd1: -- Initrd: @@ -115,111 +101,180 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... ==> node-osd1: -- Sound Type: ==> node-osd1: -- Keymap: en-us ==> node-osd1: -- TPM Path: -==> node-osd1: -- Disks: vdb(raw,20G), vdc(raw,40G), vdd(raw,40G) +==> node-osd1: -- Disks: vdb(raw,10G), vdc(raw,10G), vdd(raw,10G) ==> node-osd1: -- Disk(vdb): /var/lib/libvirt/images/cephtest_node-osd1-vdb.raw ==> node-osd1: -- Disk(vdc): /var/lib/libvirt/images/cephtest_node-osd1-vdc.raw ==> node-osd1: -- Disk(vdd): /var/lib/libvirt/images/cephtest_node-osd1-vdd.raw ==> node-osd1: -- INPUT: type=mouse, bus=ps2 +==> node-osd3: Pruning invalid NFS exports. Administrator privileges will be required... +==> node-osd2: Pruning invalid NFS exports. Administrator privileges will be required... +==> node-osd1: Pruning invalid NFS exports. Administrator privileges will be required... +==> node-osd2: Creating shared folders metadata... +==> node-osd2: Starting domain. +==> node-osd2: Waiting for domain to get an IP address... +==> node-osd1: Creating shared folders metadata... +==> node-osd3: Creating shared folders metadata... +==> node-osd1: Starting domain. +==> node-osd3: Starting domain. +==> node-osd1: Waiting for domain to get an IP address... +==> node-osd3: Waiting for domain to get an IP address... +==> node-osd2: Waiting for SSH to become available... + node-osd2: + node-osd2: Vagrant insecure key detected. Vagrant will automatically replace + node-osd2: this with a newly generated keypair for better security. +==> node-osd1: Waiting for SSH to become available... +==> node-osd3: Waiting for SSH to become available... + node-osd2: + node-osd2: Inserting generated public key within guest... + node-osd2: Removing insecure key from the guest if it's present... + node-osd2: Key inserted! Disconnecting and reconnecting using new SSH key... +==> node-osd2: Setting hostname... + node-osd1: + node-osd1: Vagrant insecure key detected. Vagrant will automatically replace + node-osd1: this with a newly generated keypair for better security. + node-osd3: + node-osd3: Vagrant insecure key detected. Vagrant will automatically replace + node-osd3: this with a newly generated keypair for better security. +==> node-osd2: Configuring and enabling network interfaces... + node-osd1: + node-osd1: Inserting generated public key within guest... + node-osd3: + node-osd3: Inserting generated public key within guest... + node-osd1: Removing insecure key from the guest if it's present... + node-osd3: Removing insecure key from the guest if it's present... + node-osd1: Key inserted! Disconnecting and reconnecting using new SSH key... + node-osd3: Key inserted! Disconnecting and reconnecting using new SSH key... +==> node-osd1: Setting hostname... +==> node-osd3: Setting hostname... +==> node-osd2: Installing NFS client... +==> node-osd1: Configuring and enabling network interfaces... +==> node-osd3: Configuring and enabling network interfaces... +==> node-osd1: Installing NFS client... +==> node-osd3: Installing NFS client... + node-admin: Host node-admin + node-admin: Hostname node-admin + node-admin: User ceph-admin + node-admin: StrictHostKeyChecking no + node-admin: Host node-osd1 + node-admin: Hostname node-osd1 + node-admin: User ceph-admin + node-admin: StrictHostKeyChecking no + node-admin: Host node-osd2 + node-admin: Hostname node-osd2 + node-admin: User ceph-admin + node-admin: StrictHostKeyChecking no + node-admin: Host node-osd3 + node-admin: Hostname node-osd3 + node-admin: User ceph-admin + node-admin: StrictHostKeyChecking no + node-admin: + node-admin: {cephtest} {pre-up} Clean up IP and PROVISION signals +==> node-admin: Checking if box 'generic/ubuntu1804' version '3.2.10' is up to date... +==> node-admin: Creating image (snapshot of base box volume). +==> node-admin: Creating domain with the following settings... +==> node-admin: -- Name: cephtest_node-admin +==> node-admin: -- Domain type: kvm +==> node-admin: -- Cpus: 2 +==> node-admin: -- Feature: acpi +==> node-admin: -- Feature: apic +==> node-admin: -- Feature: pae +==> node-admin: -- Memory: 2048M +==> node-admin: -- Management MAC: +==> node-admin: -- Loader: +==> node-admin: -- Nvram: +==> node-admin: -- Base box: generic/ubuntu1804 +==> node-admin: -- Storage pool: default +==> node-admin: -- Image: /var/lib/libvirt/images/cephtest_node-admin.img (128G) +==> node-admin: -- Volume Cache: writeback +==> node-admin: -- Kernel: +==> node-admin: -- Initrd: +==> node-admin: -- Graphics Type: spice +==> node-admin: -- Graphics Port: -1 +==> node-admin: -- Graphics IP: 127.0.0.1 +==> node-admin: -- Graphics Password: Not defined +==> node-admin: -- Video Type: qxl +==> node-admin: -- Video VRAM: 256 +==> node-admin: -- Sound Type: +==> node-admin: -- Keymap: en-us +==> node-admin: -- TPM Path: +==> node-admin: -- INPUT: type=mouse, bus=ps2 ==> node-admin: Creating shared folders metadata... ==> node-admin: Starting domain. ==> node-admin: Waiting for domain to get an IP address... -==> node-osd2: Creating shared folders metadata... -==> node-osd1: Creating shared folders metadata... -==> node-osd2: Starting domain. -==> node-osd1: Starting domain. -==> node-osd2: Waiting for domain to get an IP address... -==> node-osd1: Waiting for domain to get an IP address... ==> node-admin: Waiting for SSH to become available... node-admin: node-admin: Vagrant insecure key detected. Vagrant will automatically replace node-admin: this with a newly generated keypair for better security. - node-admin: - node-admin: Inserting generated public key within guest... - node-admin: Removing insecure key from the guest if it's present... - node-admin: Key inserted! Disconnecting and reconnecting using new SSH key... -==> node-admin: Setting hostname... -==> node-admin: Configuring and enabling network interfaces... -==> node-osd2: Waiting for SSH to become available... -==> node-osd1: Waiting for SSH to become available... - node-osd2: - node-osd2: Vagrant insecure key detected. Vagrant will automatically replace - node-osd2: this with a newly generated keypair for better security. -==> node-admin: Installing NFS client... - node-osd1: - node-osd1: Vagrant insecure key detected. Vagrant will automatically replace - node-osd1: this with a newly generated keypair for better security. - node-osd2: - node-osd2: Inserting generated public key within guest... - node-osd1: - node-osd1: Inserting generated public key within guest... - node-osd2: Removing insecure key from the guest if it's present... - node-osd1: Removing insecure key from the guest if it's present... - node-osd2: Key inserted! Disconnecting and reconnecting using new SSH key... - node-osd1: Key inserted! Disconnecting and reconnecting using new SSH key... -==> node-osd2: Setting hostname... -==> node-osd1: Setting hostname... -==> node-osd2: Configuring and enabling network interfaces... -==> node-osd1: Configuring and enabling network interfaces... -==> node-osd1: Installing NFS client... -==> node-osd2: Installing NFS client... -==> node-admin: Exporting NFS shared folders... -==> node-admin: Preparing to edit /etc/exports. Administrator privileges will be required... -==> node-admin: Mounting NFS shared folders... ==> node-osd2: Exporting NFS shared folders... ==> node-osd2: Preparing to edit /etc/exports. Administrator privileges will be required... -==> node-admin: Running provisioner: ceph-preflight (shell)... + node-admin: + node-admin: Inserting generated public key within guest... ==> node-osd2: Mounting NFS shared folders... - node-admin: Running: /tmp/vagrant-shell20210323-2165480-om10ex.sh - node-admin: Generating locales (this might take a while)... - node-admin: fr_FR.UTF-8 - node-admin: ... + node-admin: Removing insecure key from the guest if it's present... + node-admin: Key inserted! Disconnecting and reconnecting using new SSH key... ==> node-osd2: Running provisioner: ceph-preflight (shell)... - node-osd2: Running: /tmp/vagrant-shell20210323-2165480-10p8ipb.sh + node-osd2: Running: /tmp/vagrant-shell20210323-2179860-1iqw5wc.sh node-osd2: Generating locales (this might take a while)... node-osd2: fr_FR.UTF-8 node-osd2: ... - node-admin: done - node-admin: Generation complete. - node-admin: - node-admin: {cephtest} {vagrant-shell} Install ceph repository (Octopus version) - node-admin: Warning: apt-key output should not be parsed (stdout is not a terminal) +==> node-admin: Setting hostname... node-osd2: done node-osd2: Generation complete. node-osd2: node-osd2: {cephtest} {vagrant-shell} Install ceph repository (Octopus version) node-osd2: Warning: apt-key output should not be parsed (stdout is not a terminal) - node-admin: OK - node-admin: deb https://download.ceph.com/debian-octopus/ bionic main +==> node-admin: Configuring and enabling network interfaces... node-osd2: OK node-osd2: deb https://download.ceph.com/debian-octopus/ bionic main - node-admin: Hit:1 http://us.archive.ubuntu.com/ubuntu bionic InRelease - node-admin: Hit:2 http://security.ubuntu.com/ubuntu bionic-security InRelease - node-admin: Hit:3 http://us.archive.ubuntu.com/ubuntu bionic-updates InRelease node-osd2: Hit:1 http://us.archive.ubuntu.com/ubuntu bionic InRelease - node-admin: Hit:4 http://us.archive.ubuntu.com/ubuntu bionic-backports InRelease node-osd2: Hit:2 http://us.archive.ubuntu.com/ubuntu bionic-updates InRelease node-osd2: Hit:3 http://us.archive.ubuntu.com/ubuntu bionic-backports InRelease +==> node-osd3: Exporting NFS shared folders... +==> node-osd3: Preparing to edit /etc/exports. Administrator privileges will be required... +==> node-admin: Installing NFS client... node-osd2: Hit:4 http://security.ubuntu.com/ubuntu bionic-security InRelease - node-osd2: Get:5 https://download.ceph.com/debian-octopus bionic InRelease [8,557 B] - node-osd2: Get:6 https://download.ceph.com/debian-octopus bionic/main amd64 Packages [16.0 kB] - node-osd2: Get:7 https://download.ceph.com/debian-octopus bionic/main i386 Packages [3,284 B] - node-osd2: Fetched 27.8 kB in 5s (5,859 B/s) - node-osd2: Reading package lists... - node-admin: Get:5 https://download.ceph.com/debian-octopus bionic InRelease [8,557 B] - node-admin: Get:6 https://download.ceph.com/debian-octopus bionic/main i386 Packages [3,284 B] - node-admin: Get:7 https://download.ceph.com/debian-octopus bionic/main amd64 Packages [16.0 kB] - node-admin: Fetched 27.8 kB in 6s (4,290 B/s) - node-admin: Reading package lists... +==> node-osd3: Mounting NFS shared folders... ==> node-osd1: Exporting NFS shared folders... ==> node-osd1: Preparing to edit /etc/exports. Administrator privileges will be required... + node-osd2: Get:5 https://download.ceph.com/debian-octopus bionic InRelease [8,557 B] + node-osd2: Get:6 https://download.ceph.com/debian-octopus bionic/main amd64 Packages [16.0 kB] +==> node-osd1: Mounting NFS shared folders... + node-osd2: Get:7 https://download.ceph.com/debian-octopus bionic/main i386 Packages [3,284 B] +==> node-osd3: Running provisioner: ceph-preflight (shell)... + node-osd2: Fetched 27.8 kB in 3s (8,008 B/s) + node-osd2: Reading package lists... + node-osd3: Running: /tmp/vagrant-shell20210323-2179860-5opb34.sh + node-osd3: Generating locales (this might take a while)... + node-osd3: fr_FR.UTF-8 + node-osd3: ... +==> node-osd1: Running provisioner: ceph-preflight (shell)... + node-osd1: Running: /tmp/vagrant-shell20210323-2179860-s9k7c3.sh + node-osd1: Generating locales (this might take a while)... + node-osd1: fr_FR + node-osd1: .UTF-8 + node-osd1: ... + node-osd3: done + node-osd3: Generation complete. + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Install ceph repository (Octopus version) + node-osd3: Warning: apt-key output should not be parsed (stdout is not a terminal) + node-osd1: done + node-osd1: Generation complete. + node-osd1: + node-osd1: {cephtest} {vagrant-shell} Install ceph repository (Octopus version) + node-osd1: Warning: apt-key output should not be parsed (stdout is not a terminal) + node-osd3: OK + node-osd3: deb https://download.ceph.com/debian-octopus/ bionic main node-osd2: node-osd2: {cephtest} {vagrant-shell} Install chrony, gdisk, vnstat, htop, ceph-deploy node-osd2: Reading package lists... node-osd2: Building dependency tree... -==> node-osd1: Mounting NFS shared folders... - node-osd2: + node-osd3: Hit:1 http://us.archive.ubuntu.com/ubuntu bionic InRelease node-osd2: Reading state information... -==> node-osd1: Running provisioner: ceph-preflight (shell)... + node-osd3: Hit:2 http://us.archive.ubuntu.com/ubuntu bionic-updates InRelease + node-osd1: OK + node-osd1: deb https://download.ceph.com/debian-octopus/ bionic main + node-osd3: Hit:3 http://us.archive.ubuntu.com/ubuntu bionic-backports InRelease node-osd2: htop is already the newest version (2.1.0-3). node-osd2: The following additional packages will be installed: node-osd2: libnspr4 libnss3 libpython-stdlib libpython2.7-minimal libpython2.7-stdlib @@ -232,48 +287,23 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-osd2: ceph-deploy chrony gdisk libnspr4 libnss3 libpython-stdlib node-osd2: libpython2.7-minimal libpython2.7-stdlib python python-minimal node-osd2: python-pkg-resources python-setuptools python2.7 python2.7-minimal vnstat - node-admin: - node-admin: {cephtest} {vagrant-shell} Install chrony, gdisk, vnstat, htop, ceph-deploy - node-osd1: Running: /tmp/vagrant-shell20210323-2165480-4is46l.sh - node-admin: Reading package lists... - node-admin: Building dependency tree... - node-osd1: Generating locales (this might take a while)... - node-osd1: fr_FR.UTF-8 - node-osd1: ... - node-admin: - node-admin: Reading state information... - node-admin: htop is already the newest version (2.1.0-3). - node-admin: The following additional packages will be installed: - node-admin: libnspr4 libnss3 libpython-stdlib libpython2.7-minimal libpython2.7-stdlib - node-admin: python python-minimal python-pkg-resources python-setuptools python2.7 - node-admin: python2.7-minimal - node-admin: Suggested packages: - node-admin: python-doc python-tk python-setuptools-doc python2.7-doc binutils - node-admin: binfmt-support vnstati - node-osd2: 0 upgraded, 15 newly installed, 0 to remove and 172 not upgraded. + node-osd2: 0 upgraded, 15 newly installed, 0 to remove and 21 not upgraded. node-osd2: Need to get 6,345 kB of archives. node-osd2: After this operation, 25.2 MB of additional disk space will be used. node-osd2: Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libpython2.7-minimal amd64 2.7.17-1~18.04ubuntu1.6 [335 kB] - node-admin: The following NEW packages will be installed: - node-admin: ceph-deploy chrony gdisk libnspr4 libnss3 libpython-stdlib - node-admin: libpython2.7-minimal libpython2.7-stdlib python python-minimal - node-admin: python-pkg-resources python-setuptools python2.7 python2.7-minimal vnstat - node-osd2: Get:2 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-deploy all 2.0.1 [97.2 kB] - node-osd1: done - node-osd1: Generation complete. - node-osd1: - node-osd1: {cephtest} {vagrant-shell} Install ceph repository (Octopus version) - node-osd1: Warning: apt-key output should not be parsed (stdout is not a terminal) - node-admin: 0 upgraded, 15 newly installed, 0 to remove and 172 not upgraded. - node-admin: Need to get 6,345 kB of archives. - node-admin: After this operation, 25.2 MB of additional disk space will be used. - node-admin: Get:1 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-deploy all 2.0.1 [97.2 kB] - node-osd2: Get:3 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 python2.7-minimal amd64 2.7.17-1~18.04ubuntu1.6 [1,291 kB] - node-admin: Get:2 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libpython2.7-minimal amd64 2.7.17-1~18.04ubuntu1.6 [335 kB] + node-osd1: Hit:1 http://us.archive.ubuntu.com/ubuntu bionic InRelease + node-osd1: Hit:2 http://us.archive.ubuntu.com/ubuntu bionic-updates InRelease + node-osd1: Hit:3 http://us.archive.ubuntu.com/ubuntu bionic-backports InRelease + node-osd3: Get:4 https://download.ceph.com/debian-octopus bionic InRelease [8,557 B] + node-osd2: Get:2 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 python2.7-minimal amd64 2.7.17-1~18.04ubuntu1.6 [1,291 kB] + node-osd3: Hit:5 http://security.ubuntu.com/ubuntu bionic-security InRelease + node-osd3: Get:6 https://download.ceph.com/debian-octopus bionic/main i386 Packages [3,284 B] + node-osd2: Get:3 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-deploy all 2.0.1 [97.2 kB] + node-osd3: Get:7 https://download.ceph.com/debian-octopus bionic/main amd64 Packages [16.0 kB] + node-osd3: Fetched 27.8 kB in 4s (6,677 B/s) + node-osd3: Reading package lists... node-osd2: Get:4 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python-minimal amd64 2.7.15~rc1-1 [28.1 kB] node-osd2: Get:5 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libpython2.7-stdlib amd64 2.7.17-1~18.04ubuntu1.6 [1,917 kB] - node-osd1: OK - node-osd1: deb https://download.ceph.com/debian-octopus/ bionic main node-osd2: Get:6 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 python2.7 amd64 2.7.17-1~18.04ubuntu1.6 [248 kB] node-osd2: Get:7 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libpython-stdlib amd64 2.7.15~rc1-1 [7,620 B] node-osd2: Get:8 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python amd64 2.7.15~rc1-1 [140 kB] @@ -281,14 +311,21 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-osd2: Get:10 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python-setuptools all 39.0.1-2 [329 kB] node-osd2: Get:11 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libnspr4 amd64 2:4.18-1ubuntu1 [112 kB] node-osd2: Get:12 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libnss3 amd64 2:3.35-2ubuntu2.12 [1,220 kB] + node-osd1: Get:4 https://download.ceph.com/debian-octopus bionic InRelease [8,557 B] node-osd2: Get:13 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 vnstat amd64 1.18-1 [84.0 kB] node-osd2: Get:14 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 chrony amd64 3.2-4ubuntu4.5 [204 kB] node-osd2: Get:15 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 gdisk amd64 1.0.3-1 [205 kB] - node-admin: Get:3 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 python2.7-minimal amd64 2.7.17-1~18.04ubuntu1.6 [1,291 kB] - node-osd2: Fetched 6,345 kB in 5s (1,313 kB/s) + node-osd1: Get:5 https://download.ceph.com/debian-octopus bionic/main amd64 Packages [16.0 kB] + node-osd1: Hit:6 http://security.ubuntu.com/ubuntu bionic-security InRelease + node-osd1: Get:7 https://download.ceph.com/debian-octopus bionic/main i386 Packages [3,284 B] + node-osd2: Fetched 6,345 kB in 5s (1,223 kB/s) node-osd2: Selecting previously unselected package libpython2.7-minimal:amd64. node-osd2: (Reading database ... - node-osd2: (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% + node-osd2: (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% + node-osd2: (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% + node-osd2: (Reading database ... 55% + node-osd1: Fetched 27.8 kB in 6s (4,681 B/s) + node-osd1: Reading package lists... node-osd2: (Reading database ... 60% node-osd2: (Reading database ... 65% node-osd2: (Reading database ... 70% @@ -297,58 +334,52 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-osd2: (Reading database ... 85% node-osd2: (Reading database ... 90% node-osd2: (Reading database ... 95% - node-osd2: (Reading database ... 100% (Reading database ... 105730 files and directories currently installed.) + node-osd2: (Reading database ... 100% (Reading database ... 105763 files and directories currently installed.) node-osd2: Preparing to unpack .../0-libpython2.7-minimal_2.7.17-1~18.04ubuntu1.6_amd64.deb ... + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Install chrony, gdisk, vnstat, htop, ceph-deploy node-osd2: Unpacking libpython2.7-minimal:amd64 (2.7.17-1~18.04ubuntu1.6) ... - node-admin: Get:4 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python-minimal amd64 2.7.15~rc1-1 [28.1 kB] + node-osd3: Reading package lists... + node-osd3: Building dependency tree... node-osd2: Selecting previously unselected package python2.7-minimal. - node-admin: Get:5 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libpython2.7-stdlib amd64 2.7.17-1~18.04ubuntu1.6 [1,917 kB] node-osd2: Preparing to unpack .../1-python2.7-minimal_2.7.17-1~18.04ubuntu1.6_amd64.deb ... node-osd2: Unpacking python2.7-minimal (2.7.17-1~18.04ubuntu1.6) ... - node-osd1: Hit:1 http://us.archive.ubuntu.com/ubuntu bionic InRelease node-osd2: Selecting previously unselected package python-minimal. node-osd2: Preparing to unpack .../2-python-minimal_2.7.15~rc1-1_amd64.deb ... node-osd2: Unpacking python-minimal (2.7.15~rc1-1) ... - node-osd1: Hit:2 http://us.archive.ubuntu.com/ubuntu bionic-updates InRelease + node-osd3: + node-osd3: Reading state information... node-osd2: Selecting previously unselected package libpython2.7-stdlib:amd64. node-osd2: Preparing to unpack .../3-libpython2.7-stdlib_2.7.17-1~18.04ubuntu1.6_amd64.deb ... node-osd2: Unpacking libpython2.7-stdlib:amd64 (2.7.17-1~18.04ubuntu1.6) ... - node-osd1: Hit:3 http://security.ubuntu.com/ubuntu bionic-security InRelease - node-osd1: Hit:4 http://us.archive.ubuntu.com/ubuntu bionic-backports InRelease - node-admin: Get:6 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 python2.7 amd64 2.7.17-1~18.04ubuntu1.6 [248 kB] - node-admin: Get:7 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libpython-stdlib amd64 2.7.15~rc1-1 [7,620 B] - node-admin: Get:8 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python amd64 2.7.15~rc1-1 [140 kB] - node-admin: Get:9 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python-pkg-resources all 39.0.1-2 [128 kB] - node-admin: Get:10 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python-setuptools all 39.0.1-2 [329 kB] - node-admin: Get:11 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libnspr4 amd64 2:4.18-1ubuntu1 [112 kB] - node-admin: Get:12 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libnss3 amd64 2:3.35-2ubuntu2.12 [1,220 kB] + node-osd3: htop is already the newest version (2.1.0-3). + node-osd3: The following additional packages will be installed: + node-osd3: libnspr4 libnss3 libpython-stdlib libpython2.7-minimal libpython2.7-stdlib + node-osd3: python python-minimal python-pkg-resources python-setuptools python2.7 + node-osd3: python2.7-minimal + node-osd3: Suggested packages: + node-osd3: python-doc python-tk python-setuptools-doc python2.7-doc binutils + node-osd3: binfmt-support vnstati node-osd2: Selecting previously unselected package python2.7. + node-osd3: The following NEW packages will be installed: node-osd2: Preparing to unpack .../4-python2.7_2.7.17-1~18.04ubuntu1.6_amd64.deb ... + node-osd3: ceph-deploy chrony gdisk libnspr4 libnss3 libpython-stdlib + node-osd3: libpython2.7-minimal libpython2.7-stdlib python python-minimal + node-osd3: python-pkg-resources python-setuptools python2.7 python2.7-minimal vnstat node-osd2: Unpacking python2.7 (2.7.17-1~18.04ubuntu1.6) ... - node-osd1: Get:5 https://download.ceph.com/debian-octopus bionic InRelease [8,557 B] node-osd2: Selecting previously unselected package libpython-stdlib:amd64. node-osd2: Preparing to unpack .../5-libpython-stdlib_2.7.15~rc1-1_amd64.deb ... node-osd2: Unpacking libpython-stdlib:amd64 (2.7.15~rc1-1) ... node-osd2: Setting up libpython2.7-minimal:amd64 (2.7.17-1~18.04ubuntu1.6) ... node-osd2: Setting up python2.7-minimal (2.7.17-1~18.04ubuntu1.6) ... - node-admin: Get:13 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 vnstat amd64 1.18-1 [84.0 kB] - node-admin: Get:14 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 chrony amd64 3.2-4ubuntu4.5 [204 kB] - node-admin: Get:15 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 gdisk amd64 1.0.3-1 [205 kB] - node-osd1: Get:6 https://download.ceph.com/debian-octopus bionic/main amd64 Packages [16.0 kB] node-osd2: Setting up python-minimal (2.7.15~rc1-1) ... - node-osd2: Selecting previously unselected package python. + node-osd2: Selecting previously unselected package python. node-osd2: (Reading database ... - node-osd2: (Reading database ... 5% - node-osd2: (Reading database ... 10% - node-osd2: (Reading database ... 15% - node-osd2: (Reading database ... 20% - node-osd2: (Reading database ... 25% - node-osd2: (Reading database ... 30% - node-osd2: (Reading database ... 35% - node-osd2: (Reading database ... 40% - node-osd2: (Reading database ... 45% - node-osd2: (Reading database ... 50% - node-osd2: (Reading database ... 55% + node-osd2: (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% + node-osd3: 0 upgraded, 15 newly installed, 0 to remove and 21 not upgraded. + node-osd3: Need to get 6,345 kB of archives. + node-osd3: After this operation, 25.2 MB of additional disk space will be used. + node-osd3: Get:1 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-deploy all 2.0.1 [97.2 kB] node-osd2: (Reading database ... 60% node-osd2: (Reading database ... 65% node-osd2: (Reading database ... 70% @@ -357,139 +388,36 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-osd2: (Reading database ... 85% node-osd2: (Reading database ... 90% node-osd2: (Reading database ... 95% - node-osd2: (Reading database ... 100% - node-osd2: (Reading database ... - node-osd2: 106478 files and directories currently installed.) + node-osd2: (Reading database ... 100% (Reading database ... 106511 files and directories currently installed.) node-osd2: Preparing to unpack .../0-python_2.7.15~rc1-1_amd64.deb ... node-osd2: Unpacking python (2.7.15~rc1-1) ... - node-osd1: Get:7 https://download.ceph.com/debian-octopus bionic/main i386 Packages [3,284 B] + node-osd1: + node-osd1: {cephtest} {vagrant-shell} Install chrony, gdisk, vnstat, htop, ceph-deploy node-osd2: Selecting previously unselected package python-pkg-resources. node-osd2: Preparing to unpack .../1-python-pkg-resources_39.0.1-2_all.deb ... node-osd2: Unpacking python-pkg-resources (39.0.1-2) ... - node-osd1: Fetched 27.8 kB in 4s (6,967 B/s) node-osd1: Reading package lists... + node-osd1: Building dependency tree... node-osd2: Selecting previously unselected package python-setuptools. node-osd2: Preparing to unpack .../2-python-setuptools_39.0.1-2_all.deb ... node-osd2: Unpacking python-setuptools (39.0.1-2) ... - node-admin: Fetched 6,345 kB in 6s (1,070 kB/s) - node-admin: Selecting previously unselected package libpython2.7-minimal:amd64. - node-admin: (Reading database ... - node-admin: (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% node-osd2: Selecting previously unselected package ceph-deploy. - node-admin: (Reading database ... 60% - node-admin: (Reading database ... 65% - node-admin: (Reading database ... 70% - node-admin: (Reading database ... 75% node-osd2: Preparing to unpack .../3-ceph-deploy_2.0.1_all.deb ... - node-admin: (Reading database ... 80% node-osd2: Unpacking ceph-deploy (2.0.1) ... - node-admin: (Reading database ... 85% - node-admin: (Reading database ... 90% - node-admin: (Reading database ... 95% - node-admin: (Reading database ... 100% (Reading database ... 105730 files and directories currently installed.) - node-admin: Preparing to unpack .../0-libpython2.7-minimal_2.7.17-1~18.04ubuntu1.6_amd64.deb ... - node-admin: Unpacking libpython2.7-minimal:amd64 (2.7.17-1~18.04ubuntu1.6) ... node-osd2: Selecting previously unselected package libnspr4:amd64. node-osd2: Preparing to unpack .../4-libnspr4_2%3a4.18-1ubuntu1_amd64.deb ... node-osd2: Unpacking libnspr4:amd64 (2:4.18-1ubuntu1) ... - node-admin: Selecting previously unselected package python2.7-minimal. - node-admin: Preparing to unpack .../1-python2.7-minimal_2.7.17-1~18.04ubuntu1.6_amd64.deb ... + node-osd3: Get:2 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libpython2.7-minimal amd64 2.7.17-1~18.04ubuntu1.6 [335 kB] node-osd2: Selecting previously unselected package libnss3:amd64. - node-admin: Unpacking python2.7-minimal (2.7.17-1~18.04ubuntu1.6) ... node-osd2: Preparing to unpack .../5-libnss3_2%3a3.35-2ubuntu2.12_amd64.deb ... node-osd2: Unpacking libnss3:amd64 (2:3.35-2ubuntu2.12) ... + node-osd1: Reading state information... node-osd2: Selecting previously unselected package vnstat. node-osd2: Preparing to unpack .../6-vnstat_1.18-1_amd64.deb ... node-osd2: Unpacking vnstat (1.18-1) ... - node-admin: Selecting previously unselected package python-minimal. - node-admin: Preparing to unpack .../2-python-minimal_2.7.15~rc1-1_amd64.deb ... - node-admin: Unpacking python-minimal (2.7.15~rc1-1) ... node-osd2: Selecting previously unselected package chrony. node-osd2: Preparing to unpack .../7-chrony_3.2-4ubuntu4.5_amd64.deb ... node-osd2: Unpacking chrony (3.2-4ubuntu4.5) ... - node-admin: Selecting previously unselected package libpython2.7-stdlib:amd64. - node-admin: Preparing to unpack .../3-libpython2.7-stdlib_2.7.17-1~18.04ubuntu1.6_amd64.deb ... - node-admin: Unpacking libpython2.7-stdlib:amd64 (2.7.17-1~18.04ubuntu1.6) ... - node-osd2: Selecting previously unselected package gdisk. - node-osd2: Preparing to unpack .../8-gdisk_1.0.3-1_amd64.deb ... - node-osd2: Unpacking gdisk (1.0.3-1) ... - node-osd2: Setting up vnstat (1.18-1) ... - node-admin: Selecting previously unselected package python2.7. - node-admin: Preparing to unpack .../4-python2.7_2.7.17-1~18.04ubuntu1.6_amd64.deb ... - node-admin: Unpacking python2.7 (2.7.17-1~18.04ubuntu1.6) ... - node-admin: Selecting previously unselected package libpython-stdlib:amd64. - node-admin: Preparing to unpack .../5-libpython-stdlib_2.7.15~rc1-1_amd64.deb ... - node-admin: Unpacking libpython-stdlib:amd64 (2.7.15~rc1-1) ... - node-admin: Setting up libpython2.7-minimal:amd64 (2.7.17-1~18.04ubuntu1.6) ... - node-admin: Setting up python2.7-minimal (2.7.17-1~18.04ubuntu1.6) ... - node-admin: Setting up python-minimal (2.7.15~rc1-1) ... - node-osd2: Created symlink /etc/systemd/system/vnstatd.service → /lib/systemd/system/vnstat.service. - node-osd2: Created symlink /etc/systemd/system/multi-user.target.wants/vnstat.service → /lib/systemd/system/vnstat.service. - node-admin: Selecting previously unselected package python. - node-admin: (Reading database ... - node-admin: (Reading database ... 5% - node-admin: (Reading database ... 10% - node-admin: (Reading database ... 15% - node-admin: (Reading database ... 20% - node-admin: (Reading database ... 25% - node-admin: (Reading database ... 30% - node-admin: (Reading database ... 35% - node-admin: (Reading database ... 40% - node-admin: (Reading database ... 45% - node-admin: (Reading database ... 50% - node-admin: (Reading database ... 55% - node-admin: (Reading database ... 60% - node-admin: (Reading database ... 65% - node-admin: (Reading database ... 70% - node-admin: (Reading database ... 75% - node-admin: (Reading database ... 80% - node-admin: (Reading database ... 85% - node-admin: (Reading database ... 90% - node-admin: (Reading database ... 95% - node-admin: (Reading database ... 100% - node-admin: (Reading database ... - node-admin: 106478 files and directories currently installed.) - node-admin: Preparing to unpack .../0-python_2.7.15~rc1-1_amd64.deb ... - node-admin: Unpacking python (2.7.15~rc1-1) ... - node-osd2: Setting up libnspr4:amd64 (2:4.18-1ubuntu1) ... - node-osd2: Setting up gdisk (1.0.3-1) ... - node-osd2: Setting up libpython2.7-stdlib:amd64 (2.7.17-1~18.04ubuntu1.6) ... - node-osd2: Setting up python2.7 (2.7.17-1~18.04ubuntu1.6) ... - node-admin: Selecting previously unselected package python-pkg-resources. - node-admin: Preparing to unpack .../1-python-pkg-resources_39.0.1-2_all.deb ... - node-admin: Unpacking python-pkg-resources (39.0.1-2) ... - node-admin: Selecting previously unselected package python-setuptools. - node-admin: Preparing to unpack .../2-python-setuptools_39.0.1-2_all.deb ... - node-admin: Unpacking python-setuptools (39.0.1-2) ... - node-admin: Selecting previously unselected package ceph-deploy. - node-admin: Preparing to unpack .../3-ceph-deploy_2.0.1_all.deb ... - node-admin: Unpacking ceph-deploy (2.0.1) ... - node-osd1: - node-osd1: {cephtest} {vagrant-shell} Install chrony, gdisk, vnstat, htop, ceph-deploy - node-osd1: Reading package lists... - node-admin: Selecting previously unselected package libnspr4:amd64. - node-admin: Preparing to unpack .../4-libnspr4_2%3a4.18-1ubuntu1_amd64.deb ... - node-admin: Unpacking libnspr4:amd64 (2:4.18-1ubuntu1) ... - node-osd1: Building dependency tree... - node-admin: Selecting previously unselected package libnss3:amd64. - node-admin: Preparing to unpack .../5-libnss3_2%3a3.35-2ubuntu2.12_amd64.deb ... - node-admin: Unpacking libnss3:amd64 (2:3.35-2ubuntu2.12) ... - node-admin: Selecting previously unselected package vnstat. - node-admin: Preparing to unpack .../6-vnstat_1.18-1_amd64.deb ... - node-admin: Unpacking vnstat (1.18-1) ... - node-admin: Selecting previously unselected package chrony. - node-admin: Preparing to unpack .../7-chrony_3.2-4ubuntu4.5_amd64.deb ... - node-osd1: Reading state information... - node-admin: Unpacking chrony (3.2-4ubuntu4.5) ... - node-admin: Selecting previously unselected package gdisk. - node-admin: Preparing to unpack .../8-gdisk_1.0.3-1_amd64.deb ... - node-admin: Unpacking gdisk (1.0.3-1) ... - node-admin: Setting up vnstat (1.18-1) ... - node-osd2: Setting up libnss3:amd64 (2:3.35-2ubuntu2.12) ... - node-osd2: Setting up libpython-stdlib:amd64 (2.7.15~rc1-1) ... - node-osd2: Setting up python (2.7.15~rc1-1) ... - node-osd2: Setting up chrony (3.2-4ubuntu4.5) ... - node-osd2: Creating '_chrony' system user/group for the chronyd daemon… node-osd1: htop is already the newest version (2.1.0-3). node-osd1: The following additional packages will be installed: node-osd1: libnspr4 libnss3 libpython-stdlib libpython2.7-minimal libpython2.7-stdlib @@ -498,42 +426,84 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-osd1: Suggested packages: node-osd1: python-doc python-tk python-setuptools-doc python2.7-doc binutils node-osd1: binfmt-support vnstati + node-osd2: Selecting previously unselected package gdisk. + node-osd2: Preparing to unpack .../8-gdisk_1.0.3-1_amd64.deb ... + node-osd2: Unpacking gdisk (1.0.3-1) ... node-osd1: The following NEW packages will be installed: node-osd1: ceph-deploy chrony gdisk libnspr4 libnss3 libpython-stdlib node-osd1: libpython2.7-minimal libpython2.7-stdlib python python-minimal node-osd1: python-pkg-resources python-setuptools python2.7 python2.7-minimal vnstat - node-osd2: - node-osd2: Creating config file /etc/chrony/chrony.conf with new version - node-osd1: 0 upgraded, 15 newly installed, 0 to remove and 172 not upgraded. + node-osd2: Setting up vnstat (1.18-1) ... + node-osd3: Get:3 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 python2.7-minimal amd64 2.7.17-1~18.04ubuntu1.6 [1,291 kB] + node-osd2: Created symlink /etc/systemd/system/vnstatd.service → /lib/systemd/system/vnstat.service. + node-osd2: Created symlink /etc/systemd/system/multi-user.target.wants/vnstat.service → /lib/systemd/system/vnstat.service. + node-osd2: Setting up libnspr4:amd64 (2:4.18-1ubuntu1) ... + node-osd2: Setting up gdisk (1.0.3-1) ... + node-osd2: Setting up libpython2.7-stdlib:amd64 (2.7.17-1~18.04ubuntu1.6) ... + node-osd2: Setting up python2.7 (2.7.17-1~18.04ubuntu1.6) ... + node-osd1: 0 upgraded, 15 newly installed, 0 to remove and 21 not upgraded. node-osd1: Need to get 6,345 kB of archives. node-osd1: After this operation, 25.2 MB of additional disk space will be used. node-osd1: Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libpython2.7-minimal amd64 2.7.17-1~18.04ubuntu1.6 [335 kB] - node-admin: Created symlink /etc/systemd/system/vnstatd.service → /lib/systemd/system/vnstat.service. - node-admin: Created symlink /etc/systemd/system/multi-user.target.wants/vnstat.service → /lib/systemd/system/vnstat.service. - node-osd2: Creating config file /etc/chrony/chrony.keys with new version - node-admin: Setting up libnspr4:amd64 (2:4.18-1ubuntu1) ... - node-admin: Setting up gdisk (1.0.3-1) ... - node-admin: Setting up libpython2.7-stdlib:amd64 (2.7.17-1~18.04ubuntu1.6) ... - node-admin: Setting up python2.7 (2.7.17-1~18.04ubuntu1.6) ... + node-osd3: Get:4 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python-minimal amd64 2.7.15~rc1-1 [28.1 kB] + node-osd3: Get:5 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libpython2.7-stdlib amd64 2.7.17-1~18.04ubuntu1.6 [1,917 kB] + node-osd3: Get:6 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 python2.7 amd64 2.7.17-1~18.04ubuntu1.6 [248 kB] + node-osd3: Get:7 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libpython-stdlib amd64 2.7.15~rc1-1 [7,620 B] + node-osd3: Get:8 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python amd64 2.7.15~rc1-1 [140 kB] + node-osd3: Get:9 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python-pkg-resources all 39.0.1-2 [128 kB] + node-osd3: Get:10 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python-setuptools all 39.0.1-2 [329 kB] + node-osd3: Get:11 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libnspr4 amd64 2:4.18-1ubuntu1 [112 kB] + node-osd3: Get:12 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libnss3 amd64 2:3.35-2ubuntu2.12 [1,220 kB] + node-osd2: Setting up libnss3:amd64 (2:3.35-2ubuntu2.12) ... + node-osd2: Setting up libpython-stdlib:amd64 (2.7.15~rc1-1) ... + node-osd2: Setting up python (2.7.15~rc1-1) ... + node-osd2: Setting up chrony (3.2-4ubuntu4.5) ... + node-osd2: Creating '_chrony' system user/group for the chronyd daemon… + node-osd3: Get:13 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 vnstat amd64 1.18-1 [84.0 kB] + node-osd3: Get:14 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 chrony amd64 3.2-4ubuntu4.5 [204 kB] node-osd1: Get:2 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-deploy all 2.0.1 [97.2 kB] - node-osd2: Created symlink /etc/systemd/system/chronyd.service → /lib/systemd/system/chrony.service. - node-osd2: Created symlink /etc/systemd/system/multi-user.target.wants/chrony.service → /lib/systemd/system/chrony.service. - node-osd2: Setting up python-pkg-resources (39.0.1-2) ... - node-admin: Setting up libnss3:amd64 (2:3.35-2ubuntu2.12) ... - node-admin: Setting up libpython-stdlib:amd64 (2.7.15~rc1-1) ... - node-admin: Setting up python (2.7.15~rc1-1) ... + node-osd3: Get:15 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 gdisk amd64 1.0.3-1 [205 kB] + node-osd2: Creating config file /etc/chrony/chrony.conf with new version + node-osd2: Creating config file /etc/chrony/chrony.keys with new version node-osd1: Get:3 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 python2.7-minimal amd64 2.7.17-1~18.04ubuntu1.6 [1,291 kB] - node-osd2: Setting up python-setuptools (39.0.1-2) ... - node-admin: Setting up chrony (3.2-4ubuntu4.5) ... - node-admin: Creating '_chrony' system user/group for the chronyd daemon… - node-osd2: Setting up ceph-deploy (2.0.1) ... - node-osd2: Processing triggers for man-db (2.8.3-2ubuntu0.1) ... - node-admin: - node-admin: Creating config file /etc/chrony/chrony.conf with new version + node-osd3: Fetched 6,345 kB in 6s (1,143 kB/s) + node-osd3: Selecting previously unselected package libpython2.7-minimal:amd64. + node-osd3: (Reading database ... + node-osd3: (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% + node-osd3: (Reading database ... 60% + node-osd3: (Reading database ... 65% + node-osd3: (Reading database ... 70% + node-osd3: (Reading database ... 75% + node-osd3: (Reading database ... 80% + node-osd3: (Reading database ... 85% + node-osd3: (Reading database ... 90% + node-osd3: (Reading database ... 95% + node-osd3: (Reading database ... 100% (Reading database ... 105763 files and directories currently installed.) + node-osd3: Preparing to unpack .../0-libpython2.7-minimal_2.7.17-1~18.04ubuntu1.6_amd64.deb ... + node-osd3: Unpacking libpython2.7-minimal:amd64 (2.7.17-1~18.04ubuntu1.6) ... + node-osd3: Selecting previously unselected package python2.7-minimal. + node-osd3: Preparing to unpack .../1-python2.7-minimal_2.7.17-1~18.04ubuntu1.6_amd64.deb ... + node-osd3: Unpacking python2.7-minimal (2.7.17-1~18.04ubuntu1.6) ... + node-osd3: Selecting previously unselected package python-minimal. + node-osd3: Preparing to unpack .../2-python-minimal_2.7.15~rc1-1_amd64.deb ... + node-osd3: Unpacking python-minimal (2.7.15~rc1-1) ... + node-osd3: Selecting previously unselected package libpython2.7-stdlib:amd64. node-osd1: Get:4 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python-minimal amd64 2.7.15~rc1-1 [28.1 kB] node-osd1: Get:5 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libpython2.7-stdlib amd64 2.7.17-1~18.04ubuntu1.6 [1,917 kB] - node-admin: - node-admin: Creating config file /etc/chrony/chrony.keys with new version + node-osd3: Preparing to unpack .../3-libpython2.7-stdlib_2.7.17-1~18.04ubuntu1.6_amd64.deb ... + node-osd3: Unpacking libpython2.7-stdlib:amd64 (2.7.17-1~18.04ubuntu1.6) ... + node-osd2: Created symlink /etc/systemd/system/chronyd.service → /lib/systemd/system/chrony.service. + node-osd2: Created symlink /etc/systemd/system/multi-user.target.wants/chrony.service → /lib/systemd/system/chrony.service. + node-osd3: Selecting previously unselected package python2.7. + node-osd3: Preparing to unpack .../4-python2.7_2.7.17-1~18.04ubuntu1.6_amd64.deb ... + node-osd3: Unpacking python2.7 (2.7.17-1~18.04ubuntu1.6) ... + node-osd3: Selecting previously unselected package libpython-stdlib:amd64. + node-osd3: Preparing to unpack .../5-libpython-stdlib_2.7.15~rc1-1_amd64.deb ... + node-osd3: Unpacking libpython-stdlib:amd64 (2.7.15~rc1-1) ... + node-osd2: Setting up python-pkg-resources (39.0.1-2) ... + node-osd3: Setting up libpython2.7-minimal:amd64 (2.7.17-1~18.04ubuntu1.6) ... + node-osd3: Setting up python2.7-minimal (2.7.17-1~18.04ubuntu1.6) ... + node-osd2: Setting up python-setuptools (39.0.1-2) ... node-osd1: Get:6 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 python2.7 amd64 2.7.17-1~18.04ubuntu1.6 [248 kB] node-osd1: Get:7 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libpython-stdlib amd64 2.7.15~rc1-1 [7,620 B] node-osd1: Get:8 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python amd64 2.7.15~rc1-1 [140 kB] @@ -541,42 +511,79 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-osd1: Get:10 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python-setuptools all 39.0.1-2 [329 kB] node-osd1: Get:11 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libnspr4 amd64 2:4.18-1ubuntu1 [112 kB] node-osd1: Get:12 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libnss3 amd64 2:3.35-2ubuntu2.12 [1,220 kB] + node-osd2: Setting up ceph-deploy (2.0.1) ... + node-osd3: Setting up python-minimal (2.7.15~rc1-1) ... + node-osd3: Selecting previously unselected package python. + node-osd3: (Reading database ... + node-osd3: (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% + node-osd2: Processing triggers for man-db (2.8.3-2ubuntu0.1) ... + node-osd3: (Reading database ... 60% + node-osd3: (Reading database ... 65% + node-osd3: (Reading database ... 70% + node-osd3: (Reading database ... 75% + node-osd3: (Reading database ... 80% + node-osd3: (Reading database ... 85% + node-osd3: (Reading database ... 90% + node-osd3: (Reading database ... 95% + node-osd3: (Reading database ... 100% (Reading database ... 106511 files and directories currently installed.) + node-osd3: Preparing to unpack .../0-python_2.7.15~rc1-1_amd64.deb ... + node-osd3: Unpacking python (2.7.15~rc1-1) ... + node-osd3: Selecting previously unselected package python-pkg-resources. + node-osd3: Preparing to unpack .../1-python-pkg-resources_39.0.1-2_all.deb ... node-osd1: Get:13 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 vnstat amd64 1.18-1 [84.0 kB] - node-admin: Created symlink /etc/systemd/system/chronyd.service → /lib/systemd/system/chrony.service. - node-admin: Created symlink /etc/systemd/system/multi-user.target.wants/chrony.service → /lib/systemd/system/chrony.service. + node-osd3: Unpacking python-pkg-resources (39.0.1-2) ... node-osd1: Get:14 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 chrony amd64 3.2-4ubuntu4.5 [204 kB] node-osd1: Get:15 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 gdisk amd64 1.0.3-1 [205 kB] - node-admin: Setting up python-pkg-resources (39.0.1-2) ... - node-admin: Setting up python-setuptools (39.0.1-2) ... - node-admin: Setting up ceph-deploy (2.0.1) ... - node-osd1: Fetched 6,345 kB in 4s (1,516 kB/s) + node-osd3: Selecting previously unselected package python-setuptools. + node-osd3: Preparing to unpack .../2-python-setuptools_39.0.1-2_all.deb ... + node-osd3: Unpacking python-setuptools (39.0.1-2) ... + node-osd3: Selecting previously unselected package ceph-deploy. + node-osd3: Preparing to unpack .../3-ceph-deploy_2.0.1_all.deb ... + node-osd3: Unpacking ceph-deploy (2.0.1) ... + node-osd3: Selecting previously unselected package libnspr4:amd64. + node-osd3: Preparing to unpack .../4-libnspr4_2%3a4.18-1ubuntu1_amd64.deb ... + node-osd3: Unpacking libnspr4:amd64 (2:4.18-1ubuntu1) ... + node-osd3: Selecting previously unselected package libnss3:amd64. + node-osd3: Preparing to unpack .../5-libnss3_2%3a3.35-2ubuntu2.12_amd64.deb ... + node-osd3: Unpacking libnss3:amd64 (2:3.35-2ubuntu2.12) ... + node-osd1: Fetched 6,345 kB in 6s (995 kB/s) node-osd1: Selecting previously unselected package libpython2.7-minimal:amd64. - node-osd1: (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% + node-osd1: (Reading database ... + node-osd1: (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% node-osd1: (Reading database ... 60% node-osd1: (Reading database ... 65% node-osd1: (Reading database ... 70% - node-admin: Processing triggers for man-db (2.8.3-2ubuntu0.1) ... node-osd1: (Reading database ... 75% node-osd1: (Reading database ... 80% node-osd1: (Reading database ... 85% node-osd1: (Reading database ... 90% node-osd1: (Reading database ... 95% - node-osd1: (Reading database ... 100% (Reading database ... 105730 files and directories currently installed.) + node-osd1: (Reading database ... 100% (Reading database ... 105763 files and directories currently installed.) node-osd1: Preparing to unpack .../0-libpython2.7-minimal_2.7.17-1~18.04ubuntu1.6_amd64.deb ... node-osd1: Unpacking libpython2.7-minimal:amd64 (2.7.17-1~18.04ubuntu1.6) ... - node-osd2: Processing triggers for mime-support (3.60ubuntu1) ... - node-osd2: Processing triggers for ureadahead (0.100.0-21) ... + node-osd3: Selecting previously unselected package vnstat. + node-osd3: Preparing to unpack .../6-vnstat_1.18-1_amd64.deb ... + node-osd3: Unpacking vnstat (1.18-1) ... + node-osd3: Selecting previously unselected package chrony. + node-osd3: Preparing to unpack .../7-chrony_3.2-4ubuntu4.5_amd64.deb ... + node-osd3: Unpacking chrony (3.2-4ubuntu4.5) ... node-osd1: Selecting previously unselected package python2.7-minimal. - node-osd2: Processing triggers for libc-bin (2.27-3ubuntu1) ... node-osd1: Preparing to unpack .../1-python2.7-minimal_2.7.17-1~18.04ubuntu1.6_amd64.deb ... node-osd1: Unpacking python2.7-minimal (2.7.17-1~18.04ubuntu1.6) ... - node-osd2: Processing triggers for systemd (237-3ubuntu10.41) ... + node-osd3: Selecting previously unselected package gdisk. + node-osd3: Preparing to unpack .../8-gdisk_1.0.3-1_amd64.deb ... + node-osd3: Unpacking gdisk (1.0.3-1) ... + node-osd3: Setting up vnstat (1.18-1) ... node-osd1: Selecting previously unselected package python-minimal. node-osd1: Preparing to unpack .../2-python-minimal_2.7.15~rc1-1_amd64.deb ... node-osd1: Unpacking python-minimal (2.7.15~rc1-1) ... node-osd1: Selecting previously unselected package libpython2.7-stdlib:amd64. + node-osd2: Processing triggers for mime-support (3.60ubuntu1) ... node-osd1: Preparing to unpack .../3-libpython2.7-stdlib_2.7.17-1~18.04ubuntu1.6_amd64.deb ... node-osd1: Unpacking libpython2.7-stdlib:amd64 (2.7.17-1~18.04ubuntu1.6) ... + node-osd2: Processing triggers for ureadahead (0.100.0-21) ... + node-osd2: Processing triggers for libc-bin (2.27-3ubuntu1.4) ... + node-osd2: Processing triggers for systemd (237-3ubuntu10.44) ... node-osd1: Selecting previously unselected package python2.7. node-osd1: Preparing to unpack .../4-python2.7_2.7.17-1~18.04ubuntu1.6_amd64.deb ... node-osd1: Unpacking python2.7 (2.7.17-1~18.04ubuntu1.6) ... @@ -584,7 +591,13 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-osd1: Preparing to unpack .../5-libpython-stdlib_2.7.15~rc1-1_amd64.deb ... node-osd1: Unpacking libpython-stdlib:amd64 (2.7.15~rc1-1) ... node-osd1: Setting up libpython2.7-minimal:amd64 (2.7.17-1~18.04ubuntu1.6) ... + node-osd3: Created symlink /etc/systemd/system/vnstatd.service → /lib/systemd/system/vnstat.service. + node-osd3: Created symlink /etc/systemd/system/multi-user.target.wants/vnstat.service → /lib/systemd/system/vnstat.service. node-osd1: Setting up python2.7-minimal (2.7.17-1~18.04ubuntu1.6) ... + node-osd3: Setting up libnspr4:amd64 (2:4.18-1ubuntu1) ... + node-osd3: Setting up gdisk (1.0.3-1) ... + node-osd3: Setting up libpython2.7-stdlib:amd64 (2.7.17-1~18.04ubuntu1.6) ... + node-osd3: Setting up python2.7 (2.7.17-1~18.04ubuntu1.6) ... node-osd1: Setting up python-minimal (2.7.15~rc1-1) ... node-osd1: Selecting previously unselected package python. node-osd1: (Reading database ... @@ -609,7 +622,7 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-osd1: (Reading database ... 95% node-osd1: (Reading database ... 100% node-osd1: (Reading database ... - node-osd1: 106478 files and directories currently installed.) + node-osd1: 106511 files and directories currently installed.) node-osd1: Preparing to unpack .../0-python_2.7.15~rc1-1_amd64.deb ... node-osd1: Unpacking python (2.7.15~rc1-1) ... node-osd1: Selecting previously unselected package python-pkg-resources. @@ -621,28 +634,31 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-osd1: Selecting previously unselected package ceph-deploy. node-osd1: Preparing to unpack .../3-ceph-deploy_2.0.1_all.deb ... node-osd1: Unpacking ceph-deploy (2.0.1) ... - node-admin: Processing triggers for mime-support (3.60ubuntu1) ... node-osd1: Selecting previously unselected package libnspr4:amd64. - node-admin: Processing triggers for ureadahead (0.100.0-21) ... node-osd1: Preparing to unpack .../4-libnspr4_2%3a4.18-1ubuntu1_amd64.deb ... node-osd1: Unpacking libnspr4:amd64 (2:4.18-1ubuntu1) ... - node-admin: Processing triggers for libc-bin (2.27-3ubuntu1) ... - node-admin: Processing triggers for systemd (237-3ubuntu10.41) ... + node-osd3: Setting up libnss3:amd64 (2:3.35-2ubuntu2.12) ... + node-osd3: Setting up libpython-stdlib:amd64 (2.7.15~rc1-1) ... node-osd1: Selecting previously unselected package libnss3:amd64. + node-osd3: Setting up python (2.7.15~rc1-1) ... node-osd1: Preparing to unpack .../5-libnss3_2%3a3.35-2ubuntu2.12_amd64.deb ... node-osd1: Unpacking libnss3:amd64 (2:3.35-2ubuntu2.12) ... + node-osd3: Setting up chrony (3.2-4ubuntu4.5) ... + node-osd3: Creating '_chrony' system user/group for the chronyd daemon… node-osd1: Selecting previously unselected package vnstat. node-osd1: Preparing to unpack .../6-vnstat_1.18-1_amd64.deb ... node-osd1: Unpacking vnstat (1.18-1) ... node-osd1: Selecting previously unselected package chrony. node-osd1: Preparing to unpack .../7-chrony_3.2-4ubuntu4.5_amd64.deb ... node-osd1: Unpacking chrony (3.2-4ubuntu4.5) ... + node-osd3: + node-osd3: Creating config file /etc/chrony/chrony.conf with new version node-osd1: Selecting previously unselected package gdisk. node-osd1: Preparing to unpack .../8-gdisk_1.0.3-1_amd64.deb ... node-osd1: Unpacking gdisk (1.0.3-1) ... + node-osd1: Setting up vnstat (1.18-1) ... node-osd2: node-osd2: {cephtest} {vagrant-shell} Configure chrony with admin node as server and osd nodes as clients - node-osd1: Setting up vnstat (1.18-1) ... node-osd2: node-osd2: {cephtest} {vagrant-shell} Create partitions on journal disk for osd nodes node-osd2: @@ -653,61 +669,111 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-osd2: in memory. node-osd2: *************************************************************** node-osd2: - node-osd2: Disk /dev/vda: 67108864 sectors, 32.0 GiB + node-osd2: Disk /dev/vda: 268435456 sectors, 128.0 GiB node-osd2: Sector size (logical/physical): 512/512 bytes - node-osd2: Disk identifier (GUID): 47239205-0B78-4FD6-A8CD-FFA801EFA55F + node-osd2: Disk identifier (GUID): 5B7B2C56-0310-430D-9994-2C04E8FA3B76 node-osd2: Partition table holds up to 128 entries node-osd2: Main partition table begins at sector 2 and ends at sector 33 - node-osd2: First usable sector is 34, last usable sector is 67108830 + node-osd2: First usable sector is 34, last usable sector is 268435422 node-osd2: Partitions will be aligned on 2048-sector boundaries node-osd2: Total free space is 4029 sectors (2.0 MiB) node-osd2: node-osd2: Number Start (sector) End (sector) Size Code Name node-osd2: 1 2048 999423 487.0 MiB 8300 Linux filesystem node-osd2: 2 999424 4999167 1.9 GiB 8200 Linux swap - node-osd2: 3 4999168 67106815 29.6 GiB 8300 Linux filesystem + node-osd2: 3 4999168 268433407 125.6 GiB 8300 Linux filesystem node-osd2: node-osd2: {cephtest} {vagrant-shell} Modify /etc/hosts to allow ceph-deploy to resolve the guest node-osd2: node-osd2: {cephtest} {vagrant-shell} Signal that IP is ready - node-osd2: 192.168.121.204 node-osd2 + node-osd2: 192.168.121.252 node-osd2 node-osd2: node-osd2: {cephtest} {vagrant-shell} Wait for all nodes IP and update /etc/hosts - node-osd1: Created symlink /etc/systemd/system/vnstatd.service → /lib/systemd/system/vnstat.service. + node-osd3: + node-osd3: Creating config file /etc/chrony/chrony.keys with new version + node-osd1: Created symlink /etc/systemd/system/vnstatd.service → /lib/systemd/system/vnstat.service. node-osd1: Created symlink /etc/systemd/system/multi-user.target.wants/vnstat.service → /lib/systemd/system/vnstat.service. + node-osd3: Created symlink /etc/systemd/system/chronyd.service → /lib/systemd/system/chrony.service. + node-osd3: + node-osd3: Created symlink /etc/systemd/system/multi-user.target.wants/chrony.service → /lib/systemd/system/chrony.service. node-osd1: Setting up libnspr4:amd64 (2:4.18-1ubuntu1) ... node-osd1: Setting up gdisk (1.0.3-1) ... node-osd1: Setting up libpython2.7-stdlib:amd64 (2.7.17-1~18.04ubuntu1.6) ... node-osd1: Setting up python2.7 (2.7.17-1~18.04ubuntu1.6) ... + node-osd3: Setting up python-pkg-resources (39.0.1-2) ... + node-osd3: Setting up python-setuptools (39.0.1-2) ... + node-osd3: Setting up ceph-deploy (2.0.1) ... + node-osd3: Processing triggers for man-db (2.8.3-2ubuntu0.1) ... node-osd1: Setting up libnss3:amd64 (2:3.35-2ubuntu2.12) ... node-osd1: Setting up libpython-stdlib:amd64 (2.7.15~rc1-1) ... - node-admin: - node-admin: {cephtest} {vagrant-shell} Configure chrony with admin node as server and osd nodes as clients node-osd1: Setting up python (2.7.15~rc1-1) ... node-osd1: Setting up chrony (3.2-4ubuntu4.5) ... - node-admin: - node-admin: {cephtest} {vagrant-shell} Create partitions on journal disk for osd nodes - node-admin: - node-admin: {cephtest} {vagrant-shell} Modify /etc/hosts to allow ceph-deploy to resolve the guest - node-admin: - node-admin: {cephtest} {vagrant-shell} Signal that IP is ready - node-admin: 192.168.121.137 node-admin - node-admin: - node-admin: {cephtest} {vagrant-shell} Wait for all nodes IP and update /etc/hosts node-osd1: Creating '_chrony' system user/group for the chronyd daemon… + node-osd1: node-osd1: Creating config file /etc/chrony/chrony.conf with new version node-osd1: node-osd1: Creating config file /etc/chrony/chrony.keys with new version + node-osd3: Processing triggers for mime-support (3.60ubuntu1) ... + node-osd3: Processing triggers for ureadahead (0.100.0-21) ... + node-osd3: Processing triggers for libc-bin (2.27-3ubuntu1.4) ... + node-osd3: Processing triggers for systemd (237-3ubuntu10.44) ... node-osd1: Created symlink /etc/systemd/system/chronyd.service → /lib/systemd/system/chrony.service. node-osd1: Created symlink /etc/systemd/system/multi-user.target.wants/chrony.service → /lib/systemd/system/chrony.service. node-osd1: Setting up python-pkg-resources (39.0.1-2) ... node-osd1: Setting up python-setuptools (39.0.1-2) ... node-osd1: Setting up ceph-deploy (2.0.1) ... node-osd1: Processing triggers for man-db (2.8.3-2ubuntu0.1) ... +==> node-admin: Exporting NFS shared folders... +==> node-admin: Preparing to edit /etc/exports. Administrator privileges will be required... +==> node-admin: Mounting NFS shared folders... + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Configure chrony with admin node as server and osd nodes as clients + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Create partitions on journal disk for osd nodes + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Create partitions on node-osd3 + node-osd3: + node-osd3: *************************************************************** + node-osd3: Found invalid GPT and valid MBR; converting MBR to GPT format + node-osd3: in memory. + node-osd3: *************************************************************** + node-osd3: + node-osd3: Disk /dev/vda: 268435456 sectors, 128.0 GiB + node-osd3: Sector size (logical/physical): 512/512 bytes + node-osd3: Disk identifier (GUID): CA862BB3-ADC6-4622-AA82-D4EBE4528008 + node-osd3: Partition table holds up to 128 entries + node-osd3: Main partition table begins at sector 2 and ends at sector 33 + node-osd3: First usable sector is 34, last usable sector is 268435422 + node-osd3: Partitions will be aligned on 2048-sector boundaries + node-osd3: Total free space is 4029 sectors (2.0 MiB) + node-osd3: + node-osd3: Number Start (sector) End (sector) Size Code Name + node-osd3: 1 2048 999423 487.0 MiB 8300 Linux filesystem + node-osd3: 2 999424 4999167 1.9 GiB 8200 Linux swap + node-osd3: 3 4999168 268433407 125.6 GiB 8300 Linux filesystem + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Modify /etc/hosts to allow ceph-deploy to resolve the guest + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Signal that IP is ready + node-osd3: 192.168.121.236 node-osd3 + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Wait for all nodes IP and update /etc/hosts +==> node-admin: Running provisioner: ceph-preflight (shell)... node-osd1: Processing triggers for mime-support (3.60ubuntu1) ... + node-admin: Running: /tmp/vagrant-shell20210323-2179860-nza351.sh node-osd1: Processing triggers for ureadahead (0.100.0-21) ... - node-osd1: Processing triggers for libc-bin (2.27-3ubuntu1) ... - node-osd1: Processing triggers for systemd (237-3ubuntu10.41) ... + node-admin: Generating locales (this might take a while)... + node-admin: fr_FR.UTF-8 + node-admin: ... + node-osd1: Processing triggers for libc-bin (2.27-3ubuntu1.4) ... + node-osd1: Processing triggers for systemd (237-3ubuntu10.44) ... + node-admin: done + node-admin: Generation complete. + node-admin: + node-admin: {cephtest} {vagrant-shell} Install ceph repository (Octopus version) + node-admin: Warning: apt-key output should not be parsed (stdout is not a terminal) + node-admin: OK + node-admin: deb https://download.ceph.com/debian-octopus/ bionic main node-osd1: node-osd1: {cephtest} {vagrant-shell} Configure chrony with admin node as server and osd nodes as clients node-osd1: @@ -720,32 +786,232 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-osd1: in memory. node-osd1: *************************************************************** node-osd1: - node-osd1: Disk /dev/vda: 67108864 sectors, 32.0 GiB + node-osd1: Disk /dev/vda: 268435456 sectors, 128.0 GiB node-osd1: Sector size (logical/physical): 512/512 bytes - node-osd1: Disk identifier (GUID): ED138300-1F8F-4702-8405-BC4837FD904A + node-osd1: Disk identifier (GUID): 315FC112-F4B2-4D8F-BD13-D58509E48C2A node-osd1: Partition table holds up to 128 entries node-osd1: Main partition table begins at sector 2 and ends at sector 33 - node-osd1: First usable sector is 34, last usable sector is 67108830 + node-osd1: First usable sector is 34, last usable sector is 268435422 node-osd1: Partitions will be aligned on 2048-sector boundaries node-osd1: Total free space is 4029 sectors (2.0 MiB) node-osd1: node-osd1: Number Start (sector) End (sector) Size Code Name node-osd1: 1 2048 999423 487.0 MiB 8300 Linux filesystem node-osd1: 2 999424 4999167 1.9 GiB 8200 Linux swap - node-osd1: 3 4999168 67106815 29.6 GiB 8300 Linux filesystem + node-osd1: 3 4999168 268433407 125.6 GiB 8300 Linux filesystem node-osd1: node-osd1: {cephtest} {vagrant-shell} Modify /etc/hosts to allow ceph-deploy to resolve the guest node-osd1: node-osd1: {cephtest} {vagrant-shell} Signal that IP is ready - node-osd1: 192.168.121.109 node-osd1 + node-osd1: 192.168.121.124 node-osd1 node-osd1: node-osd1: {cephtest} {vagrant-shell} Wait for all nodes IP and update /etc/hosts - node-osd1: + node-admin: Hit:1 http://us.archive.ubuntu.com/ubuntu bionic InRelease + node-admin: Hit:2 http://us.archive.ubuntu.com/ubuntu bionic-updates InRelease + node-admin: Hit:3 http://us.archive.ubuntu.com/ubuntu bionic-backports InRelease + node-admin: Hit:4 http://security.ubuntu.com/ubuntu bionic-security InRelease + node-admin: Get:5 https://download.ceph.com/debian-octopus bionic InRelease [8,557 B] + node-admin: Get:6 https://download.ceph.com/debian-octopus bionic/main amd64 Packages [16.0 kB] + node-admin: Get:7 https://download.ceph.com/debian-octopus bionic/main i386 Packages [3,284 B] + node-admin: Fetched 27.8 kB in 3s (8,408 B/s) + node-admin: Reading package lists... + node-admin: + node-admin: {cephtest} {vagrant-shell} Install chrony, gdisk, vnstat, htop, ceph-deploy + node-admin: Reading package lists... + node-admin: Building dependency tree... + node-admin: Reading state information... + node-admin: htop is already the newest version (2.1.0-3). + node-admin: The following additional packages will be installed: + node-admin: libnspr4 libnss3 libpython-stdlib libpython2.7-minimal libpython2.7-stdlib + node-admin: python python-minimal python-pkg-resources python-setuptools python2.7 + node-admin: python2.7-minimal + node-admin: Suggested packages: + node-admin: python-doc python-tk python-setuptools-doc python2.7-doc binutils + node-admin: binfmt-support vnstati + node-admin: The following NEW packages will be installed: + node-admin: ceph-deploy chrony gdisk libnspr4 libnss3 libpython-stdlib + node-admin: libpython2.7-minimal libpython2.7-stdlib python python-minimal + node-admin: python-pkg-resources python-setuptools python2.7 python2.7-minimal vnstat + node-admin: 0 upgraded, 15 newly installed, 0 to remove and 21 not upgraded. + node-admin: Need to get 6,345 kB of archives. + node-admin: After this operation, 25.2 MB of additional disk space will be used. + node-admin: Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libpython2.7-minimal amd64 2.7.17-1~18.04ubuntu1.6 [335 kB] + node-admin: Get:2 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-deploy all 2.0.1 [97.2 kB] + node-admin: Get:3 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 python2.7-minimal amd64 2.7.17-1~18.04ubuntu1.6 [1,291 kB] + node-admin: Get:4 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python-minimal amd64 2.7.15~rc1-1 [28.1 kB] + node-admin: Get:5 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libpython2.7-stdlib amd64 2.7.17-1~18.04ubuntu1.6 [1,917 kB] + node-admin: Get:6 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 python2.7 amd64 2.7.17-1~18.04ubuntu1.6 [248 kB] + node-admin: Get:7 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libpython-stdlib amd64 2.7.15~rc1-1 [7,620 B] + node-admin: Get:8 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python amd64 2.7.15~rc1-1 [140 kB] + node-admin: Get:9 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python-pkg-resources all 39.0.1-2 [128 kB] + node-admin: Get:10 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python-setuptools all 39.0.1-2 [329 kB] + node-admin: Get:11 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libnspr4 amd64 2:4.18-1ubuntu1 [112 kB] + node-admin: Get:12 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libnss3 amd64 2:3.35-2ubuntu2.12 [1,220 kB] + node-admin: Get:13 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 vnstat amd64 1.18-1 [84.0 kB] + node-admin: Get:14 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 chrony amd64 3.2-4ubuntu4.5 [204 kB] + node-admin: Get:15 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 gdisk amd64 1.0.3-1 [205 kB] + node-admin: Fetched 6,345 kB in 4s (1,619 kB/s) + node-admin: Selecting previously unselected package libpython2.7-minimal:amd64. + node-admin: (Reading database ... + node-admin: (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% + node-admin: (Reading database ... 60% + node-admin: (Reading database ... 65% + node-admin: (Reading database ... 70% + node-admin: (Reading database ... 75% + node-admin: (Reading database ... 80% + node-admin: (Reading database ... 85% + node-admin: (Reading database ... 90% + node-admin: (Reading database ... 95% + node-admin: (Reading database ... 100% (Reading database ... 105763 files and directories currently installed.) + node-admin: Preparing to unpack .../0-libpython2.7-minimal_2.7.17-1~18.04ubuntu1.6_amd64.deb ... + node-admin: Unpacking libpython2.7-minimal:amd64 (2.7.17-1~18.04ubuntu1.6) ... + node-admin: Selecting previously unselected package python2.7-minimal. + node-admin: Preparing to unpack .../1-python2.7-minimal_2.7.17-1~18.04ubuntu1.6_amd64.deb ... + node-admin: Unpacking python2.7-minimal (2.7.17-1~18.04ubuntu1.6) ... + node-admin: Selecting previously unselected package python-minimal. + node-admin: Preparing to unpack .../2-python-minimal_2.7.15~rc1-1_amd64.deb ... + node-admin: Unpacking python-minimal (2.7.15~rc1-1) ... + node-admin: Selecting previously unselected package libpython2.7-stdlib:amd64. + node-admin: Preparing to unpack .../3-libpython2.7-stdlib_2.7.17-1~18.04ubuntu1.6_amd64.deb ... + node-admin: Unpacking libpython2.7-stdlib:amd64 (2.7.17-1~18.04ubuntu1.6) ... + node-admin: Selecting previously unselected package python2.7. + node-admin: Preparing to unpack .../4-python2.7_2.7.17-1~18.04ubuntu1.6_amd64.deb ... + node-admin: Unpacking python2.7 (2.7.17-1~18.04ubuntu1.6) ... + node-admin: Selecting previously unselected package libpython-stdlib:amd64. + node-admin: Preparing to unpack .../5-libpython-stdlib_2.7.15~rc1-1_amd64.deb ... + node-admin: Unpacking libpython-stdlib:amd64 (2.7.15~rc1-1) ... + node-admin: Setting up libpython2.7-minimal:amd64 (2.7.17-1~18.04ubuntu1.6) ... + node-admin: Setting up python2.7-minimal (2.7.17-1~18.04ubuntu1.6) ... + node-admin: Setting up python-minimal (2.7.15~rc1-1) ... + node-admin: Selecting previously unselected package python. + node-admin: (Reading database ... + node-admin: (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% + node-admin: (Reading database ... 60% + node-admin: (Reading database ... 65% + node-admin: (Reading database ... 70% + node-admin: (Reading database ... 75% + node-admin: (Reading database ... 80% + node-admin: (Reading database ... 85% + node-admin: (Reading database ... 90% + node-admin: (Reading database ... 95% + node-admin: (Reading database ... 100% (Reading database ... 106511 files and directories currently installed.) + node-admin: Preparing to unpack .../0-python_2.7.15~rc1-1_amd64.deb ... + node-admin: Unpacking python (2.7.15~rc1-1) ... + node-admin: Selecting previously unselected package python-pkg-resources. + node-admin: Preparing to unpack .../1-python-pkg-resources_39.0.1-2_all.deb ... + node-admin: Unpacking python-pkg-resources (39.0.1-2) ... + node-admin: Selecting previously unselected package python-setuptools. + node-admin: Preparing to unpack .../2-python-setuptools_39.0.1-2_all.deb ... + node-admin: Unpacking python-setuptools (39.0.1-2) ... + node-admin: Selecting previously unselected package ceph-deploy. + node-admin: Preparing to unpack .../3-ceph-deploy_2.0.1_all.deb ... + node-admin: Unpacking ceph-deploy (2.0.1) ... + node-admin: Selecting previously unselected package libnspr4:amd64. + node-admin: Preparing to unpack .../4-libnspr4_2%3a4.18-1ubuntu1_amd64.deb ... + node-admin: Unpacking libnspr4:amd64 (2:4.18-1ubuntu1) ... + node-admin: Selecting previously unselected package libnss3:amd64. + node-admin: Preparing to unpack .../5-libnss3_2%3a3.35-2ubuntu2.12_amd64.deb ... + node-admin: Unpacking libnss3:amd64 (2:3.35-2ubuntu2.12) ... + node-admin: Selecting previously unselected package vnstat. + node-admin: Preparing to unpack .../6-vnstat_1.18-1_amd64.deb ... + node-admin: Unpacking vnstat (1.18-1) ... + node-admin: Selecting previously unselected package chrony. + node-admin: Preparing to unpack .../7-chrony_3.2-4ubuntu4.5_amd64.deb ... + node-admin: Unpacking chrony (3.2-4ubuntu4.5) ... + node-admin: Selecting previously unselected package gdisk. + node-admin: Preparing to unpack .../8-gdisk_1.0.3-1_amd64.deb ... + node-admin: Unpacking gdisk (1.0.3-1) ... + node-admin: Setting up vnstat (1.18-1) ... + node-admin: Created symlink /etc/systemd/system/vnstatd.service → /lib/systemd/system/vnstat.service. + node-admin: Created symlink /etc/systemd/system/multi-user.target.wants/vnstat.service → /lib/systemd/system/vnstat.service. + node-admin: Setting up libnspr4:amd64 (2:4.18-1ubuntu1) ... + node-admin: Setting up gdisk (1.0.3-1) ... + node-admin: Setting up libpython2.7-stdlib:amd64 (2.7.17-1~18.04ubuntu1.6) ... + node-admin: Setting up python2.7 (2.7.17-1~18.04ubuntu1.6) ... + node-admin: Setting up libnss3:amd64 (2:3.35-2ubuntu2.12) ... + node-admin: Setting up libpython-stdlib:amd64 (2.7.15~rc1-1) ... + node-admin: Setting up python (2.7.15~rc1-1) ... + node-admin: Setting up chrony (3.2-4ubuntu4.5) ... + node-admin: Creating '_chrony' system user/group for the chronyd daemon… + node-admin: + node-admin: Creating config file /etc/chrony/chrony.conf with new version + node-admin: + node-admin: Creating config file /etc/chrony/chrony.keys with new version + node-admin: Created symlink /etc/systemd/system/chronyd.service → /lib/systemd/system/chrony.service. + node-admin: Created symlink /etc/systemd/system/multi-user.target.wants/chrony.service → /lib/systemd/system/chrony.service. + node-admin: Setting up python-pkg-resources (39.0.1-2) ... + node-admin: Setting up python-setuptools (39.0.1-2) ... + node-admin: Setting up ceph-deploy (2.0.1) ... + node-admin: Processing triggers for man-db (2.8.3-2ubuntu0.1) ... + node-admin: Processing triggers for mime-support (3.60ubuntu1) ... + node-admin: Processing triggers for ureadahead (0.100.0-21) ... + node-admin: Processing triggers for libc-bin (2.27-3ubuntu1.4) ... + node-admin: Processing triggers for systemd (237-3ubuntu10.44) ... + node-admin: + node-admin: {cephtest} {vagrant-shell} Configure chrony with admin node as server and osd nodes as clients + node-admin: + node-admin: {cephtest} {vagrant-shell} Create partitions on journal disk for osd nodes + node-admin: + node-admin: {cephtest} {vagrant-shell} Modify /etc/hosts to allow ceph-deploy to resolve the guest + node-admin: + node-admin: {cephtest} {vagrant-shell} Signal that IP is ready + node-admin: 192.168.121.38 node-admin + node-admin: + node-admin: {cephtest} {vagrant-shell} Wait for all nodes IP and update /etc/hosts + node-admin: + node-admin: {cephtest} {vagrant-shell} Add IP for node-osd1 + node-admin: 192.168.121.124 node-osd1 + node-admin: + node-admin: {cephtest} {vagrant-shell} Add IP for node-osd2 + node-admin: 192.168.121.252 node-osd2 + node-admin: + node-admin: {cephtest} {vagrant-shell} Add IP for node-osd3 + node-admin: 192.168.121.236 node-osd3 + node-admin: + node-admin: {cephtest} {vagrant-shell} Create user ceph-admin if not existing and make it paswordless sudoer + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Add IP for node-admin + node-osd3: 192.168.121.38 node-admin + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Add IP for node-osd1 + node-osd3: 192.168.121.124 node-osd1 + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Add IP for node-osd2 + node-osd3: 192.168.121.252 node-osd2 + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Create user ceph-admin if not existing and make it paswordless sudoer + node-osd3: ceph-admin ALL = (root) NOPASSWD:ALL + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Copy ssh keys, config and authorized keys + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Switch to ceph-admin context + node-admin: ceph-admin ALL = (root) NOPASSWD:ALL + node-admin: + node-admin: {cephtest} {vagrant-shell} Copy ssh keys, config and authorized keys + node-admin: + node-admin: {cephtest} {vagrant-shell} Switch to ceph-admin context + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Switch to root context + node-osd3: + node-osd3: {cephtest} {vagrant-shell} Signal that provision is done + node-osd3: 2021-03-23 01:44:24.764970617+00:00 - Done! + node-admin: + node-admin: {cephtest} {vagrant-shell} Switch to root context + node-admin: + node-admin: {cephtest} {vagrant-shell} Signal that provision is done + node-admin: 2021-03-23 01:44:24.758588255+00:00 - Done! +==> node-admin: Running provisioner: ceph-install (shell)... + node-admin: Running: /tmp/vagrant-shell20210323-2179860-1qpr12c.sh + node-admin: + node-admin: {cephtest} {vagrant-shell} Wait of all nodes ready, with provision done node-osd1: {cephtest} {vagrant-shell} Add IP for node-admin - node-osd1: 192.168.121.137 node-admin + node-osd1: 192.168.121.38 node-admin node-osd1: node-osd1: {cephtest} {vagrant-shell} Add IP for node-osd2 - node-osd1: 192.168.121.204 node-osd2 + node-osd1: 192.168.121.252 node-osd2 + node-osd1: + node-osd1: {cephtest} {vagrant-shell} Add IP for node-osd3 + node-osd1: 192.168.121.236 node-osd3 node-osd1: node-osd1: {cephtest} {vagrant-shell} Create user ceph-admin if not existing and make it paswordless sudoer node-osd1: ceph-admin ALL = (root) NOPASSWD:ALL @@ -757,13 +1023,16 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-osd1: {cephtest} {vagrant-shell} Switch to root context node-osd1: node-osd1: {cephtest} {vagrant-shell} Signal that provision is done - node-osd1: 2021-03-22 23:44:42.003144549+00:00 - Done! + node-osd1: 2021-03-23 01:44:27.853503159+00:00 - Done! node-osd2: node-osd2: {cephtest} {vagrant-shell} Add IP for node-admin - node-osd2: 192.168.121.137 node-admin + node-osd2: 192.168.121.38 node-admin node-osd2: node-osd2: {cephtest} {vagrant-shell} Add IP for node-osd1 - node-osd2: 192.168.121.109 node-osd1 + node-osd2: 192.168.121.124 node-osd1 + node-osd2: + node-osd2: {cephtest} {vagrant-shell} Add IP for node-osd3 + node-osd2: 192.168.121.236 node-osd3 node-osd2: node-osd2: {cephtest} {vagrant-shell} Create user ceph-admin if not existing and make it paswordless sudoer node-osd2: ceph-admin ALL = (root) NOPASSWD:ALL @@ -775,29 +1044,7 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-osd2: {cephtest} {vagrant-shell} Switch to root context node-osd2: node-osd2: {cephtest} {vagrant-shell} Signal that provision is done - node-osd2: 2021-03-22 23:44:59.827617584+00:00 - Done! - node-admin: - node-admin: {cephtest} {vagrant-shell} Add IP for node-osd1 - node-admin: 192.168.121.109 node-osd1 - node-admin: - node-admin: {cephtest} {vagrant-shell} Add IP for node-osd2 - node-admin: 192.168.121.204 node-osd2 - node-admin: - node-admin: {cephtest} {vagrant-shell} Create user ceph-admin if not existing and make it paswordless sudoer - node-admin: ceph-admin ALL = (root) NOPASSWD:ALL - node-admin: - node-admin: {cephtest} {vagrant-shell} Copy ssh keys, config and authorized keys - node-admin: - node-admin: {cephtest} {vagrant-shell} Switch to ceph-admin context - node-admin: - node-admin: {cephtest} {vagrant-shell} Switch to root context - node-admin: - node-admin: {cephtest} {vagrant-shell} Signal that provision is done - node-admin: 2021-03-22 23:45:02.214724656+00:00 - Done! -==> node-admin: Running provisioner: ceph-install (shell)... - node-admin: Running: /tmp/vagrant-shell20210323-2165480-1qkt69m.sh - node-admin: - node-admin: {cephtest} {vagrant-shell} Wait of all nodes ready, with provision done + node-osd2: 2021-03-23 01:44:46.676051853+00:00 - Done! node-admin: node-admin: {cephtest} {ceph-install} Create new cluster configuration node-admin: [ceph_deploy.conf][DEBUG ] found configuration file at: /home/ceph-admin/.cephdeploy.conf @@ -807,11 +1054,11 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [ceph_deploy.cli][INFO ] verbose : False node-admin: [ceph_deploy.cli][INFO ] overwrite_conf : False node-admin: [ceph_deploy.cli][INFO ] quiet : False - node-admin: [ceph_deploy.cli][INFO ] cd_conf : + node-admin: [ceph_deploy.cli][INFO ] cd_conf : node-admin: [ceph_deploy.cli][INFO ] cluster : ceph node-admin: [ceph_deploy.cli][INFO ] ssh_copykey : True node-admin: [ceph_deploy.cli][INFO ] mon : ['node-admin'] - node-admin: [ceph_deploy.cli][INFO ] func : + node-admin: [ceph_deploy.cli][INFO ] func : node-admin: [ceph_deploy.cli][INFO ] public_network : None node-admin: [ceph_deploy.cli][INFO ] ceph_conf : None node-admin: [ceph_deploy.cli][INFO ] cluster_network : None @@ -826,9 +1073,15 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-admin][DEBUG ] find the location of an executable node-admin: [node-admin][INFO ] Running command: sudo /bin/ip link show node-admin: [node-admin][INFO ] Running command: sudo /bin/ip addr show - node-admin: [node-admin][DEBUG ] IP addresses found: [u'192.168.121.137'] + node-admin: [node-admin][DEBUG ] IP addresses found: [u'192.168.121.38'] node-admin: [ceph_deploy.new][DEBUG ] Resolving host node-admin - node-admin: [ceph_deploy][ERROR ] UnableToResolveError: Unable to resolve host: node-admin + node-admin: [ceph_deploy.new][DEBUG ] Monitor node-admin at 192.168.121.38 + node-admin: [ceph_deploy.new][DEBUG ] Monitor initial members are ['node-admin'] + node-admin: [ceph_deploy.new][DEBUG ] Monitor addrs are ['192.168.121.38'] + node-admin: [ceph_deploy.new][DEBUG ] Creating a random mon key... + node-admin: [ceph_deploy.new][DEBUG ] Writing monitor keyring to ceph.mon.keyring... + node-admin: [ceph_deploy.new][DEBUG ] Writing initial config to ceph.conf... + node-admin: Unhandled exception in thread started by node-admin: node-admin: {cephtest} {ceph-install} Initialize cluster configuration node-admin: [global] @@ -849,13 +1102,17 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: mds standby replay = true node-admin: mds standby for rank = 0 node-admin: + node-admin: [mds.node-osd3] + node-admin: mds standby replay = true + node-admin: mds standby for rank = 0 + node-admin: node-admin: {cephtest} {ceph-install} Deploy ceph on all nodes node-admin: [ceph_deploy.conf][DEBUG ] found configuration file at: /home/ceph-admin/.cephdeploy.conf - node-admin: [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy install --release octopus node-admin node-osd1 node-osd2 + node-admin: [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy install --release octopus node-admin node-osd1 node-osd2 node-osd3 node-admin: [ceph_deploy.cli][INFO ] ceph-deploy options: node-admin: [ceph_deploy.cli][INFO ] verbose : False node-admin: [ceph_deploy.cli][INFO ] testing : None - node-admin: [ceph_deploy.cli][INFO ] cd_conf : + node-admin: [ceph_deploy.cli][INFO ] cd_conf : node-admin: [ceph_deploy.cli][INFO ] cluster : ceph node-admin: [ceph_deploy.cli][INFO ] dev_commit : None node-admin: [ceph_deploy.cli][INFO ] install_mds : False @@ -863,11 +1120,11 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [ceph_deploy.cli][INFO ] default_release : False node-admin: [ceph_deploy.cli][INFO ] username : None node-admin: [ceph_deploy.cli][INFO ] adjust_repos : True - node-admin: [ceph_deploy.cli][INFO ] func : + node-admin: [ceph_deploy.cli][INFO ] func : node-admin: [ceph_deploy.cli][INFO ] install_mgr : False node-admin: [ceph_deploy.cli][INFO ] install_all : False node-admin: [ceph_deploy.cli][INFO ] repo : False - node-admin: [ceph_deploy.cli][INFO ] host : ['node-admin', 'node-osd1', 'node-osd2'] + node-admin: [ceph_deploy.cli][INFO ] host : ['node-admin', 'node-osd1', 'node-osd2', 'node-osd3'] node-admin: [ceph_deploy.cli][INFO ] install_rgw : False node-admin: [ceph_deploy.cli][INFO ] install_tests : False node-admin: [ceph_deploy.cli][INFO ] repo_url : None @@ -883,7 +1140,7 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [ceph_deploy.cli][INFO ] release : octopus node-admin: [ceph_deploy.cli][INFO ] install_mon : False node-admin: [ceph_deploy.cli][INFO ] gpg_url : None - node-admin: [ceph_deploy.install][DEBUG ] Installing stable version octopus on cluster ceph hosts node-admin node-osd1 node-osd2 + node-admin: [ceph_deploy.install][DEBUG ] Installing stable version octopus on cluster ceph hosts node-admin node-osd1 node-osd2 node-osd3 node-admin: [ceph_deploy.install][DEBUG ] Detecting platform for host node-admin ... node-admin: [node-admin][DEBUG ] connection detected need for sudo node-admin: [node-admin][DEBUG ] connected to host: node-admin @@ -894,52 +1151,38 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-admin][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q update node-admin: [node-admin][DEBUG ] Hit:1 http://us.archive.ubuntu.com/ubuntu bionic InRelease node-admin: [node-admin][DEBUG ] Hit:2 http://us.archive.ubuntu.com/ubuntu bionic-updates InRelease - node-admin: [node-admin][DEBUG ] Hit:3 http://us.archive.ubuntu.com/ubuntu bionic-backports InRelease + node-admin: [node-admin][DEBUG ] Hit:3 http://security.ubuntu.com/ubuntu bionic-security InRelease node-admin: [node-admin][DEBUG ] Hit:4 https://download.ceph.com/debian-octopus bionic InRelease - node-admin: [node-admin][DEBUG ] Hit:5 http://security.ubuntu.com/ubuntu bionic-security InRelease + node-admin: [node-admin][DEBUG ] Hit:5 http://us.archive.ubuntu.com/ubuntu bionic-backports InRelease node-admin: [node-admin][DEBUG ] Reading package lists... node-admin: [node-admin][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install ca-certificates apt-transport-https node-admin: [node-admin][DEBUG ] Reading package lists... node-admin: [node-admin][DEBUG ] Building dependency tree... node-admin: [node-admin][DEBUG ] Reading state information... + node-admin: [node-admin][DEBUG ] ca-certificates is already the newest version (20210119~18.04.1). node-admin: [node-admin][DEBUG ] The following NEW packages will be installed: node-admin: [node-admin][DEBUG ] apt-transport-https - node-admin: [node-admin][DEBUG ] The following packages will be upgraded: - node-admin: [node-admin][DEBUG ] ca-certificates - node-admin: [node-admin][DEBUG ] 1 upgraded, 1 newly installed, 0 to remove and 171 not upgraded. - node-admin: [node-admin][DEBUG ] Need to get 148 kB of archives. - node-admin: [node-admin][DEBUG ] After this operation, 152 kB of additional disk space will be used. - node-admin: [node-admin][DEBUG ] Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 ca-certificates all 20210119~18.04.1 [147 kB] - node-admin: [node-admin][DEBUG ] Get:2 http://us.archive.ubuntu.com/ubuntu bionic-updates/universe amd64 apt-transport-https all 1.6.12ubuntu0.2 [1,696 B] - node-admin: [node-admin][DEBUG ] Preconfiguring packages ... - node-admin: [node-admin][DEBUG ] Fetched 148 kB in 2s (59.7 kB/s) - node-admin: [node-admin][DEBUG ] (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 106941 files and directories currently installed.) - node-admin: [node-admin][DEBUG ] Preparing to unpack .../ca-certificates_20210119~18.04.1_all.deb ... - node-admin: [node-admin][DEBUG ] Unpacking ca-certificates (20210119~18.04.1) over (20190110~18.04.1) ... + node-admin: [node-admin][DEBUG ] 0 upgraded, 1 newly installed, 0 to remove and 21 not upgraded. + node-admin: [node-admin][DEBUG ] Need to get 1,696 B of archives. + node-admin: [node-admin][DEBUG ] After this operation, 153 kB of additional disk space will be used. + node-admin: [node-admin][DEBUG ] Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/universe amd64 apt-transport-https all 1.6.12ubuntu0.2 [1,696 B] + node-admin: [node-admin][DEBUG ] Fetched 1,696 B in 1s (3,276 B/s) node-admin: [node-admin][DEBUG ] Selecting previously unselected package apt-transport-https. + node-admin: [node-admin][DEBUG ] (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 106974 files and directories currently installed.) node-admin: [node-admin][DEBUG ] Preparing to unpack .../apt-transport-https_1.6.12ubuntu0.2_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking apt-transport-https (1.6.12ubuntu0.2) ... node-admin: [node-admin][DEBUG ] Setting up apt-transport-https (1.6.12ubuntu0.2) ... - node-admin: [node-admin][DEBUG ] Setting up ca-certificates (20210119~18.04.1) ... - node-admin: [node-admin][DEBUG ] Updating certificates in /etc/ssl/certs... - node-admin: [node-admin][DEBUG ] 21 added, 19 removed; done. - node-admin: [node-admin][DEBUG ] Processing triggers for man-db (2.8.3-2ubuntu0.1) ... - node-admin: [node-admin][DEBUG ] Processing triggers for ca-certificates (20210119~18.04.1) ... - node-admin: [node-admin][DEBUG ] Updating certificates in /etc/ssl/certs... - node-admin: [node-admin][DEBUG ] 0 added, 0 removed; done. - node-admin: [node-admin][DEBUG ] Running hooks in /etc/ca-certificates/update.d... - node-admin: [node-admin][DEBUG ] done. node-admin: [node-admin][INFO ] Running command: sudo wget -O release.asc https://download.ceph.com/keys/release.asc - node-admin: [node-admin][WARNING] --2021-03-22 23:45:22-- https://download.ceph.com/keys/release.asc + node-admin: [node-admin][WARNING] --2021-03-23 01:45:05-- https://download.ceph.com/keys/release.asc node-admin: [node-admin][WARNING] Resolving download.ceph.com (download.ceph.com)... 158.69.68.124, 2607:5300:201:2000::3:58a1 node-admin: [node-admin][WARNING] Connecting to download.ceph.com (download.ceph.com)|158.69.68.124|:443... connected. node-admin: [node-admin][WARNING] HTTP request sent, awaiting response... 200 OK node-admin: [node-admin][WARNING] Length: 1645 (1.6K) [application/octet-stream] node-admin: [node-admin][WARNING] Saving to: ‘release.asc’ node-admin: [node-admin][WARNING] - node-admin: [node-admin][WARNING] 0K . 100% 203M=0s + node-admin: [node-admin][WARNING] 0K . 100% 185M=0s node-admin: [node-admin][WARNING] - node-admin: [node-admin][WARNING] 2021-03-22 23:45:23 (203 MB/s) - ‘release.asc’ saved [1645/1645] + node-admin: [node-admin][WARNING] 2021-03-23 01:45:06 (185 MB/s) - ‘release.asc’ saved [1645/1645] node-admin: [node-admin][WARNING] node-admin: [node-admin][INFO ] Running command: sudo apt-key add release.asc node-admin: [node-admin][WARNING] Warning: apt-key output should not be parsed (stdout is not a terminal) @@ -947,10 +1190,10 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-admin][DEBUG ] add deb repo to /etc/apt/sources.list.d/ node-admin: [node-admin][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q update node-admin: [node-admin][DEBUG ] Hit:1 http://us.archive.ubuntu.com/ubuntu bionic InRelease - node-admin: [node-admin][DEBUG ] Hit:2 http://us.archive.ubuntu.com/ubuntu bionic-updates InRelease - node-admin: [node-admin][DEBUG ] Hit:3 http://us.archive.ubuntu.com/ubuntu bionic-backports InRelease + node-admin: [node-admin][DEBUG ] Hit:2 http://security.ubuntu.com/ubuntu bionic-security InRelease + node-admin: [node-admin][DEBUG ] Hit:3 http://us.archive.ubuntu.com/ubuntu bionic-updates InRelease node-admin: [node-admin][DEBUG ] Hit:4 https://download.ceph.com/debian-octopus bionic InRelease - node-admin: [node-admin][DEBUG ] Hit:5 http://security.ubuntu.com/ubuntu bionic-security InRelease + node-admin: [node-admin][DEBUG ] Hit:5 http://us.archive.ubuntu.com/ubuntu bionic-backports InRelease node-admin: [node-admin][DEBUG ] Reading package lists... node-admin: [node-admin][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install -o Dpkg::Options::=--force-confnew ceph ceph-osd ceph-mds ceph-mon radosgw node-admin: [node-admin][DEBUG ] Reading package lists... @@ -981,28 +1224,26 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-admin][DEBUG ] ceph-mgr-rook nvme-cli smartmontools ibverbs-providers python3-lxml node-admin: [node-admin][DEBUG ] python3-routes python3-simplejson python3-pastescript python3-pyinotify node-admin: [node-admin][DEBUG ] The following NEW packages will be installed: - node-admin: [node-admin][DEBUG ] binutils binutils-x86-64-linux-gnu ceph ceph-base ceph-common ceph-mds - node-admin: [node-admin][DEBUG ] ceph-mgr ceph-mgr-modules-core ceph-mon ceph-osd libaio1 libbabeltrace1 - node-admin: [node-admin][DEBUG ] libcephfs2 libgoogle-perftools4 libibverbs1 libleveldb1v5 liblttng-ust-ctl4 - node-admin: [node-admin][DEBUG ] liblttng-ust0 libnl-route-3-200 liboath0 librabbitmq4 librados2 - node-admin: [node-admin][DEBUG ] libradosstriper1 librbd1 librdkafka1 librdmacm1 librgw2 libsnappy1v5 - node-admin: [node-admin][DEBUG ] libtcmalloc-minimal4 liburcu6 python-pastedeploy-tpl python3-bcrypt - node-admin: [node-admin][DEBUG ] python3-bs4 python3-ceph-argparse python3-ceph-common python3-cephfs - node-admin: [node-admin][DEBUG ] python3-cherrypy3 python3-dateutil python3-distutils python3-jwt - node-admin: [node-admin][DEBUG ] python3-lib2to3 python3-logutils python3-mako python3-markupsafe - node-admin: [node-admin][DEBUG ] python3-paste python3-pastedeploy python3-pecan python3-prettytable - node-admin: [node-admin][DEBUG ] python3-rados python3-rbd python3-rgw python3-simplegeneric - node-admin: [node-admin][DEBUG ] python3-singledispatch python3-tempita python3-waitress python3-webob - node-admin: [node-admin][DEBUG ] python3-webtest python3-werkzeug radosgw - node-admin: [node-admin][DEBUG ] The following packages will be upgraded: - node-admin: [node-admin][DEBUG ] binutils-common libbinutils - node-admin: [node-admin][DEBUG ] 2 upgraded, 59 newly installed, 0 to remove and 169 not upgraded. + node-admin: [node-admin][DEBUG ] binutils binutils-common binutils-x86-64-linux-gnu ceph ceph-base + node-admin: [node-admin][DEBUG ] ceph-common ceph-mds ceph-mgr ceph-mgr-modules-core ceph-mon ceph-osd + node-admin: [node-admin][DEBUG ] libaio1 libbabeltrace1 libbinutils libcephfs2 libgoogle-perftools4 + node-admin: [node-admin][DEBUG ] libibverbs1 libleveldb1v5 liblttng-ust-ctl4 liblttng-ust0 libnl-route-3-200 + node-admin: [node-admin][DEBUG ] liboath0 librabbitmq4 librados2 libradosstriper1 librbd1 librdkafka1 + node-admin: [node-admin][DEBUG ] librdmacm1 librgw2 libsnappy1v5 libtcmalloc-minimal4 liburcu6 + node-admin: [node-admin][DEBUG ] python-pastedeploy-tpl python3-bcrypt python3-bs4 python3-ceph-argparse + node-admin: [node-admin][DEBUG ] python3-ceph-common python3-cephfs python3-cherrypy3 python3-dateutil + node-admin: [node-admin][DEBUG ] python3-distutils python3-jwt python3-lib2to3 python3-logutils python3-mako + node-admin: [node-admin][DEBUG ] python3-markupsafe python3-paste python3-pastedeploy python3-pecan + node-admin: [node-admin][DEBUG ] python3-prettytable python3-rados python3-rbd python3-rgw + node-admin: [node-admin][DEBUG ] python3-simplegeneric python3-singledispatch python3-tempita + node-admin: [node-admin][DEBUG ] python3-waitress python3-webob python3-webtest python3-werkzeug radosgw + node-admin: [node-admin][DEBUG ] 0 upgraded, 61 newly installed, 0 to remove and 21 not upgraded. node-admin: [node-admin][DEBUG ] Need to get 79.8 MB of archives. - node-admin: [node-admin][DEBUG ] After this operation, 350 MB of additional disk space will be used. - node-admin: [node-admin][DEBUG ] Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libbinutils amd64 2.30-21ubuntu1~18.04.5 [489 kB] - node-admin: [node-admin][DEBUG ] Get:2 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils-common amd64 2.30-21ubuntu1~18.04.5 [197 kB] - node-admin: [node-admin][DEBUG ] Get:3 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils-x86-64-linux-gnu amd64 2.30-21ubuntu1~18.04.5 [1,839 kB] - node-admin: [node-admin][DEBUG ] Get:4 https://download.ceph.com/debian-octopus bionic/main amd64 librados2 amd64 15.2.10-1bionic [3,059 kB] + node-admin: [node-admin][DEBUG ] After this operation, 353 MB of additional disk space will be used. + node-admin: [node-admin][DEBUG ] Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils-common amd64 2.30-21ubuntu1~18.04.5 [197 kB] + node-admin: [node-admin][DEBUG ] Get:2 https://download.ceph.com/debian-octopus bionic/main amd64 librados2 amd64 15.2.10-1bionic [3,059 kB] + node-admin: [node-admin][DEBUG ] Get:3 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libbinutils amd64 2.30-21ubuntu1~18.04.5 [489 kB] + node-admin: [node-admin][DEBUG ] Get:4 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils-x86-64-linux-gnu amd64 2.30-21ubuntu1~18.04.5 [1,839 kB] node-admin: [node-admin][DEBUG ] Get:5 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils amd64 2.30-21ubuntu1~18.04.5 [3,388 B] node-admin: [node-admin][DEBUG ] Get:6 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libnl-route-3-200 amd64 3.2.29-0ubuntu3 [146 kB] node-admin: [node-admin][DEBUG ] Get:7 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libibverbs1 amd64 17.1-1ubuntu0.2 [44.4 kB] @@ -1060,12 +1301,14 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-admin][DEBUG ] Get:59 https://download.ceph.com/debian-octopus bionic/main amd64 ceph amd64 15.2.10-1bionic [3,912 B] node-admin: [node-admin][DEBUG ] Get:60 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-mds amd64 15.2.10-1bionic [1,854 kB] node-admin: [node-admin][DEBUG ] Get:61 https://download.ceph.com/debian-octopus bionic/main amd64 radosgw amd64 15.2.10-1bionic [8,879 kB] - node-admin: [node-admin][DEBUG ] Fetched 79.8 MB in 1min 1s (1,314 kB/s) - node-admin: [node-admin][DEBUG ] (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 106947 files and directories currently installed.) - node-admin: [node-admin][DEBUG ] Preparing to unpack .../00-libbinutils_2.30-21ubuntu1~18.04.5_amd64.deb ... - node-admin: [node-admin][DEBUG ] Unpacking libbinutils:amd64 (2.30-21ubuntu1~18.04.5) over (2.30-21ubuntu1~18.04.3) ... - node-admin: [node-admin][DEBUG ] Preparing to unpack .../01-binutils-common_2.30-21ubuntu1~18.04.5_amd64.deb ... - node-admin: [node-admin][DEBUG ] Unpacking binutils-common:amd64 (2.30-21ubuntu1~18.04.5) over (2.30-21ubuntu1~18.04.3) ... + node-admin: [node-admin][DEBUG ] Fetched 79.8 MB in 1min 39s (808 kB/s) + node-admin: [node-admin][DEBUG ] Selecting previously unselected package binutils-common:amd64. + node-admin: [node-admin][DEBUG ] (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 106978 files and directories currently installed.) + node-admin: [node-admin][DEBUG ] Preparing to unpack .../00-binutils-common_2.30-21ubuntu1~18.04.5_amd64.deb ... + node-admin: [node-admin][DEBUG ] Unpacking binutils-common:amd64 (2.30-21ubuntu1~18.04.5) ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package libbinutils:amd64. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../01-libbinutils_2.30-21ubuntu1~18.04.5_amd64.deb ... + node-admin: [node-admin][DEBUG ] Unpacking libbinutils:amd64 (2.30-21ubuntu1~18.04.5) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package binutils-x86-64-linux-gnu. node-admin: [node-admin][DEBUG ] Preparing to unpack .../02-binutils-x86-64-linux-gnu_2.30-21ubuntu1~18.04.5_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking binutils-x86-64-linux-gnu (2.30-21ubuntu1~18.04.5) ... @@ -1102,52 +1345,52 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-rados. node-admin: [node-admin][DEBUG ] Preparing to unpack .../13-python3-rados_15.2.10-1bionic_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-rados (15.2.10-1bionic) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-ceph-argparse. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../14-python3-ceph-argparse_15.2.10-1bionic_all.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-ceph-argparse. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../14-python3-ceph-argparse_15.2.10-1bionic_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-ceph-argparse (15.2.10-1bionic) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-cephfs. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../15-python3-cephfs_15.2.10-1bionic_amd64.deb ... + node-admin: [node-admin][DEBUG ] Preparing to unpack .../15-python3-cephfs_15.2.10-1bionic_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-cephfs (15.2.10-1bionic) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-ceph-common. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../16-python3-ceph-common_15.2.10-1bionic_all.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-ceph-common. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../16-python3-ceph-common_15.2.10-1bionic_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-ceph-common (15.2.10-1bionic) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-prettytable. + node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-prettytable. node-admin: [node-admin][DEBUG ] Preparing to unpack .../17-python3-prettytable_0.7.2-3_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-prettytable (0.7.2-3) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-rbd. + node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-rbd. node-admin: [node-admin][DEBUG ] Preparing to unpack .../18-python3-rbd_15.2.10-1bionic_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-rbd (15.2.10-1bionic) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package librabbitmq4:amd64. node-admin: [node-admin][DEBUG ] Preparing to unpack .../19-librabbitmq4_0.8.0-1ubuntu0.18.04.2_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking librabbitmq4:amd64 (0.8.0-1ubuntu0.18.04.2) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package librdkafka1:amd64. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../20-librdkafka1_0.11.3-1build1_amd64.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package librdkafka1:amd64. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../20-librdkafka1_0.11.3-1build1_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking librdkafka1:amd64 (0.11.3-1build1) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package librgw2. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../21-librgw2_15.2.10-1bionic_amd64.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package librgw2. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../21-librgw2_15.2.10-1bionic_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking librgw2 (15.2.10-1bionic) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-rgw. node-admin: [node-admin][DEBUG ] Preparing to unpack .../22-python3-rgw_15.2.10-1bionic_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-rgw (15.2.10-1bionic) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package libaio1:amd64. + node-admin: [node-admin][DEBUG ] Selecting previously unselected package libaio1:amd64. node-admin: [node-admin][DEBUG ] Preparing to unpack .../23-libaio1_0.3.110-5ubuntu0.1_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking libaio1:amd64 (0.3.110-5ubuntu0.1) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package libbabeltrace1:amd64. node-admin: [node-admin][DEBUG ] Preparing to unpack .../24-libbabeltrace1_1.5.5-1_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking libbabeltrace1:amd64 (1.5.5-1) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package libtcmalloc-minimal4. + node-admin: [node-admin][DEBUG ] Selecting previously unselected package libtcmalloc-minimal4. node-admin: [node-admin][DEBUG ] Preparing to unpack .../25-libtcmalloc-minimal4_2.5-2.2ubuntu3_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking libtcmalloc-minimal4 (2.5-2.2ubuntu3) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package libgoogle-perftools4. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../26-libgoogle-perftools4_2.5-2.2ubuntu3_amd64.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package libgoogle-perftools4. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../26-libgoogle-perftools4_2.5-2.2ubuntu3_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking libgoogle-perftools4 (2.5-2.2ubuntu3) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package libsnappy1v5:amd64. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../27-libsnappy1v5_1.1.7-1_amd64.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package libsnappy1v5:amd64. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../27-libsnappy1v5_1.1.7-1_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking libsnappy1v5:amd64 (1.1.7-1) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package libleveldb1v5:amd64. node-admin: [node-admin][DEBUG ] Preparing to unpack .../28-libleveldb1v5_1.20-2_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking libleveldb1v5:amd64 (1.20-2) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package liboath0. + node-admin: [node-admin][DEBUG ] Selecting previously unselected package liboath0. node-admin: [node-admin][DEBUG ] Preparing to unpack .../29-liboath0_2.6.1-1_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking liboath0 (2.6.1-1) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package libradosstriper1. @@ -1159,7 +1402,7 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-admin][DEBUG ] Selecting previously unselected package ceph-base. node-admin: [node-admin][DEBUG ] Preparing to unpack .../32-ceph-base_15.2.10-1bionic_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking ceph-base (15.2.10-1bionic) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-dateutil. + node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-dateutil. node-admin: [node-admin][DEBUG ] Preparing to unpack .../33-python3-dateutil_2.6.1-1_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-dateutil (2.6.1-1) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package ceph-mgr-modules-core. @@ -1174,8 +1417,8 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-lib2to3. node-admin: [node-admin][DEBUG ] Preparing to unpack .../37-python3-lib2to3_3.6.9-1~18.04_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-lib2to3 (3.6.9-1~18.04) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-distutils. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../38-python3-distutils_3.6.9-1~18.04_all.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-distutils. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../38-python3-distutils_3.6.9-1~18.04_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-distutils (3.6.9-1~18.04) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-jwt. node-admin: [node-admin][DEBUG ] Preparing to unpack .../39-python3-jwt_1.5.3+ds1-1_all.deb ... @@ -1183,20 +1426,20 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-logutils. node-admin: [node-admin][DEBUG ] Preparing to unpack .../40-python3-logutils_0.3.3-5_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-logutils (0.3.3-5) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-markupsafe. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../41-python3-markupsafe_1.0-1build1_amd64.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-markupsafe. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../41-python3-markupsafe_1.0-1build1_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-markupsafe (1.0-1build1) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-mako. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../42-python3-mako_1.0.7+ds1-1_all.deb ... + node-admin: [node-admin][DEBUG ] Preparing to unpack .../42-python3-mako_1.0.7+ds1-1_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-mako (1.0.7+ds1-1) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-simplegeneric. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../43-python3-simplegeneric_0.8.1-1_all.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-simplegeneric. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../43-python3-simplegeneric_0.8.1-1_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-simplegeneric (0.8.1-1) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-singledispatch. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../44-python3-singledispatch_3.4.0.3-2_all.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-singledispatch. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../44-python3-singledispatch_3.4.0.3-2_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-singledispatch (3.4.0.3-2) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-webob. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../45-python3-webob_1%3a1.7.3-2fakesync1_all.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-webob. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../45-python3-webob_1%3a1.7.3-2fakesync1_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-webob (1:1.7.3-2fakesync1) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-bs4. node-admin: [node-admin][DEBUG ] Preparing to unpack .../46-python3-bs4_4.6.0-1_all.deb ... @@ -1204,7 +1447,7 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-waitress. node-admin: [node-admin][DEBUG ] Preparing to unpack .../47-python3-waitress_1.0.1-1_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-waitress (1.0.1-1) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-tempita. + node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-tempita. node-admin: [node-admin][DEBUG ] Preparing to unpack .../48-python3-tempita_0.5.2-2_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-tempita (0.5.2-2) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-paste. @@ -1216,26 +1459,26 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-pastedeploy. node-admin: [node-admin][DEBUG ] Preparing to unpack .../51-python3-pastedeploy_1.5.2-4_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-pastedeploy (1.5.2-4) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-webtest. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../52-python3-webtest_2.0.28-1ubuntu1_all.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-webtest. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../52-python3-webtest_2.0.28-1ubuntu1_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-webtest (2.0.28-1ubuntu1) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-pecan. node-admin: [node-admin][DEBUG ] Preparing to unpack .../53-python3-pecan_1.2.1-2_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-pecan (1.2.1-2) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-werkzeug. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../54-python3-werkzeug_0.14.1+dfsg1-1ubuntu0.1_all.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package python3-werkzeug. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../54-python3-werkzeug_0.14.1+dfsg1-1ubuntu0.1_all.deb ... node-admin: [node-admin][DEBUG ] Unpacking python3-werkzeug (0.14.1+dfsg1-1ubuntu0.1) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package ceph-mgr. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../55-ceph-mgr_15.2.10-1bionic_amd64.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package ceph-mgr. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../55-ceph-mgr_15.2.10-1bionic_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking ceph-mgr (15.2.10-1bionic) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package ceph-mon. node-admin: [node-admin][DEBUG ] Preparing to unpack .../56-ceph-mon_15.2.10-1bionic_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking ceph-mon (15.2.10-1bionic) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package ceph-osd. + node-admin: [node-admin][DEBUG ] Selecting previously unselected package ceph-osd. node-admin: [node-admin][DEBUG ] Preparing to unpack .../57-ceph-osd_15.2.10-1bionic_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking ceph-osd (15.2.10-1bionic) ... - node-admin: [node-admin][DEBUG ] Selecting previously unselected package ceph. - node-admin: [node-admin][DEBUG ] Preparing to unpack .../58-ceph_15.2.10-1bionic_amd64.deb ... + node-admin: [node-admin][DEBUG ] Selecting previously unselected package ceph. + node-admin: [node-admin][DEBUG ] Preparing to unpack .../58-ceph_15.2.10-1bionic_amd64.deb ... node-admin: [node-admin][DEBUG ] Unpacking ceph (15.2.10-1bionic) ... node-admin: [node-admin][DEBUG ] Selecting previously unselected package ceph-mds. node-admin: [node-admin][DEBUG ] Preparing to unpack .../59-ceph-mds_15.2.10-1bionic_amd64.deb ... @@ -1314,7 +1557,7 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-admin][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. node-admin: [node-admin][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. node-admin: [node-admin][DEBUG ] Setting up ceph-osd (15.2.10-1bionic) ... - node-admin: [node-admin][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. + node-admin: [node-admin][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. node-admin: [node-admin][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. node-admin: [node-admin][DEBUG ] Setting up ceph-mds (15.2.10-1bionic) ... node-admin: [node-admin][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. @@ -1322,15 +1565,15 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-admin][DEBUG ] Setting up ceph-mon (15.2.10-1bionic) ... node-admin: [node-admin][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. node-admin: [node-admin][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. - node-admin: [node-admin][DEBUG ] Setting up ceph (15.2.10-1bionic) ... - node-admin: [node-admin][DEBUG ] Processing triggers for systemd (237-3ubuntu10.41) ... + node-admin: [node-admin][DEBUG ] Setting up ceph (15.2.10-1bionic) ... + node-admin: [node-admin][DEBUG ] Processing triggers for systemd (237-3ubuntu10.44) ... node-admin: [node-admin][DEBUG ] Processing triggers for man-db (2.8.3-2ubuntu0.1) ... - node-admin: [node-admin][DEBUG ] Processing triggers for ureadahead (0.100.0-21) ... - node-admin: [node-admin][DEBUG ] Processing triggers for libc-bin (2.27-3ubuntu1) ... + node-admin: [node-admin][DEBUG ] Processing triggers for ureadahead (0.100.0-21) ... + node-admin: [node-admin][DEBUG ] Processing triggers for libc-bin (2.27-3ubuntu1.4) ... node-admin: [node-admin][INFO ] Running command: sudo ceph --version node-admin: [node-admin][DEBUG ] ceph version 15.2.10 (27917a557cca91e4da407489bbaa64ad4352cc02) octopus (stable) node-admin: [ceph_deploy.install][DEBUG ] Detecting platform for host node-osd1 ... - node-admin: Warning: Permanently added 'node-osd1,192.168.121.109' (ECDSA) to the list of known hosts. + node-admin: Warning: Permanently added 'node-osd1,192.168.121.124' (ECDSA) to the list of known hosts. node-admin: [node-osd1][DEBUG ] connection detected need for sudo node-admin: [node-osd1][DEBUG ] connected to host: node-osd1 node-admin: [node-osd1][DEBUG ] detect platform information from remote host @@ -1338,54 +1581,40 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [ceph_deploy.install][INFO ] Distro info: Ubuntu 18.04 bionic node-admin: [node-osd1][INFO ] installing Ceph on node-osd1 node-admin: [node-osd1][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q update - node-admin: [node-osd1][DEBUG ] Hit:1 https://download.ceph.com/debian-octopus bionic InRelease + node-admin: [node-osd1][DEBUG ] Hit:1 http://us.archive.ubuntu.com/ubuntu bionic InRelease node-admin: [node-osd1][DEBUG ] Hit:2 http://security.ubuntu.com/ubuntu bionic-security InRelease - node-admin: [node-osd1][DEBUG ] Hit:3 http://us.archive.ubuntu.com/ubuntu bionic InRelease - node-admin: [node-osd1][DEBUG ] Hit:4 http://us.archive.ubuntu.com/ubuntu bionic-updates InRelease + node-admin: [node-osd1][DEBUG ] Hit:3 http://us.archive.ubuntu.com/ubuntu bionic-updates InRelease + node-admin: [node-osd1][DEBUG ] Hit:4 https://download.ceph.com/debian-octopus bionic InRelease node-admin: [node-osd1][DEBUG ] Hit:5 http://us.archive.ubuntu.com/ubuntu bionic-backports InRelease node-admin: [node-osd1][DEBUG ] Reading package lists... node-admin: [node-osd1][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install ca-certificates apt-transport-https node-admin: [node-osd1][DEBUG ] Reading package lists... node-admin: [node-osd1][DEBUG ] Building dependency tree... node-admin: [node-osd1][DEBUG ] Reading state information... + node-admin: [node-osd1][DEBUG ] ca-certificates is already the newest version (20210119~18.04.1). node-admin: [node-osd1][DEBUG ] The following NEW packages will be installed: node-admin: [node-osd1][DEBUG ] apt-transport-https - node-admin: [node-osd1][DEBUG ] The following packages will be upgraded: - node-admin: [node-osd1][DEBUG ] ca-certificates - node-admin: [node-osd1][DEBUG ] 1 upgraded, 1 newly installed, 0 to remove and 171 not upgraded. - node-admin: [node-osd1][DEBUG ] Need to get 148 kB of archives. - node-admin: [node-osd1][DEBUG ] After this operation, 152 kB of additional disk space will be used. - node-admin: [node-osd1][DEBUG ] Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 ca-certificates all 20210119~18.04.1 [147 kB] - node-admin: [node-osd1][DEBUG ] Get:2 http://us.archive.ubuntu.com/ubuntu bionic-updates/universe amd64 apt-transport-https all 1.6.12ubuntu0.2 [1,696 B] - node-admin: [node-osd1][DEBUG ] Preconfiguring packages ... - node-admin: [node-osd1][DEBUG ] Fetched 148 kB in 4s (39.0 kB/s) - node-admin: [node-osd1][DEBUG ] (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 106941 files and directories currently installed.) - node-admin: [node-osd1][DEBUG ] Preparing to unpack .../ca-certificates_20210119~18.04.1_all.deb ... - node-admin: [node-osd1][DEBUG ] Unpacking ca-certificates (20210119~18.04.1) over (20190110~18.04.1) ... + node-admin: [node-osd1][DEBUG ] 0 upgraded, 1 newly installed, 0 to remove and 21 not upgraded. + node-admin: [node-osd1][DEBUG ] Need to get 1,696 B of archives. + node-admin: [node-osd1][DEBUG ] After this operation, 153 kB of additional disk space will be used. + node-admin: [node-osd1][DEBUG ] Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/universe amd64 apt-transport-https all 1.6.12ubuntu0.2 [1,696 B] + node-admin: [node-osd1][DEBUG ] Fetched 1,696 B in 1s (1,535 B/s) node-admin: [node-osd1][DEBUG ] Selecting previously unselected package apt-transport-https. + node-admin: [node-osd1][DEBUG ] (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 106974 files and directories currently installed.) node-admin: [node-osd1][DEBUG ] Preparing to unpack .../apt-transport-https_1.6.12ubuntu0.2_all.deb ... node-admin: [node-osd1][DEBUG ] Unpacking apt-transport-https (1.6.12ubuntu0.2) ... node-admin: [node-osd1][DEBUG ] Setting up apt-transport-https (1.6.12ubuntu0.2) ... - node-admin: [node-osd1][DEBUG ] Setting up ca-certificates (20210119~18.04.1) ... - node-admin: [node-osd1][DEBUG ] Updating certificates in /etc/ssl/certs... - node-admin: [node-osd1][DEBUG ] 21 added, 19 removed; done. - node-admin: [node-osd1][DEBUG ] Processing triggers for man-db (2.8.3-2ubuntu0.1) ... - node-admin: [node-osd1][DEBUG ] Processing triggers for ca-certificates (20210119~18.04.1) ... - node-admin: [node-osd1][DEBUG ] Updating certificates in /etc/ssl/certs... - node-admin: [node-osd1][DEBUG ] 0 added, 0 removed; done. - node-admin: [node-osd1][DEBUG ] Running hooks in /etc/ca-certificates/update.d... - node-admin: [node-osd1][DEBUG ] done. node-admin: [node-osd1][INFO ] Running command: sudo wget -O release.asc https://download.ceph.com/keys/release.asc - node-admin: [node-osd1][WARNING] --2021-03-22 23:47:46-- https://download.ceph.com/keys/release.asc + node-admin: [node-osd1][WARNING] --2021-03-23 01:49:19-- https://download.ceph.com/keys/release.asc node-admin: [node-osd1][WARNING] Resolving download.ceph.com (download.ceph.com)... 158.69.68.124, 2607:5300:201:2000::3:58a1 node-admin: [node-osd1][WARNING] Connecting to download.ceph.com (download.ceph.com)|158.69.68.124|:443... connected. node-admin: [node-osd1][WARNING] HTTP request sent, awaiting response... 200 OK node-admin: [node-osd1][WARNING] Length: 1645 (1.6K) [application/octet-stream] node-admin: [node-osd1][WARNING] Saving to: ‘release.asc’ node-admin: [node-osd1][WARNING] - node-admin: [node-osd1][WARNING] 0K . 100% 200M=0s + node-admin: [node-osd1][WARNING] 0K . 100% 201M=0s node-admin: [node-osd1][WARNING] - node-admin: [node-osd1][WARNING] 2021-03-22 23:47:47 (200 MB/s) - ‘release.asc’ saved [1645/1645] + node-admin: [node-osd1][WARNING] 2021-03-23 01:49:21 (201 MB/s) - ‘release.asc’ saved [1645/1645] node-admin: [node-osd1][WARNING] node-admin: [node-osd1][INFO ] Running command: sudo apt-key add release.asc node-admin: [node-osd1][WARNING] Warning: apt-key output should not be parsed (stdout is not a terminal) @@ -1427,27 +1656,25 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-osd1][DEBUG ] ceph-mgr-rook nvme-cli smartmontools ibverbs-providers python3-lxml node-admin: [node-osd1][DEBUG ] python3-routes python3-simplejson python3-pastescript python3-pyinotify node-admin: [node-osd1][DEBUG ] The following NEW packages will be installed: - node-admin: [node-osd1][DEBUG ] binutils binutils-x86-64-linux-gnu ceph ceph-base ceph-common ceph-mds - node-admin: [node-osd1][DEBUG ] ceph-mgr ceph-mgr-modules-core ceph-mon ceph-osd libaio1 libbabeltrace1 - node-admin: [node-osd1][DEBUG ] libcephfs2 libgoogle-perftools4 libibverbs1 libleveldb1v5 liblttng-ust-ctl4 - node-admin: [node-osd1][DEBUG ] liblttng-ust0 libnl-route-3-200 liboath0 librabbitmq4 librados2 - node-admin: [node-osd1][DEBUG ] libradosstriper1 librbd1 librdkafka1 librdmacm1 librgw2 libsnappy1v5 - node-admin: [node-osd1][DEBUG ] libtcmalloc-minimal4 liburcu6 python-pastedeploy-tpl python3-bcrypt - node-admin: [node-osd1][DEBUG ] python3-bs4 python3-ceph-argparse python3-ceph-common python3-cephfs - node-admin: [node-osd1][DEBUG ] python3-cherrypy3 python3-dateutil python3-distutils python3-jwt - node-admin: [node-osd1][DEBUG ] python3-lib2to3 python3-logutils python3-mako python3-markupsafe - node-admin: [node-osd1][DEBUG ] python3-paste python3-pastedeploy python3-pecan python3-prettytable - node-admin: [node-osd1][DEBUG ] python3-rados python3-rbd python3-rgw python3-simplegeneric - node-admin: [node-osd1][DEBUG ] python3-singledispatch python3-tempita python3-waitress python3-webob - node-admin: [node-osd1][DEBUG ] python3-webtest python3-werkzeug radosgw - node-admin: [node-osd1][DEBUG ] The following packages will be upgraded: - node-admin: [node-osd1][DEBUG ] binutils-common libbinutils - node-admin: [node-osd1][DEBUG ] 2 upgraded, 59 newly installed, 0 to remove and 169 not upgraded. + node-admin: [node-osd1][DEBUG ] binutils binutils-common binutils-x86-64-linux-gnu ceph ceph-base + node-admin: [node-osd1][DEBUG ] ceph-common ceph-mds ceph-mgr ceph-mgr-modules-core ceph-mon ceph-osd + node-admin: [node-osd1][DEBUG ] libaio1 libbabeltrace1 libbinutils libcephfs2 libgoogle-perftools4 + node-admin: [node-osd1][DEBUG ] libibverbs1 libleveldb1v5 liblttng-ust-ctl4 liblttng-ust0 libnl-route-3-200 + node-admin: [node-osd1][DEBUG ] liboath0 librabbitmq4 librados2 libradosstriper1 librbd1 librdkafka1 + node-admin: [node-osd1][DEBUG ] librdmacm1 librgw2 libsnappy1v5 libtcmalloc-minimal4 liburcu6 + node-admin: [node-osd1][DEBUG ] python-pastedeploy-tpl python3-bcrypt python3-bs4 python3-ceph-argparse + node-admin: [node-osd1][DEBUG ] python3-ceph-common python3-cephfs python3-cherrypy3 python3-dateutil + node-admin: [node-osd1][DEBUG ] python3-distutils python3-jwt python3-lib2to3 python3-logutils python3-mako + node-admin: [node-osd1][DEBUG ] python3-markupsafe python3-paste python3-pastedeploy python3-pecan + node-admin: [node-osd1][DEBUG ] python3-prettytable python3-rados python3-rbd python3-rgw + node-admin: [node-osd1][DEBUG ] python3-simplegeneric python3-singledispatch python3-tempita + node-admin: [node-osd1][DEBUG ] python3-waitress python3-webob python3-webtest python3-werkzeug radosgw + node-admin: [node-osd1][DEBUG ] 0 upgraded, 61 newly installed, 0 to remove and 21 not upgraded. node-admin: [node-osd1][DEBUG ] Need to get 79.8 MB of archives. - node-admin: [node-osd1][DEBUG ] After this operation, 350 MB of additional disk space will be used. - node-admin: [node-osd1][DEBUG ] Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libbinutils amd64 2.30-21ubuntu1~18.04.5 [489 kB] + node-admin: [node-osd1][DEBUG ] After this operation, 353 MB of additional disk space will be used. + node-admin: [node-osd1][DEBUG ] Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils-common amd64 2.30-21ubuntu1~18.04.5 [197 kB] node-admin: [node-osd1][DEBUG ] Get:2 https://download.ceph.com/debian-octopus bionic/main amd64 librados2 amd64 15.2.10-1bionic [3,059 kB] - node-admin: [node-osd1][DEBUG ] Get:3 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils-common amd64 2.30-21ubuntu1~18.04.5 [197 kB] + node-admin: [node-osd1][DEBUG ] Get:3 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libbinutils amd64 2.30-21ubuntu1~18.04.5 [489 kB] node-admin: [node-osd1][DEBUG ] Get:4 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils-x86-64-linux-gnu amd64 2.30-21ubuntu1~18.04.5 [1,839 kB] node-admin: [node-osd1][DEBUG ] Get:5 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils amd64 2.30-21ubuntu1~18.04.5 [3,388 B] node-admin: [node-osd1][DEBUG ] Get:6 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libnl-route-3-200 amd64 3.2.29-0ubuntu3 [146 kB] @@ -1506,12 +1733,14 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-osd1][DEBUG ] Get:59 https://download.ceph.com/debian-octopus bionic/main amd64 ceph amd64 15.2.10-1bionic [3,912 B] node-admin: [node-osd1][DEBUG ] Get:60 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-mds amd64 15.2.10-1bionic [1,854 kB] node-admin: [node-osd1][DEBUG ] Get:61 https://download.ceph.com/debian-octopus bionic/main amd64 radosgw amd64 15.2.10-1bionic [8,879 kB] - node-admin: [node-osd1][DEBUG ] Fetched 79.8 MB in 52s (1,522 kB/s) - node-admin: [node-osd1][DEBUG ] (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 106947 files and directories currently installed.) - node-admin: [node-osd1][DEBUG ] Preparing to unpack .../00-libbinutils_2.30-21ubuntu1~18.04.5_amd64.deb ... - node-admin: [node-osd1][DEBUG ] Unpacking libbinutils:amd64 (2.30-21ubuntu1~18.04.5) over (2.30-21ubuntu1~18.04.3) ... - node-admin: [node-osd1][DEBUG ] Preparing to unpack .../01-binutils-common_2.30-21ubuntu1~18.04.5_amd64.deb ... - node-admin: [node-osd1][DEBUG ] Unpacking binutils-common:amd64 (2.30-21ubuntu1~18.04.5) over (2.30-21ubuntu1~18.04.3) ... + node-admin: [node-osd1][DEBUG ] Fetched 79.8 MB in 1min 48s (736 kB/s) + node-admin: [node-osd1][DEBUG ] Selecting previously unselected package binutils-common:amd64. + node-admin: [node-osd1][DEBUG ] (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 106978 files and directories currently installed.) + node-admin: [node-osd1][DEBUG ] Preparing to unpack .../00-binutils-common_2.30-21ubuntu1~18.04.5_amd64.deb ... + node-admin: [node-osd1][DEBUG ] Unpacking binutils-common:amd64 (2.30-21ubuntu1~18.04.5) ... + node-admin: [node-osd1][DEBUG ] Selecting previously unselected package libbinutils:amd64. + node-admin: [node-osd1][DEBUG ] Preparing to unpack .../01-libbinutils_2.30-21ubuntu1~18.04.5_amd64.deb ... + node-admin: [node-osd1][DEBUG ] Unpacking libbinutils:amd64 (2.30-21ubuntu1~18.04.5) ... node-admin: [node-osd1][DEBUG ] Selecting previously unselected package binutils-x86-64-linux-gnu. node-admin: [node-osd1][DEBUG ] Preparing to unpack .../02-binutils-x86-64-linux-gnu_2.30-21ubuntu1~18.04.5_amd64.deb ... node-admin: [node-osd1][DEBUG ] Unpacking binutils-x86-64-linux-gnu (2.30-21ubuntu1~18.04.5) ... @@ -1769,14 +1998,14 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-osd1][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. node-admin: [node-osd1][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. node-admin: [node-osd1][DEBUG ] Setting up ceph (15.2.10-1bionic) ... - node-admin: [node-osd1][DEBUG ] Processing triggers for systemd (237-3ubuntu10.41) ... + node-admin: [node-osd1][DEBUG ] Processing triggers for systemd (237-3ubuntu10.44) ... node-admin: [node-osd1][DEBUG ] Processing triggers for man-db (2.8.3-2ubuntu0.1) ... node-admin: [node-osd1][DEBUG ] Processing triggers for ureadahead (0.100.0-21) ... - node-admin: [node-osd1][DEBUG ] Processing triggers for libc-bin (2.27-3ubuntu1) ... + node-admin: [node-osd1][DEBUG ] Processing triggers for libc-bin (2.27-3ubuntu1.4) ... node-admin: [node-osd1][INFO ] Running command: sudo ceph --version node-admin: [node-osd1][DEBUG ] ceph version 15.2.10 (27917a557cca91e4da407489bbaa64ad4352cc02) octopus (stable) node-admin: [ceph_deploy.install][DEBUG ] Detecting platform for host node-osd2 ... - node-admin: Warning: Permanently added 'node-osd2,192.168.121.204' (ECDSA) to the list of known hosts. + node-admin: Warning: Permanently added 'node-osd2,192.168.121.252' (ECDSA) to the list of known hosts. node-admin: [node-osd2][DEBUG ] connection detected need for sudo node-admin: [node-osd2][DEBUG ] connected to host: node-osd2 node-admin: [node-osd2][DEBUG ] detect platform information from remote host @@ -1794,44 +2023,30 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-osd2][DEBUG ] Reading package lists... node-admin: [node-osd2][DEBUG ] Building dependency tree... node-admin: [node-osd2][DEBUG ] Reading state information... + node-admin: [node-osd2][DEBUG ] ca-certificates is already the newest version (20210119~18.04.1). node-admin: [node-osd2][DEBUG ] The following NEW packages will be installed: node-admin: [node-osd2][DEBUG ] apt-transport-https - node-admin: [node-osd2][DEBUG ] The following packages will be upgraded: - node-admin: [node-osd2][DEBUG ] ca-certificates - node-admin: [node-osd2][DEBUG ] 1 upgraded, 1 newly installed, 0 to remove and 171 not upgraded. - node-admin: [node-osd2][DEBUG ] Need to get 148 kB of archives. - node-admin: [node-osd2][DEBUG ] After this operation, 152 kB of additional disk space will be used. - node-admin: [node-osd2][DEBUG ] Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 ca-certificates all 20210119~18.04.1 [147 kB] - node-admin: [node-osd2][DEBUG ] Get:2 http://us.archive.ubuntu.com/ubuntu bionic-updates/universe amd64 apt-transport-https all 1.6.12ubuntu0.2 [1,696 B] - node-admin: [node-osd2][DEBUG ] Preconfiguring packages ... - node-admin: [node-osd2][DEBUG ] Fetched 148 kB in 2s (89.2 kB/s) - node-admin: [node-osd2][DEBUG ] (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 106941 files and directories currently installed.) - node-admin: [node-osd2][DEBUG ] Preparing to unpack .../ca-certificates_20210119~18.04.1_all.deb ... - node-admin: [node-osd2][DEBUG ] Unpacking ca-certificates (20210119~18.04.1) over (20190110~18.04.1) ... + node-admin: [node-osd2][DEBUG ] 0 upgraded, 1 newly installed, 0 to remove and 21 not upgraded. + node-admin: [node-osd2][DEBUG ] Need to get 1,696 B of archives. + node-admin: [node-osd2][DEBUG ] After this operation, 153 kB of additional disk space will be used. + node-admin: [node-osd2][DEBUG ] Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/universe amd64 apt-transport-https all 1.6.12ubuntu0.2 [1,696 B] + node-admin: [node-osd2][DEBUG ] Fetched 1,696 B in 1s (1,378 B/s) node-admin: [node-osd2][DEBUG ] Selecting previously unselected package apt-transport-https. + node-admin: [node-osd2][DEBUG ] (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 106974 files and directories currently installed.) node-admin: [node-osd2][DEBUG ] Preparing to unpack .../apt-transport-https_1.6.12ubuntu0.2_all.deb ... node-admin: [node-osd2][DEBUG ] Unpacking apt-transport-https (1.6.12ubuntu0.2) ... node-admin: [node-osd2][DEBUG ] Setting up apt-transport-https (1.6.12ubuntu0.2) ... - node-admin: [node-osd2][DEBUG ] Setting up ca-certificates (20210119~18.04.1) ... - node-admin: [node-osd2][DEBUG ] Updating certificates in /etc/ssl/certs... - node-admin: [node-osd2][DEBUG ] 21 added, 19 removed; done. - node-admin: [node-osd2][DEBUG ] Processing triggers for man-db (2.8.3-2ubuntu0.1) ... - node-admin: [node-osd2][DEBUG ] Processing triggers for ca-certificates (20210119~18.04.1) ... - node-admin: [node-osd2][DEBUG ] Updating certificates in /etc/ssl/certs... - node-admin: [node-osd2][DEBUG ] 0 added, 0 removed; done. - node-admin: [node-osd2][DEBUG ] Running hooks in /etc/ca-certificates/update.d... - node-admin: [node-osd2][DEBUG ] done. node-admin: [node-osd2][INFO ] Running command: sudo wget -O release.asc https://download.ceph.com/keys/release.asc - node-admin: [node-osd2][WARNING] --2021-03-22 23:49:48-- https://download.ceph.com/keys/release.asc + node-admin: [node-osd2][WARNING] --2021-03-23 01:52:29-- https://download.ceph.com/keys/release.asc node-admin: [node-osd2][WARNING] Resolving download.ceph.com (download.ceph.com)... 158.69.68.124, 2607:5300:201:2000::3:58a1 node-admin: [node-osd2][WARNING] Connecting to download.ceph.com (download.ceph.com)|158.69.68.124|:443... connected. node-admin: [node-osd2][WARNING] HTTP request sent, awaiting response... 200 OK node-admin: [node-osd2][WARNING] Length: 1645 (1.6K) [application/octet-stream] node-admin: [node-osd2][WARNING] Saving to: ‘release.asc’ node-admin: [node-osd2][WARNING] - node-admin: [node-osd2][WARNING] 0K . 100% 215M=0s + node-admin: [node-osd2][WARNING] 0K . 100% 211M=0s node-admin: [node-osd2][WARNING] - node-admin: [node-osd2][WARNING] 2021-03-22 23:49:49 (215 MB/s) - ‘release.asc’ saved [1645/1645] + node-admin: [node-osd2][WARNING] 2021-03-23 01:52:30 (211 MB/s) - ‘release.asc’ saved [1645/1645] node-admin: [node-osd2][WARNING] node-admin: [node-osd2][INFO ] Running command: sudo apt-key add release.asc node-admin: [node-osd2][WARNING] Warning: apt-key output should not be parsed (stdout is not a terminal) @@ -1873,27 +2088,25 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-osd2][DEBUG ] ceph-mgr-rook nvme-cli smartmontools ibverbs-providers python3-lxml node-admin: [node-osd2][DEBUG ] python3-routes python3-simplejson python3-pastescript python3-pyinotify node-admin: [node-osd2][DEBUG ] The following NEW packages will be installed: - node-admin: [node-osd2][DEBUG ] binutils binutils-x86-64-linux-gnu ceph ceph-base ceph-common ceph-mds - node-admin: [node-osd2][DEBUG ] ceph-mgr ceph-mgr-modules-core ceph-mon ceph-osd libaio1 libbabeltrace1 - node-admin: [node-osd2][DEBUG ] libcephfs2 libgoogle-perftools4 libibverbs1 libleveldb1v5 liblttng-ust-ctl4 - node-admin: [node-osd2][DEBUG ] liblttng-ust0 libnl-route-3-200 liboath0 librabbitmq4 librados2 - node-admin: [node-osd2][DEBUG ] libradosstriper1 librbd1 librdkafka1 librdmacm1 librgw2 libsnappy1v5 - node-admin: [node-osd2][DEBUG ] libtcmalloc-minimal4 liburcu6 python-pastedeploy-tpl python3-bcrypt - node-admin: [node-osd2][DEBUG ] python3-bs4 python3-ceph-argparse python3-ceph-common python3-cephfs - node-admin: [node-osd2][DEBUG ] python3-cherrypy3 python3-dateutil python3-distutils python3-jwt - node-admin: [node-osd2][DEBUG ] python3-lib2to3 python3-logutils python3-mako python3-markupsafe - node-admin: [node-osd2][DEBUG ] python3-paste python3-pastedeploy python3-pecan python3-prettytable - node-admin: [node-osd2][DEBUG ] python3-rados python3-rbd python3-rgw python3-simplegeneric - node-admin: [node-osd2][DEBUG ] python3-singledispatch python3-tempita python3-waitress python3-webob - node-admin: [node-osd2][DEBUG ] python3-webtest python3-werkzeug radosgw - node-admin: [node-osd2][DEBUG ] The following packages will be upgraded: - node-admin: [node-osd2][DEBUG ] binutils-common libbinutils - node-admin: [node-osd2][DEBUG ] 2 upgraded, 59 newly installed, 0 to remove and 169 not upgraded. + node-admin: [node-osd2][DEBUG ] binutils binutils-common binutils-x86-64-linux-gnu ceph ceph-base + node-admin: [node-osd2][DEBUG ] ceph-common ceph-mds ceph-mgr ceph-mgr-modules-core ceph-mon ceph-osd + node-admin: [node-osd2][DEBUG ] libaio1 libbabeltrace1 libbinutils libcephfs2 libgoogle-perftools4 + node-admin: [node-osd2][DEBUG ] libibverbs1 libleveldb1v5 liblttng-ust-ctl4 liblttng-ust0 libnl-route-3-200 + node-admin: [node-osd2][DEBUG ] liboath0 librabbitmq4 librados2 libradosstriper1 librbd1 librdkafka1 + node-admin: [node-osd2][DEBUG ] librdmacm1 librgw2 libsnappy1v5 libtcmalloc-minimal4 liburcu6 + node-admin: [node-osd2][DEBUG ] python-pastedeploy-tpl python3-bcrypt python3-bs4 python3-ceph-argparse + node-admin: [node-osd2][DEBUG ] python3-ceph-common python3-cephfs python3-cherrypy3 python3-dateutil + node-admin: [node-osd2][DEBUG ] python3-distutils python3-jwt python3-lib2to3 python3-logutils python3-mako + node-admin: [node-osd2][DEBUG ] python3-markupsafe python3-paste python3-pastedeploy python3-pecan + node-admin: [node-osd2][DEBUG ] python3-prettytable python3-rados python3-rbd python3-rgw + node-admin: [node-osd2][DEBUG ] python3-simplegeneric python3-singledispatch python3-tempita + node-admin: [node-osd2][DEBUG ] python3-waitress python3-webob python3-webtest python3-werkzeug radosgw + node-admin: [node-osd2][DEBUG ] 0 upgraded, 61 newly installed, 0 to remove and 21 not upgraded. node-admin: [node-osd2][DEBUG ] Need to get 79.8 MB of archives. - node-admin: [node-osd2][DEBUG ] After this operation, 350 MB of additional disk space will be used. - node-admin: [node-osd2][DEBUG ] Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libbinutils amd64 2.30-21ubuntu1~18.04.5 [489 kB] + node-admin: [node-osd2][DEBUG ] After this operation, 353 MB of additional disk space will be used. + node-admin: [node-osd2][DEBUG ] Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils-common amd64 2.30-21ubuntu1~18.04.5 [197 kB] node-admin: [node-osd2][DEBUG ] Get:2 https://download.ceph.com/debian-octopus bionic/main amd64 librados2 amd64 15.2.10-1bionic [3,059 kB] - node-admin: [node-osd2][DEBUG ] Get:3 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils-common amd64 2.30-21ubuntu1~18.04.5 [197 kB] + node-admin: [node-osd2][DEBUG ] Get:3 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libbinutils amd64 2.30-21ubuntu1~18.04.5 [489 kB] node-admin: [node-osd2][DEBUG ] Get:4 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils-x86-64-linux-gnu amd64 2.30-21ubuntu1~18.04.5 [1,839 kB] node-admin: [node-osd2][DEBUG ] Get:5 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils amd64 2.30-21ubuntu1~18.04.5 [3,388 B] node-admin: [node-osd2][DEBUG ] Get:6 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libnl-route-3-200 amd64 3.2.29-0ubuntu3 [146 kB] @@ -1952,12 +2165,14 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-osd2][DEBUG ] Get:59 https://download.ceph.com/debian-octopus bionic/main amd64 ceph amd64 15.2.10-1bionic [3,912 B] node-admin: [node-osd2][DEBUG ] Get:60 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-mds amd64 15.2.10-1bionic [1,854 kB] node-admin: [node-osd2][DEBUG ] Get:61 https://download.ceph.com/debian-octopus bionic/main amd64 radosgw amd64 15.2.10-1bionic [8,879 kB] - node-admin: [node-osd2][DEBUG ] Fetched 79.8 MB in 46s (1,718 kB/s) - node-admin: [node-osd2][DEBUG ] (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 106947 files and directories currently installed.) - node-admin: [node-osd2][DEBUG ] Preparing to unpack .../00-libbinutils_2.30-21ubuntu1~18.04.5_amd64.deb ... - node-admin: [node-osd2][DEBUG ] Unpacking libbinutils:amd64 (2.30-21ubuntu1~18.04.5) over (2.30-21ubuntu1~18.04.3) ... - node-admin: [node-osd2][DEBUG ] Preparing to unpack .../01-binutils-common_2.30-21ubuntu1~18.04.5_amd64.deb ... - node-admin: [node-osd2][DEBUG ] Unpacking binutils-common:amd64 (2.30-21ubuntu1~18.04.5) over (2.30-21ubuntu1~18.04.3) ... + node-admin: [node-osd2][DEBUG ] Fetched 79.8 MB in 1min 36s (830 kB/s) + node-admin: [node-osd2][DEBUG ] Selecting previously unselected package binutils-common:amd64. + node-admin: [node-osd2][DEBUG ] (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 106978 files and directories currently installed.) + node-admin: [node-osd2][DEBUG ] Preparing to unpack .../00-binutils-common_2.30-21ubuntu1~18.04.5_amd64.deb ... + node-admin: [node-osd2][DEBUG ] Unpacking binutils-common:amd64 (2.30-21ubuntu1~18.04.5) ... + node-admin: [node-osd2][DEBUG ] Selecting previously unselected package libbinutils:amd64. + node-admin: [node-osd2][DEBUG ] Preparing to unpack .../01-libbinutils_2.30-21ubuntu1~18.04.5_amd64.deb ... + node-admin: [node-osd2][DEBUG ] Unpacking libbinutils:amd64 (2.30-21ubuntu1~18.04.5) ... node-admin: [node-osd2][DEBUG ] Selecting previously unselected package binutils-x86-64-linux-gnu. node-admin: [node-osd2][DEBUG ] Preparing to unpack .../02-binutils-x86-64-linux-gnu_2.30-21ubuntu1~18.04.5_amd64.deb ... node-admin: [node-osd2][DEBUG ] Unpacking binutils-x86-64-linux-gnu (2.30-21ubuntu1~18.04.5) ... @@ -2215,12 +2430,444 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [node-osd2][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. node-admin: [node-osd2][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. node-admin: [node-osd2][DEBUG ] Setting up ceph (15.2.10-1bionic) ... - node-admin: [node-osd2][DEBUG ] Processing triggers for systemd (237-3ubuntu10.41) ... + node-admin: [node-osd2][DEBUG ] Processing triggers for systemd (237-3ubuntu10.44) ... node-admin: [node-osd2][DEBUG ] Processing triggers for man-db (2.8.3-2ubuntu0.1) ... node-admin: [node-osd2][DEBUG ] Processing triggers for ureadahead (0.100.0-21) ... - node-admin: [node-osd2][DEBUG ] Processing triggers for libc-bin (2.27-3ubuntu1) ... + node-admin: [node-osd2][DEBUG ] Processing triggers for libc-bin (2.27-3ubuntu1.4) ... node-admin: [node-osd2][INFO ] Running command: sudo ceph --version node-admin: [node-osd2][DEBUG ] ceph version 15.2.10 (27917a557cca91e4da407489bbaa64ad4352cc02) octopus (stable) + node-admin: [ceph_deploy.install][DEBUG ] Detecting platform for host node-osd3 ... + node-admin: Warning: Permanently added 'node-osd3,192.168.121.236' (ECDSA) to the list of known hosts. + node-admin: [node-osd3][DEBUG ] connection detected need for sudo + node-admin: [node-osd3][DEBUG ] connected to host: node-osd3 + node-admin: [node-osd3][DEBUG ] detect platform information from remote host + node-admin: [node-osd3][DEBUG ] detect machine type + node-admin: [ceph_deploy.install][INFO ] Distro info: Ubuntu 18.04 bionic + node-admin: [node-osd3][INFO ] installing Ceph on node-osd3 + node-admin: [node-osd3][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q update + node-admin: [node-osd3][DEBUG ] Hit:1 http://us.archive.ubuntu.com/ubuntu bionic InRelease + node-admin: [node-osd3][DEBUG ] Hit:2 http://us.archive.ubuntu.com/ubuntu bionic-updates InRelease + node-admin: [node-osd3][DEBUG ] Hit:3 http://us.archive.ubuntu.com/ubuntu bionic-backports InRelease + node-admin: [node-osd3][DEBUG ] Hit:4 http://security.ubuntu.com/ubuntu bionic-security InRelease + node-admin: [node-osd3][DEBUG ] Hit:5 https://download.ceph.com/debian-octopus bionic InRelease + node-admin: [node-osd3][DEBUG ] Reading package lists... + node-admin: [node-osd3][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install ca-certificates apt-transport-https + node-admin: [node-osd3][DEBUG ] Reading package lists... + node-admin: [node-osd3][DEBUG ] Building dependency tree... + node-admin: [node-osd3][DEBUG ] Reading state information... + node-admin: [node-osd3][DEBUG ] ca-certificates is already the newest version (20210119~18.04.1). + node-admin: [node-osd3][DEBUG ] The following NEW packages will be installed: + node-admin: [node-osd3][DEBUG ] apt-transport-https + node-admin: [node-osd3][DEBUG ] 0 upgraded, 1 newly installed, 0 to remove and 21 not upgraded. + node-admin: [node-osd3][DEBUG ] Need to get 1,696 B of archives. + node-admin: [node-osd3][DEBUG ] After this operation, 153 kB of additional disk space will be used. + node-admin: [node-osd3][DEBUG ] Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/universe amd64 apt-transport-https all 1.6.12ubuntu0.2 [1,696 B] + node-admin: [node-osd3][DEBUG ] Fetched 1,696 B in 1s (3,252 B/s) + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package apt-transport-https. + node-admin: [node-osd3][DEBUG ] (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 106974 files and directories currently installed.) + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../apt-transport-https_1.6.12ubuntu0.2_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking apt-transport-https (1.6.12ubuntu0.2) ... + node-admin: [node-osd3][DEBUG ] Setting up apt-transport-https (1.6.12ubuntu0.2) ... + node-admin: [node-osd3][INFO ] Running command: sudo wget -O release.asc https://download.ceph.com/keys/release.asc + node-admin: [node-osd3][WARNING] --2021-03-23 01:56:54-- https://download.ceph.com/keys/release.asc + node-admin: [node-osd3][WARNING] Resolving download.ceph.com (download.ceph.com)... 158.69.68.124, 2607:5300:201:2000::3:58a1 + node-admin: [node-osd3][WARNING] Connecting to download.ceph.com (download.ceph.com)|158.69.68.124|:443... connected. + node-admin: [node-osd3][WARNING] HTTP request sent, awaiting response... 200 OK + node-admin: [node-osd3][WARNING] Length: 1645 (1.6K) [application/octet-stream] + node-admin: [node-osd3][WARNING] Saving to: ‘release.asc’ + node-admin: [node-osd3][WARNING] + node-admin: [node-osd3][WARNING] 0K . 100% 156M=0s + node-admin: [node-osd3][WARNING] + node-admin: [node-osd3][WARNING] 2021-03-23 01:56:55 (156 MB/s) - ‘release.asc’ saved [1645/1645] + node-admin: [node-osd3][WARNING] + node-admin: [node-osd3][INFO ] Running command: sudo apt-key add release.asc + node-admin: [node-osd3][WARNING] Warning: apt-key output should not be parsed (stdout is not a terminal) + node-admin: [node-osd3][DEBUG ] OK + node-admin: [node-osd3][DEBUG ] add deb repo to /etc/apt/sources.list.d/ + node-admin: [node-osd3][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q update + node-admin: [node-osd3][DEBUG ] Hit:1 http://us.archive.ubuntu.com/ubuntu bionic InRelease + node-admin: [node-osd3][DEBUG ] Hit:2 http://us.archive.ubuntu.com/ubuntu bionic-updates InRelease + node-admin: [node-osd3][DEBUG ] Hit:3 https://download.ceph.com/debian-octopus bionic InRelease + node-admin: [node-osd3][DEBUG ] Hit:4 http://us.archive.ubuntu.com/ubuntu bionic-backports InRelease + node-admin: [node-osd3][DEBUG ] Hit:5 http://security.ubuntu.com/ubuntu bionic-security InRelease + node-admin: [node-osd3][DEBUG ] Reading package lists... + node-admin: [node-osd3][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install -o Dpkg::Options::=--force-confnew ceph ceph-osd ceph-mds ceph-mon radosgw + node-admin: [node-osd3][DEBUG ] Reading package lists... + node-admin: [node-osd3][DEBUG ] Building dependency tree... + node-admin: [node-osd3][DEBUG ] Reading state information... + node-admin: [node-osd3][DEBUG ] The following additional packages will be installed: + node-admin: [node-osd3][DEBUG ] binutils binutils-common binutils-x86-64-linux-gnu ceph-base ceph-common + node-admin: [node-osd3][DEBUG ] ceph-mgr ceph-mgr-modules-core libaio1 libbabeltrace1 libbinutils libcephfs2 + node-admin: [node-osd3][DEBUG ] libgoogle-perftools4 libibverbs1 libleveldb1v5 liblttng-ust-ctl4 + node-admin: [node-osd3][DEBUG ] liblttng-ust0 libnl-route-3-200 liboath0 librabbitmq4 librados2 + node-admin: [node-osd3][DEBUG ] libradosstriper1 librbd1 librdkafka1 librdmacm1 librgw2 libsnappy1v5 + node-admin: [node-osd3][DEBUG ] libtcmalloc-minimal4 liburcu6 python-pastedeploy-tpl python3-bcrypt + node-admin: [node-osd3][DEBUG ] python3-bs4 python3-ceph-argparse python3-ceph-common python3-cephfs + node-admin: [node-osd3][DEBUG ] python3-cherrypy3 python3-dateutil python3-distutils python3-jwt + node-admin: [node-osd3][DEBUG ] python3-lib2to3 python3-logutils python3-mako python3-markupsafe + node-admin: [node-osd3][DEBUG ] python3-paste python3-pastedeploy python3-pecan python3-prettytable + node-admin: [node-osd3][DEBUG ] python3-rados python3-rbd python3-rgw python3-simplegeneric + node-admin: [node-osd3][DEBUG ] python3-singledispatch python3-tempita python3-waitress python3-webob + node-admin: [node-osd3][DEBUG ] python3-webtest python3-werkzeug + node-admin: [node-osd3][DEBUG ] Suggested packages: + node-admin: [node-osd3][DEBUG ] binutils-doc python3-influxdb python3-crypto python3-beaker python-mako-doc + node-admin: [node-osd3][DEBUG ] httpd-wsgi libapache2-mod-python libapache2-mod-scgi libjs-mochikit + node-admin: [node-osd3][DEBUG ] python-pecan-doc python-waitress-doc python-webob-doc python-webtest-doc + node-admin: [node-osd3][DEBUG ] ipython3 python3-lxml python3-termcolor python3-watchdog python-werkzeug-doc + node-admin: [node-osd3][DEBUG ] Recommended packages: + node-admin: [node-osd3][DEBUG ] ceph-fuse ceph-mgr-dashboard ceph-mgr-diskprediction-local + node-admin: [node-osd3][DEBUG ] ceph-mgr-diskprediction-cloud ceph-mgr-k8sevents ceph-mgr-cephadm + node-admin: [node-osd3][DEBUG ] ceph-mgr-rook nvme-cli smartmontools ibverbs-providers python3-lxml + node-admin: [node-osd3][DEBUG ] python3-routes python3-simplejson python3-pastescript python3-pyinotify + node-admin: [node-osd3][DEBUG ] The following NEW packages will be installed: + node-admin: [node-osd3][DEBUG ] binutils binutils-common binutils-x86-64-linux-gnu ceph ceph-base + node-admin: [node-osd3][DEBUG ] ceph-common ceph-mds ceph-mgr ceph-mgr-modules-core ceph-mon ceph-osd + node-admin: [node-osd3][DEBUG ] libaio1 libbabeltrace1 libbinutils libcephfs2 libgoogle-perftools4 + node-admin: [node-osd3][DEBUG ] libibverbs1 libleveldb1v5 liblttng-ust-ctl4 liblttng-ust0 libnl-route-3-200 + node-admin: [node-osd3][DEBUG ] liboath0 librabbitmq4 librados2 libradosstriper1 librbd1 librdkafka1 + node-admin: [node-osd3][DEBUG ] librdmacm1 librgw2 libsnappy1v5 libtcmalloc-minimal4 liburcu6 + node-admin: [node-osd3][DEBUG ] python-pastedeploy-tpl python3-bcrypt python3-bs4 python3-ceph-argparse + node-admin: [node-osd3][DEBUG ] python3-ceph-common python3-cephfs python3-cherrypy3 python3-dateutil + node-admin: [node-osd3][DEBUG ] python3-distutils python3-jwt python3-lib2to3 python3-logutils python3-mako + node-admin: [node-osd3][DEBUG ] python3-markupsafe python3-paste python3-pastedeploy python3-pecan + node-admin: [node-osd3][DEBUG ] python3-prettytable python3-rados python3-rbd python3-rgw + node-admin: [node-osd3][DEBUG ] python3-simplegeneric python3-singledispatch python3-tempita + node-admin: [node-osd3][DEBUG ] python3-waitress python3-webob python3-webtest python3-werkzeug radosgw + node-admin: [node-osd3][DEBUG ] 0 upgraded, 61 newly installed, 0 to remove and 21 not upgraded. + node-admin: [node-osd3][DEBUG ] Need to get 79.8 MB of archives. + node-admin: [node-osd3][DEBUG ] After this operation, 353 MB of additional disk space will be used. + node-admin: [node-osd3][DEBUG ] Get:1 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils-common amd64 2.30-21ubuntu1~18.04.5 [197 kB] + node-admin: [node-osd3][DEBUG ] Get:2 https://download.ceph.com/debian-octopus bionic/main amd64 librados2 amd64 15.2.10-1bionic [3,059 kB] + node-admin: [node-osd3][DEBUG ] Get:3 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libbinutils amd64 2.30-21ubuntu1~18.04.5 [489 kB] + node-admin: [node-osd3][DEBUG ] Get:4 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils-x86-64-linux-gnu amd64 2.30-21ubuntu1~18.04.5 [1,839 kB] + node-admin: [node-osd3][DEBUG ] Get:5 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 binutils amd64 2.30-21ubuntu1~18.04.5 [3,388 B] + node-admin: [node-osd3][DEBUG ] Get:6 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libnl-route-3-200 amd64 3.2.29-0ubuntu3 [146 kB] + node-admin: [node-osd3][DEBUG ] Get:7 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libibverbs1 amd64 17.1-1ubuntu0.2 [44.4 kB] + node-admin: [node-osd3][DEBUG ] Get:8 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 liburcu6 amd64 0.10.1-1ubuntu1 [52.2 kB] + node-admin: [node-osd3][DEBUG ] Get:9 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 liblttng-ust-ctl4 amd64 2.10.1-1 [80.8 kB] + node-admin: [node-osd3][DEBUG ] Get:10 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 liblttng-ust0 amd64 2.10.1-1 [154 kB] + node-admin: [node-osd3][DEBUG ] Get:11 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 librdmacm1 amd64 17.1-1ubuntu0.2 [56.1 kB] + node-admin: [node-osd3][DEBUG ] Get:12 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python3-prettytable all 0.7.2-3 [19.7 kB] + node-admin: [node-osd3][DEBUG ] Get:13 http://us.archive.ubuntu.com/ubuntu bionic-updates/universe amd64 librabbitmq4 amd64 0.8.0-1ubuntu0.18.04.2 [33.9 kB] + node-admin: [node-osd3][DEBUG ] Get:14 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 librdkafka1 amd64 0.11.3-1build1 [293 kB] + node-admin: [node-osd3][DEBUG ] Get:15 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 libaio1 amd64 0.3.110-5ubuntu0.1 [6,476 B] + node-admin: [node-osd3][DEBUG ] Get:16 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libbabeltrace1 amd64 1.5.5-1 [154 kB] + node-admin: [node-osd3][DEBUG ] Get:17 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libtcmalloc-minimal4 amd64 2.5-2.2ubuntu3 [91.6 kB] + node-admin: [node-osd3][DEBUG ] Get:18 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libgoogle-perftools4 amd64 2.5-2.2ubuntu3 [190 kB] + node-admin: [node-osd3][DEBUG ] Get:19 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libsnappy1v5 amd64 1.1.7-1 [16.0 kB] + node-admin: [node-osd3][DEBUG ] Get:20 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 libleveldb1v5 amd64 1.20-2 [136 kB] + node-admin: [node-osd3][DEBUG ] Get:21 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 liboath0 amd64 2.6.1-1 [44.7 kB] + node-admin: [node-osd3][DEBUG ] Get:22 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python3-dateutil all 2.6.1-1 [52.3 kB] + node-admin: [node-osd3][DEBUG ] Get:23 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 python3-bcrypt amd64 3.1.4-2 [29.9 kB] + node-admin: [node-osd3][DEBUG ] Get:24 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 python3-cherrypy3 all 8.9.1-2 [160 kB] + node-admin: [node-osd3][DEBUG ] Get:25 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-lib2to3 all 3.6.9-1~18.04 [77.4 kB] + node-admin: [node-osd3][DEBUG ] Get:26 http://us.archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-distutils all 3.6.9-1~18.04 [144 kB] + node-admin: [node-osd3][DEBUG ] Get:27 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python3-jwt all 1.5.3+ds1-1 [15.9 kB] + node-admin: [node-osd3][DEBUG ] Get:28 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 python3-logutils all 0.3.3-5 [16.7 kB] + node-admin: [node-osd3][DEBUG ] Get:29 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python3-markupsafe amd64 1.0-1build1 [13.5 kB] + node-admin: [node-osd3][DEBUG ] Get:30 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python3-mako all 1.0.7+ds1-1 [59.3 kB] + node-admin: [node-osd3][DEBUG ] Get:31 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 python3-simplegeneric all 0.8.1-1 [11.5 kB] + node-admin: [node-osd3][DEBUG ] Get:32 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 python3-singledispatch all 3.4.0.3-2 [7,022 B] + node-admin: [node-osd3][DEBUG ] Get:33 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python3-webob all 1:1.7.3-2fakesync1 [64.3 kB] + node-admin: [node-osd3][DEBUG ] Get:34 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python3-bs4 all 4.6.0-1 [67.8 kB] + node-admin: [node-osd3][DEBUG ] Get:35 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python3-waitress all 1.0.1-1 [53.4 kB] + node-admin: [node-osd3][DEBUG ] Get:36 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python3-tempita all 0.5.2-2 [13.9 kB] + node-admin: [node-osd3][DEBUG ] Get:37 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python3-paste all 2.0.3+dfsg-4ubuntu1 [456 kB] + node-admin: [node-osd3][DEBUG ] Get:38 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python-pastedeploy-tpl all 1.5.2-4 [4,796 B] + node-admin: [node-osd3][DEBUG ] Get:39 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python3-pastedeploy all 1.5.2-4 [13.4 kB] + node-admin: [node-osd3][DEBUG ] Get:40 http://us.archive.ubuntu.com/ubuntu bionic/main amd64 python3-webtest all 2.0.28-1ubuntu1 [27.9 kB] + node-admin: [node-osd3][DEBUG ] Get:41 http://us.archive.ubuntu.com/ubuntu bionic/universe amd64 python3-pecan all 1.2.1-2 [86.1 kB] + node-admin: [node-osd3][DEBUG ] Get:42 http://us.archive.ubuntu.com/ubuntu bionic-updates/universe amd64 python3-werkzeug all 0.14.1+dfsg1-1ubuntu0.1 [174 kB] + node-admin: [node-osd3][DEBUG ] Get:43 https://download.ceph.com/debian-octopus bionic/main amd64 librbd1 amd64 15.2.10-1bionic [1,639 kB] + node-admin: [node-osd3][DEBUG ] Get:44 https://download.ceph.com/debian-octopus bionic/main amd64 libcephfs2 amd64 15.2.10-1bionic [435 kB] + node-admin: [node-osd3][DEBUG ] Get:45 https://download.ceph.com/debian-octopus bionic/main amd64 python3-rados amd64 15.2.10-1bionic [345 kB] + node-admin: [node-osd3][DEBUG ] Get:46 https://download.ceph.com/debian-octopus bionic/main amd64 python3-ceph-argparse all 15.2.10-1bionic [20.7 kB] + node-admin: [node-osd3][DEBUG ] Get:47 https://download.ceph.com/debian-octopus bionic/main amd64 python3-cephfs amd64 15.2.10-1bionic [133 kB] + node-admin: [node-osd3][DEBUG ] Get:48 https://download.ceph.com/debian-octopus bionic/main amd64 python3-ceph-common all 15.2.10-1bionic [29.0 kB] + node-admin: [node-osd3][DEBUG ] Get:49 https://download.ceph.com/debian-octopus bionic/main amd64 python3-rbd amd64 15.2.10-1bionic [326 kB] + node-admin: [node-osd3][DEBUG ] Get:50 https://download.ceph.com/debian-octopus bionic/main amd64 librgw2 amd64 15.2.10-1bionic [2,901 kB] + node-admin: [node-osd3][DEBUG ] Get:51 https://download.ceph.com/debian-octopus bionic/main amd64 python3-rgw amd64 15.2.10-1bionic [99.7 kB] + node-admin: [node-osd3][DEBUG ] Get:52 https://download.ceph.com/debian-octopus bionic/main amd64 libradosstriper1 amd64 15.2.10-1bionic [338 kB] + node-admin: [node-osd3][DEBUG ] Get:53 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-common amd64 15.2.10-1bionic [18.8 MB] + node-admin: [node-osd3][DEBUG ] Get:54 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-base amd64 15.2.10-1bionic [5,173 kB] + node-admin: [node-osd3][DEBUG ] Get:55 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-mgr-modules-core all 15.2.10-1bionic [161 kB] + node-admin: [node-osd3][DEBUG ] Get:56 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-mgr amd64 15.2.10-1bionic [1,285 kB] + node-admin: [node-osd3][DEBUG ] Get:57 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-mon amd64 15.2.10-1bionic [5,942 kB] + node-admin: [node-osd3][DEBUG ] Get:58 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-osd amd64 15.2.10-1bionic [22.8 MB] + node-admin: [node-osd3][DEBUG ] Get:59 https://download.ceph.com/debian-octopus bionic/main amd64 ceph amd64 15.2.10-1bionic [3,912 B] + node-admin: [node-osd3][DEBUG ] Get:60 https://download.ceph.com/debian-octopus bionic/main amd64 ceph-mds amd64 15.2.10-1bionic [1,854 kB] + node-admin: [node-osd3][DEBUG ] Get:61 https://download.ceph.com/debian-octopus bionic/main amd64 radosgw amd64 15.2.10-1bionic [8,879 kB] + node-admin: [node-osd3][DEBUG ] Fetched 79.8 MB in 1min 38s (813 kB/s) + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package binutils-common:amd64. + node-admin: [node-osd3][DEBUG ] (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 106978 files and directories currently installed.) + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../00-binutils-common_2.30-21ubuntu1~18.04.5_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking binutils-common:amd64 (2.30-21ubuntu1~18.04.5) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package libbinutils:amd64. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../01-libbinutils_2.30-21ubuntu1~18.04.5_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking libbinutils:amd64 (2.30-21ubuntu1~18.04.5) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package binutils-x86-64-linux-gnu. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../02-binutils-x86-64-linux-gnu_2.30-21ubuntu1~18.04.5_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking binutils-x86-64-linux-gnu (2.30-21ubuntu1~18.04.5) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package binutils. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../03-binutils_2.30-21ubuntu1~18.04.5_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking binutils (2.30-21ubuntu1~18.04.5) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package libnl-route-3-200:amd64. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../04-libnl-route-3-200_3.2.29-0ubuntu3_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking libnl-route-3-200:amd64 (3.2.29-0ubuntu3) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package libibverbs1:amd64. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../05-libibverbs1_17.1-1ubuntu0.2_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking libibverbs1:amd64 (17.1-1ubuntu0.2) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package liburcu6:amd64. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../06-liburcu6_0.10.1-1ubuntu1_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking liburcu6:amd64 (0.10.1-1ubuntu1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package liblttng-ust-ctl4:amd64. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../07-liblttng-ust-ctl4_2.10.1-1_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking liblttng-ust-ctl4:amd64 (2.10.1-1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package liblttng-ust0:amd64. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../08-liblttng-ust0_2.10.1-1_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking liblttng-ust0:amd64 (2.10.1-1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package librdmacm1:amd64. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../09-librdmacm1_17.1-1ubuntu0.2_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking librdmacm1:amd64 (17.1-1ubuntu0.2) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package librados2. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../10-librados2_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking librados2 (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package librbd1. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../11-librbd1_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking librbd1 (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package libcephfs2. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../12-libcephfs2_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking libcephfs2 (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-rados. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../13-python3-rados_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-rados (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-ceph-argparse. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../14-python3-ceph-argparse_15.2.10-1bionic_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-ceph-argparse (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-cephfs. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../15-python3-cephfs_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-cephfs (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-ceph-common. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../16-python3-ceph-common_15.2.10-1bionic_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-ceph-common (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-prettytable. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../17-python3-prettytable_0.7.2-3_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-prettytable (0.7.2-3) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-rbd. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../18-python3-rbd_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-rbd (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package librabbitmq4:amd64. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../19-librabbitmq4_0.8.0-1ubuntu0.18.04.2_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking librabbitmq4:amd64 (0.8.0-1ubuntu0.18.04.2) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package librdkafka1:amd64. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../20-librdkafka1_0.11.3-1build1_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking librdkafka1:amd64 (0.11.3-1build1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package librgw2. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../21-librgw2_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking librgw2 (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-rgw. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../22-python3-rgw_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-rgw (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package libaio1:amd64. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../23-libaio1_0.3.110-5ubuntu0.1_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking libaio1:amd64 (0.3.110-5ubuntu0.1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package libbabeltrace1:amd64. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../24-libbabeltrace1_1.5.5-1_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking libbabeltrace1:amd64 (1.5.5-1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package libtcmalloc-minimal4. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../25-libtcmalloc-minimal4_2.5-2.2ubuntu3_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking libtcmalloc-minimal4 (2.5-2.2ubuntu3) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package libgoogle-perftools4. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../26-libgoogle-perftools4_2.5-2.2ubuntu3_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking libgoogle-perftools4 (2.5-2.2ubuntu3) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package libsnappy1v5:amd64. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../27-libsnappy1v5_1.1.7-1_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking libsnappy1v5:amd64 (1.1.7-1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package libleveldb1v5:amd64. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../28-libleveldb1v5_1.20-2_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking libleveldb1v5:amd64 (1.20-2) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package liboath0. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../29-liboath0_2.6.1-1_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking liboath0 (2.6.1-1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package libradosstriper1. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../30-libradosstriper1_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking libradosstriper1 (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package ceph-common. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../31-ceph-common_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking ceph-common (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package ceph-base. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../32-ceph-base_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking ceph-base (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-dateutil. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../33-python3-dateutil_2.6.1-1_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-dateutil (2.6.1-1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package ceph-mgr-modules-core. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../34-ceph-mgr-modules-core_15.2.10-1bionic_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking ceph-mgr-modules-core (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-bcrypt. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../35-python3-bcrypt_3.1.4-2_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-bcrypt (3.1.4-2) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-cherrypy3. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../36-python3-cherrypy3_8.9.1-2_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-cherrypy3 (8.9.1-2) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-lib2to3. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../37-python3-lib2to3_3.6.9-1~18.04_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-lib2to3 (3.6.9-1~18.04) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-distutils. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../38-python3-distutils_3.6.9-1~18.04_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-distutils (3.6.9-1~18.04) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-jwt. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../39-python3-jwt_1.5.3+ds1-1_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-jwt (1.5.3+ds1-1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-logutils. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../40-python3-logutils_0.3.3-5_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-logutils (0.3.3-5) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-markupsafe. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../41-python3-markupsafe_1.0-1build1_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-markupsafe (1.0-1build1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-mako. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../42-python3-mako_1.0.7+ds1-1_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-mako (1.0.7+ds1-1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-simplegeneric. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../43-python3-simplegeneric_0.8.1-1_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-simplegeneric (0.8.1-1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-singledispatch. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../44-python3-singledispatch_3.4.0.3-2_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-singledispatch (3.4.0.3-2) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-webob. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../45-python3-webob_1%3a1.7.3-2fakesync1_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-webob (1:1.7.3-2fakesync1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-bs4. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../46-python3-bs4_4.6.0-1_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-bs4 (4.6.0-1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-waitress. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../47-python3-waitress_1.0.1-1_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-waitress (1.0.1-1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-tempita. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../48-python3-tempita_0.5.2-2_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-tempita (0.5.2-2) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-paste. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../49-python3-paste_2.0.3+dfsg-4ubuntu1_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-paste (2.0.3+dfsg-4ubuntu1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python-pastedeploy-tpl. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../50-python-pastedeploy-tpl_1.5.2-4_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python-pastedeploy-tpl (1.5.2-4) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-pastedeploy. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../51-python3-pastedeploy_1.5.2-4_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-pastedeploy (1.5.2-4) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-webtest. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../52-python3-webtest_2.0.28-1ubuntu1_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-webtest (2.0.28-1ubuntu1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-pecan. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../53-python3-pecan_1.2.1-2_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-pecan (1.2.1-2) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package python3-werkzeug. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../54-python3-werkzeug_0.14.1+dfsg1-1ubuntu0.1_all.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking python3-werkzeug (0.14.1+dfsg1-1ubuntu0.1) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package ceph-mgr. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../55-ceph-mgr_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking ceph-mgr (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package ceph-mon. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../56-ceph-mon_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking ceph-mon (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package ceph-osd. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../57-ceph-osd_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking ceph-osd (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package ceph. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../58-ceph_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking ceph (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package ceph-mds. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../59-ceph-mds_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking ceph-mds (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Selecting previously unselected package radosgw. + node-admin: [node-osd3][DEBUG ] Preparing to unpack .../60-radosgw_15.2.10-1bionic_amd64.deb ... + node-admin: [node-osd3][DEBUG ] Unpacking radosgw (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-logutils (0.3.3-5) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-werkzeug (0.14.1+dfsg1-1ubuntu0.1) ... + node-admin: [node-osd3][DEBUG ] Setting up librdkafka1:amd64 (0.11.3-1build1) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-simplegeneric (0.8.1-1) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-waitress (1.0.1-1) ... + node-admin: [node-osd3][DEBUG ] update-alternatives: using /usr/bin/waitress-serve-python3 to provide /usr/bin/waitress-serve (waitress-serve) in auto mode + node-admin: [node-osd3][DEBUG ] Setting up python3-ceph-argparse (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-tempita (0.5.2-2) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-webob (1:1.7.3-2fakesync1) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-bcrypt (3.1.4-2) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-singledispatch (3.4.0.3-2) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-cherrypy3 (8.9.1-2) ... + node-admin: [node-osd3][DEBUG ] Setting up libnl-route-3-200:amd64 (3.2.29-0ubuntu3) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-bs4 (4.6.0-1) ... + node-admin: [node-osd3][DEBUG ] Setting up liburcu6:amd64 (0.10.1-1ubuntu1) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-markupsafe (1.0-1build1) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-prettytable (0.7.2-3) ... + node-admin: [node-osd3][DEBUG ] Setting up binutils-common:amd64 (2.30-21ubuntu1~18.04.5) ... + node-admin: [node-osd3][DEBUG ] Setting up liblttng-ust-ctl4:amd64 (2.10.1-1) ... + node-admin: [node-osd3][DEBUG ] Setting up libtcmalloc-minimal4 (2.5-2.2ubuntu3) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-ceph-common (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Setting up libgoogle-perftools4 (2.5-2.2ubuntu3) ... + node-admin: [node-osd3][DEBUG ] Setting up libaio1:amd64 (0.3.110-5ubuntu0.1) ... + node-admin: [node-osd3][DEBUG ] Setting up libsnappy1v5:amd64 (1.1.7-1) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-paste (2.0.3+dfsg-4ubuntu1) ... + node-admin: [node-osd3][DEBUG ] Setting up liboath0 (2.6.1-1) ... + node-admin: [node-osd3][DEBUG ] Setting up librabbitmq4:amd64 (0.8.0-1ubuntu0.18.04.2) ... + node-admin: [node-osd3][DEBUG ] Setting up python-pastedeploy-tpl (1.5.2-4) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-lib2to3 (3.6.9-1~18.04) ... + node-admin: [node-osd3][DEBUG ] Setting up liblttng-ust0:amd64 (2.10.1-1) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-distutils (3.6.9-1~18.04) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-jwt (1.5.3+ds1-1) ... + node-admin: [node-osd3][DEBUG ] Setting up libbabeltrace1:amd64 (1.5.5-1) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-dateutil (2.6.1-1) ... + node-admin: [node-osd3][DEBUG ] Setting up libibverbs1:amd64 (17.1-1ubuntu0.2) ... + node-admin: [node-osd3][DEBUG ] Setting up libbinutils:amd64 (2.30-21ubuntu1~18.04.5) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-mako (1.0.7+ds1-1) ... + node-admin: [node-osd3][DEBUG ] Setting up librdmacm1:amd64 (17.1-1ubuntu0.2) ... + node-admin: [node-osd3][DEBUG ] Setting up libleveldb1v5:amd64 (1.20-2) ... + node-admin: [node-osd3][DEBUG ] Setting up librados2 (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Setting up libcephfs2 (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Setting up ceph-mgr-modules-core (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-rados (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-pastedeploy (1.5.2-4) ... + node-admin: [node-osd3][DEBUG ] Setting up binutils-x86-64-linux-gnu (2.30-21ubuntu1~18.04.5) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-webtest (2.0.28-1ubuntu1) ... + node-admin: [node-osd3][DEBUG ] Setting up libradosstriper1 (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-cephfs (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Setting up librgw2 (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Setting up librbd1 (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-pecan (1.2.1-2) ... + node-admin: [node-osd3][DEBUG ] update-alternatives: using /usr/bin/python3-pecan to provide /usr/bin/pecan (pecan) in auto mode + node-admin: [node-osd3][DEBUG ] update-alternatives: using /usr/bin/python3-gunicorn_pecan to provide /usr/bin/gunicorn_pecan (gunicorn_pecan) in auto mode + node-admin: [node-osd3][DEBUG ] Setting up binutils (2.30-21ubuntu1~18.04.5) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-rgw (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Setting up python3-rbd (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Setting up ceph-common (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Adding group ceph....done + node-admin: [node-osd3][DEBUG ] Adding system user ceph....done + node-admin: [node-osd3][DEBUG ] Setting system user ceph properties....done + node-admin: [node-osd3][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /lib/systemd/system/ceph.target. + node-admin: [node-osd3][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service → /lib/systemd/system/rbdmap.service. + node-admin: [node-osd3][DEBUG ] Setting up radosgw (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. + node-admin: [node-osd3][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. + node-admin: [node-osd3][DEBUG ] Setting up ceph-base (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /lib/systemd/system/ceph-crash.service. + node-admin: [node-osd3][DEBUG ] Setting up ceph-mgr (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. + node-admin: [node-osd3][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. + node-admin: [node-osd3][DEBUG ] Setting up ceph-osd (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. + node-admin: [node-osd3][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. + node-admin: [node-osd3][DEBUG ] Setting up ceph-mds (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. + node-admin: [node-osd3][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. + node-admin: [node-osd3][DEBUG ] Setting up ceph-mon (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. + node-admin: [node-osd3][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. + node-admin: [node-osd3][DEBUG ] Setting up ceph (15.2.10-1bionic) ... + node-admin: [node-osd3][DEBUG ] Processing triggers for systemd (237-3ubuntu10.44) ... + node-admin: [node-osd3][DEBUG ] Processing triggers for man-db (2.8.3-2ubuntu0.1) ... + node-admin: [node-osd3][DEBUG ] Processing triggers for ureadahead (0.100.0-21) ... + node-admin: [node-osd3][DEBUG ] Processing triggers for libc-bin (2.27-3ubuntu1.4) ... + node-admin: [node-osd3][INFO ] Running command: sudo ceph --version + node-admin: [node-osd3][DEBUG ] ceph version 15.2.10 (27917a557cca91e4da407489bbaa64ad4352cc02) octopus (stable) node-admin: node-admin: {cephtest} {ceph-install} Create initial monitor node-admin: [ceph_deploy.conf][DEBUG ] found configuration file at: /home/ceph-admin/.cephdeploy.conf @@ -2231,29 +2878,199 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [ceph_deploy.cli][INFO ] overwrite_conf : True node-admin: [ceph_deploy.cli][INFO ] subcommand : create-initial node-admin: [ceph_deploy.cli][INFO ] quiet : False - node-admin: [ceph_deploy.cli][INFO ] cd_conf : + node-admin: [ceph_deploy.cli][INFO ] cd_conf : node-admin: [ceph_deploy.cli][INFO ] cluster : ceph - node-admin: [ceph_deploy.cli][INFO ] func : + node-admin: [ceph_deploy.cli][INFO ] func : node-admin: [ceph_deploy.cli][INFO ] ceph_conf : None node-admin: [ceph_deploy.cli][INFO ] keyrings : None node-admin: [ceph_deploy.cli][INFO ] default_release : False - node-admin: [ceph_deploy][ERROR ] NeedHostError: No hosts specified to deploy to.: could not find `mon initial members` defined in ceph.conf + node-admin: [ceph_deploy.mon][DEBUG ] Deploying mon, cluster ceph hosts node-admin + node-admin: [ceph_deploy.mon][DEBUG ] detecting platform for host node-admin ... + node-admin: [node-admin][DEBUG ] connection detected need for sudo + node-admin: [node-admin][DEBUG ] connected to host: node-admin + node-admin: [node-admin][DEBUG ] detect platform information from remote host + node-admin: [node-admin][DEBUG ] detect machine type + node-admin: [node-admin][DEBUG ] find the location of an executable + node-admin: [ceph_deploy.mon][INFO ] distro info: Ubuntu 18.04 bionic + node-admin: [node-admin][DEBUG ] determining if provided host has same hostname in remote + node-admin: [node-admin][DEBUG ] get remote short hostname + node-admin: [node-admin][DEBUG ] deploying mon to node-admin + node-admin: [node-admin][DEBUG ] get remote short hostname + node-admin: [node-admin][DEBUG ] remote hostname: node-admin + node-admin: [node-admin][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [node-admin][DEBUG ] create the mon path if it does not exist + node-admin: [node-admin][DEBUG ] checking for done path: /var/lib/ceph/mon/ceph-node-admin/done + node-admin: [node-admin][DEBUG ] done path does not exist: /var/lib/ceph/mon/ceph-node-admin/done + node-admin: [node-admin][INFO ] creating keyring file: /var/lib/ceph/tmp/ceph-node-admin.mon.keyring + node-admin: [node-admin][DEBUG ] create the monitor keyring file + node-admin: [node-admin][INFO ] Running command: sudo ceph-mon --cluster ceph --mkfs -i node-admin --keyring /var/lib/ceph/tmp/ceph-node-admin.mon.keyring --setuser 64045 --setgroup 64045 + node-admin: [node-admin][INFO ] unlinking keyring file /var/lib/ceph/tmp/ceph-node-admin.mon.keyring + node-admin: [node-admin][DEBUG ] create a done file to avoid re-doing the mon deployment + node-admin: [node-admin][DEBUG ] create the init path if it does not exist + node-admin: [node-admin][INFO ] Running command: sudo systemctl enable ceph.target + node-admin: [node-admin][INFO ] Running command: sudo systemctl enable ceph-mon@node-admin + node-admin: [node-admin][WARNING] Created symlink /etc/systemd/system/ceph-mon.target.wants/ceph-mon@node-admin.service → /lib/systemd/system/ceph-mon@.service. + node-admin: [node-admin][INFO ] Running command: sudo systemctl start ceph-mon@node-admin + node-admin: [node-admin][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-admin.asok mon_status + node-admin: [node-admin][DEBUG ] ******************************************************************************** + node-admin: [node-admin][DEBUG ] status for monitor: mon.node-admin + node-admin: [node-admin][DEBUG ] { + node-admin: [node-admin][DEBUG ] "election_epoch": 3, + node-admin: [node-admin][DEBUG ] "extra_probe_peers": [], + node-admin: [node-admin][DEBUG ] "feature_map": { + node-admin: [node-admin][DEBUG ] "mon": [ + node-admin: [node-admin][DEBUG ] { + node-admin: [node-admin][DEBUG ] "features": "0x3f01cfb8ffedffff", + node-admin: [node-admin][DEBUG ] "num": 1, + node-admin: [node-admin][DEBUG ] "release": "luminous" + node-admin: [node-admin][DEBUG ] } + node-admin: [node-admin][DEBUG ] ] + node-admin: [node-admin][DEBUG ] }, + node-admin: [node-admin][DEBUG ] "features": { + node-admin: [node-admin][DEBUG ] "quorum_con": "4540138292840890367", + node-admin: [node-admin][DEBUG ] "quorum_mon": [ + node-admin: [node-admin][DEBUG ] "kraken", + node-admin: [node-admin][DEBUG ] "luminous", + node-admin: [node-admin][DEBUG ] "mimic", + node-admin: [node-admin][DEBUG ] "osdmap-prune", + node-admin: [node-admin][DEBUG ] "nautilus", + node-admin: [node-admin][DEBUG ] "octopus" + node-admin: [node-admin][DEBUG ] ], + node-admin: [node-admin][DEBUG ] "required_con": "2449958747315978244", + node-admin: [node-admin][DEBUG ] "required_mon": [ + node-admin: [node-admin][DEBUG ] "kraken", + node-admin: [node-admin][DEBUG ] "luminous", + node-admin: [node-admin][DEBUG ] "mimic", + node-admin: [node-admin][DEBUG ] "osdmap-prune", + node-admin: [node-admin][DEBUG ] "nautilus", + node-admin: [node-admin][DEBUG ] "octopus" + node-admin: [node-admin][DEBUG ] ] + node-admin: [node-admin][DEBUG ] }, + node-admin: [node-admin][DEBUG ] "monmap": { + node-admin: [node-admin][DEBUG ] "created": "2021-03-23T01:59:18.768135Z", + node-admin: [node-admin][DEBUG ] "epoch": 1, + node-admin: [node-admin][DEBUG ] "features": { + node-admin: [node-admin][DEBUG ] "optional": [], + node-admin: [node-admin][DEBUG ] "persistent": [ + node-admin: [node-admin][DEBUG ] "kraken", + node-admin: [node-admin][DEBUG ] "luminous", + node-admin: [node-admin][DEBUG ] "mimic", + node-admin: [node-admin][DEBUG ] "osdmap-prune", + node-admin: [node-admin][DEBUG ] "nautilus", + node-admin: [node-admin][DEBUG ] "octopus" + node-admin: [node-admin][DEBUG ] ] + node-admin: [node-admin][DEBUG ] }, + node-admin: [node-admin][DEBUG ] "fsid": "2adbfe38-dcf4-4606-8139-5948a116d429", + node-admin: [node-admin][DEBUG ] "min_mon_release": 15, + node-admin: [node-admin][DEBUG ] "min_mon_release_name": "octopus", + node-admin: [node-admin][DEBUG ] "modified": "2021-03-23T01:59:18.768135Z", + node-admin: [node-admin][DEBUG ] "mons": [ + node-admin: [node-admin][DEBUG ] { + node-admin: [node-admin][DEBUG ] "addr": "192.168.121.38:6789/0", + node-admin: [node-admin][DEBUG ] "name": "node-admin", + node-admin: [node-admin][DEBUG ] "priority": 0, + node-admin: [node-admin][DEBUG ] "public_addr": "192.168.121.38:6789/0", + node-admin: [node-admin][DEBUG ] "public_addrs": { + node-admin: [node-admin][DEBUG ] "addrvec": [ + node-admin: [node-admin][DEBUG ] { + node-admin: [node-admin][DEBUG ] "addr": "192.168.121.38:3300", + node-admin: [node-admin][DEBUG ] "nonce": 0, + node-admin: [node-admin][DEBUG ] "type": "v2" + node-admin: [node-admin][DEBUG ] }, + node-admin: [node-admin][DEBUG ] { + node-admin: [node-admin][DEBUG ] "addr": "192.168.121.38:6789", + node-admin: [node-admin][DEBUG ] "nonce": 0, + node-admin: [node-admin][DEBUG ] "type": "v1" + node-admin: [node-admin][DEBUG ] } + node-admin: [node-admin][DEBUG ] ] + node-admin: [node-admin][DEBUG ] }, + node-admin: [node-admin][DEBUG ] "rank": 0, + node-admin: [node-admin][DEBUG ] "weight": 0 + node-admin: [node-admin][DEBUG ] } + node-admin: [node-admin][DEBUG ] ] + node-admin: [node-admin][DEBUG ] }, + node-admin: [node-admin][DEBUG ] "name": "node-admin", + node-admin: [node-admin][DEBUG ] "outside_quorum": [], + node-admin: [node-admin][DEBUG ] "quorum": [ + node-admin: [node-admin][DEBUG ] 0 + node-admin: [node-admin][DEBUG ] ], + node-admin: [node-admin][DEBUG ] "quorum_age": 1, + node-admin: [node-admin][DEBUG ] "rank": 0, + node-admin: [node-admin][DEBUG ] "state": "leader", + node-admin: [node-admin][DEBUG ] "sync_provider": [] + node-admin: [node-admin][DEBUG ] } + node-admin: [node-admin][DEBUG ] ******************************************************************************** + node-admin: [node-admin][INFO ] monitor: mon.node-admin is running + node-admin: [node-admin][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-admin.asok mon_status + node-admin: [ceph_deploy.mon][INFO ] processing monitor mon.node-admin + node-admin: [node-admin][DEBUG ] connection detected need for sudo + node-admin: [node-admin][DEBUG ] connected to host: node-admin + node-admin: [node-admin][DEBUG ] detect platform information from remote host + node-admin: [node-admin][DEBUG ] detect machine type + node-admin: [node-admin][DEBUG ] find the location of an executable + node-admin: [node-admin][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-admin.asok mon_status + node-admin: [ceph_deploy.mon][INFO ] mon.node-admin monitor has reached quorum! + node-admin: [ceph_deploy.mon][INFO ] all initial monitors are running and have formed quorum + node-admin: [ceph_deploy.mon][INFO ] Running gatherkeys... + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing keys in temp directory /tmp/tmp4dUCY1 + node-admin: [node-admin][DEBUG ] connection detected need for sudo + node-admin: [node-admin][DEBUG ] connected to host: node-admin + node-admin: [node-admin][DEBUG ] detect platform information from remote host + node-admin: [node-admin][DEBUG ] detect machine type + node-admin: [node-admin][DEBUG ] get remote short hostname + node-admin: [node-admin][DEBUG ] fetch remote file + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --admin-daemon=/var/run/ceph/ceph-mon.node-admin.asok mon_status + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.admin + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-mds + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-mgr + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-osd + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-rgw + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.client.admin.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-mds.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-mgr.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] keyring 'ceph.mon.keyring' already exists + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-osd.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-rgw.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Destroy temp directory /tmp/tmp4dUCY1 node-admin: node-admin: {cephtest} {ceph-install} Deploy configuration file and client keys on all nodes node-admin: [ceph_deploy.conf][DEBUG ] found configuration file at: /home/ceph-admin/.cephdeploy.conf - node-admin: [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy admin node-admin node-osd1 node-osd2 + node-admin: [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy admin node-admin node-osd1 node-osd2 node-osd3 node-admin: [ceph_deploy.cli][INFO ] ceph-deploy options: node-admin: [ceph_deploy.cli][INFO ] username : None node-admin: [ceph_deploy.cli][INFO ] verbose : False node-admin: [ceph_deploy.cli][INFO ] overwrite_conf : False node-admin: [ceph_deploy.cli][INFO ] quiet : False - node-admin: [ceph_deploy.cli][INFO ] cd_conf : + node-admin: [ceph_deploy.cli][INFO ] cd_conf : node-admin: [ceph_deploy.cli][INFO ] cluster : ceph - node-admin: [ceph_deploy.cli][INFO ] client : ['node-admin', 'node-osd1', 'node-osd2'] - node-admin: [ceph_deploy.cli][INFO ] func : + node-admin: [ceph_deploy.cli][INFO ] client : ['node-admin', 'node-osd1', 'node-osd2', 'node-osd3'] + node-admin: [ceph_deploy.cli][INFO ] func : node-admin: [ceph_deploy.cli][INFO ] ceph_conf : None node-admin: [ceph_deploy.cli][INFO ] default_release : False - node-admin: [ceph_deploy][ERROR ] RuntimeError: ceph.client.admin.keyring not found + node-admin: [ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to node-admin + node-admin: [node-admin][DEBUG ] connection detected need for sudo + node-admin: [node-admin][DEBUG ] connected to host: node-admin + node-admin: [node-admin][DEBUG ] detect platform information from remote host + node-admin: [node-admin][DEBUG ] detect machine type + node-admin: [node-admin][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to node-osd1 + node-admin: [node-osd1][DEBUG ] connection detected need for sudo + node-admin: [node-osd1][DEBUG ] connected to host: node-osd1 + node-admin: [node-osd1][DEBUG ] detect platform information from remote host + node-admin: [node-osd1][DEBUG ] detect machine type + node-admin: [node-osd1][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to node-osd2 + node-admin: [node-osd2][DEBUG ] connection detected need for sudo + node-admin: [node-osd2][DEBUG ] connected to host: node-osd2 + node-admin: [node-osd2][DEBUG ] detect platform information from remote host + node-admin: [node-osd2][DEBUG ] detect machine type + node-admin: [node-osd2][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to node-osd3 + node-admin: [node-osd3][DEBUG ] connection detected need for sudo + node-admin: [node-osd3][DEBUG ] connected to host: node-osd3 + node-admin: [node-osd3][DEBUG ] detect platform information from remote host + node-admin: [node-osd3][DEBUG ] detect machine type + node-admin: [node-osd3][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf node-admin: node-admin: {cephtest} {ceph-install} Add monitor on node-osd1 node-admin: [ceph_deploy.conf][DEBUG ] found configuration file at: /home/ceph-admin/.cephdeploy.conf @@ -2264,14 +3081,154 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [ceph_deploy.cli][INFO ] overwrite_conf : False node-admin: [ceph_deploy.cli][INFO ] subcommand : add node-admin: [ceph_deploy.cli][INFO ] quiet : False - node-admin: [ceph_deploy.cli][INFO ] cd_conf : + node-admin: [ceph_deploy.cli][INFO ] cd_conf : node-admin: [ceph_deploy.cli][INFO ] cluster : ceph node-admin: [ceph_deploy.cli][INFO ] mon : ['node-osd1'] - node-admin: [ceph_deploy.cli][INFO ] func : + node-admin: [ceph_deploy.cli][INFO ] func : node-admin: [ceph_deploy.cli][INFO ] address : None node-admin: [ceph_deploy.cli][INFO ] ceph_conf : None node-admin: [ceph_deploy.cli][INFO ] default_release : False - node-admin: [ceph_deploy][ERROR ] RuntimeError: mon keyring not found; run 'new' to create a new cluster + node-admin: [ceph_deploy.mon][INFO ] ensuring configuration of new mon host: node-osd1 + node-admin: [ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to node-osd1 + node-admin: [node-osd1][DEBUG ] connection detected need for sudo + node-admin: [node-osd1][DEBUG ] connected to host: node-osd1 + node-admin: [node-osd1][DEBUG ] detect platform information from remote host + node-admin: [node-osd1][DEBUG ] detect machine type + node-admin: [node-osd1][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [ceph_deploy.mon][DEBUG ] Adding mon to cluster ceph, host node-osd1 + node-admin: [ceph_deploy.mon][DEBUG ] using mon address by resolving host: 192.168.121.124 + node-admin: [ceph_deploy.mon][DEBUG ] detecting platform for host node-osd1 ... + node-admin: [node-osd1][DEBUG ] connection detected need for sudo + node-admin: [node-osd1][DEBUG ] connected to host: node-osd1 + node-admin: [node-osd1][DEBUG ] detect platform information from remote host + node-admin: [node-osd1][DEBUG ] detect machine type + node-admin: [node-osd1][DEBUG ] find the location of an executable + node-admin: [ceph_deploy.mon][INFO ] distro info: Ubuntu 18.04 bionic + node-admin: [node-osd1][DEBUG ] determining if provided host has same hostname in remote + node-admin: [node-osd1][DEBUG ] get remote short hostname + node-admin: [node-osd1][DEBUG ] adding mon to node-osd1 + node-admin: [node-osd1][DEBUG ] get remote short hostname + node-admin: [node-osd1][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [node-osd1][DEBUG ] create the mon path if it does not exist + node-admin: [node-osd1][DEBUG ] checking for done path: /var/lib/ceph/mon/ceph-node-osd1/done + node-admin: [node-osd1][DEBUG ] done path does not exist: /var/lib/ceph/mon/ceph-node-osd1/done + node-admin: [node-osd1][INFO ] creating keyring file: /var/lib/ceph/tmp/ceph-node-osd1.mon.keyring + node-admin: [node-osd1][DEBUG ] create the monitor keyring file + node-admin: [node-osd1][INFO ] Running command: sudo ceph --cluster ceph mon getmap -o /var/lib/ceph/tmp/ceph.node-osd1.monmap + node-admin: [node-osd1][WARNING] got monmap epoch 1 + node-admin: [node-osd1][INFO ] Running command: sudo ceph-mon --cluster ceph --mkfs -i node-osd1 --monmap /var/lib/ceph/tmp/ceph.node-osd1.monmap --keyring /var/lib/ceph/tmp/ceph-node-osd1.mon.keyring --setuser 64045 --setgroup 64045 + node-admin: [node-osd1][INFO ] unlinking keyring file /var/lib/ceph/tmp/ceph-node-osd1.mon.keyring + node-admin: [node-osd1][DEBUG ] create a done file to avoid re-doing the mon deployment + node-admin: [node-osd1][DEBUG ] create the init path if it does not exist + node-admin: [node-osd1][INFO ] Running command: sudo systemctl enable ceph.target + node-admin: [node-osd1][INFO ] Running command: sudo systemctl enable ceph-mon@node-osd1 + node-admin: [node-osd1][WARNING] Created symlink /etc/systemd/system/ceph-mon.target.wants/ceph-mon@node-osd1.service → /lib/systemd/system/ceph-mon@.service. + node-admin: [node-osd1][INFO ] Running command: sudo systemctl start ceph-mon@node-osd1 + node-admin: [node-osd1][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-osd1.asok mon_status + node-admin: [node-osd1][WARNING] node-osd1 is not defined in `mon initial members` + node-admin: [node-osd1][WARNING] monitor node-osd1 does not exist in monmap + node-admin: [node-osd1][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-osd1.asok mon_status + node-admin: [node-osd1][DEBUG ] ******************************************************************************** + node-admin: [node-osd1][DEBUG ] status for monitor: mon.node-osd1 + node-admin: [node-osd1][DEBUG ] { + node-admin: [node-osd1][DEBUG ] "election_epoch": 1, + node-admin: [node-osd1][DEBUG ] "extra_probe_peers": [], + node-admin: [node-osd1][DEBUG ] "feature_map": { + node-admin: [node-osd1][DEBUG ] "mon": [ + node-admin: [node-osd1][DEBUG ] { + node-admin: [node-osd1][DEBUG ] "features": "0x3f01cfb8ffedffff", + node-admin: [node-osd1][DEBUG ] "num": 1, + node-admin: [node-osd1][DEBUG ] "release": "luminous" + node-admin: [node-osd1][DEBUG ] } + node-admin: [node-osd1][DEBUG ] ] + node-admin: [node-osd1][DEBUG ] }, + node-admin: [node-osd1][DEBUG ] "features": { + node-admin: [node-osd1][DEBUG ] "quorum_con": "0", + node-admin: [node-osd1][DEBUG ] "quorum_mon": [], + node-admin: [node-osd1][DEBUG ] "required_con": "2449958197560098820", + node-admin: [node-osd1][DEBUG ] "required_mon": [ + node-admin: [node-osd1][DEBUG ] "kraken", + node-admin: [node-osd1][DEBUG ] "luminous", + node-admin: [node-osd1][DEBUG ] "mimic", + node-admin: [node-osd1][DEBUG ] "osdmap-prune", + node-admin: [node-osd1][DEBUG ] "nautilus", + node-admin: [node-osd1][DEBUG ] "octopus" + node-admin: [node-osd1][DEBUG ] ] + node-admin: [node-osd1][DEBUG ] }, + node-admin: [node-osd1][DEBUG ] "monmap": { + node-admin: [node-osd1][DEBUG ] "created": "2021-03-23T01:59:18.768135Z", + node-admin: [node-osd1][DEBUG ] "epoch": 2, + node-admin: [node-osd1][DEBUG ] "features": { + node-admin: [node-osd1][DEBUG ] "optional": [], + node-admin: [node-osd1][DEBUG ] "persistent": [ + node-admin: [node-osd1][DEBUG ] "kraken", + node-admin: [node-osd1][DEBUG ] "luminous", + node-admin: [node-osd1][DEBUG ] "mimic", + node-admin: [node-osd1][DEBUG ] "osdmap-prune", + node-admin: [node-osd1][DEBUG ] "nautilus", + node-admin: [node-osd1][DEBUG ] "octopus" + node-admin: [node-osd1][DEBUG ] ] + node-admin: [node-osd1][DEBUG ] }, + node-admin: [node-osd1][DEBUG ] "fsid": "2adbfe38-dcf4-4606-8139-5948a116d429", + node-admin: [node-osd1][DEBUG ] "min_mon_release": 15, + node-admin: [node-osd1][DEBUG ] "min_mon_release_name": "octopus", + node-admin: [node-osd1][DEBUG ] "modified": "2021-03-23T01:59:37.240978Z", + node-admin: [node-osd1][DEBUG ] "mons": [ + node-admin: [node-osd1][DEBUG ] { + node-admin: [node-osd1][DEBUG ] "addr": "192.168.121.38:6789/0", + node-admin: [node-osd1][DEBUG ] "name": "node-admin", + node-admin: [node-osd1][DEBUG ] "priority": 0, + node-admin: [node-osd1][DEBUG ] "public_addr": "192.168.121.38:6789/0", + node-admin: [node-osd1][DEBUG ] "public_addrs": { + node-admin: [node-osd1][DEBUG ] "addrvec": [ + node-admin: [node-osd1][DEBUG ] { + node-admin: [node-osd1][DEBUG ] "addr": "192.168.121.38:3300", + node-admin: [node-osd1][DEBUG ] "nonce": 0, + node-admin: [node-osd1][DEBUG ] "type": "v2" + node-admin: [node-osd1][DEBUG ] }, + node-admin: [node-osd1][DEBUG ] { + node-admin: [node-osd1][DEBUG ] "addr": "192.168.121.38:6789", + node-admin: [node-osd1][DEBUG ] "nonce": 0, + node-admin: [node-osd1][DEBUG ] "type": "v1" + node-admin: [node-osd1][DEBUG ] } + node-admin: [node-osd1][DEBUG ] ] + node-admin: [node-osd1][DEBUG ] }, + node-admin: [node-osd1][DEBUG ] "rank": 0, + node-admin: [node-osd1][DEBUG ] "weight": 0 + node-admin: [node-osd1][DEBUG ] }, + node-admin: [node-osd1][DEBUG ] { + node-admin: [node-osd1][DEBUG ] "addr": "192.168.121.124:6789/0", + node-admin: [node-osd1][DEBUG ] "name": "node-osd1", + node-admin: [node-osd1][DEBUG ] "priority": 0, + node-admin: [node-osd1][DEBUG ] "public_addr": "192.168.121.124:6789/0", + node-admin: [node-osd1][DEBUG ] "public_addrs": { + node-admin: [node-osd1][DEBUG ] "addrvec": [ + node-admin: [node-osd1][DEBUG ] { + node-admin: [node-osd1][DEBUG ] "addr": "192.168.121.124:3300", + node-admin: [node-osd1][DEBUG ] "nonce": 0, + node-admin: [node-osd1][DEBUG ] "type": "v2" + node-admin: [node-osd1][DEBUG ] }, + node-admin: [node-osd1][DEBUG ] { + node-admin: [node-osd1][DEBUG ] "addr": "192.168.121.124:6789", + node-admin: [node-osd1][DEBUG ] "nonce": 0, + node-admin: [node-osd1][DEBUG ] "type": "v1" + node-admin: [node-osd1][DEBUG ] } + node-admin: [node-osd1][DEBUG ] ] + node-admin: [node-osd1][DEBUG ] }, + node-admin: [node-osd1][DEBUG ] "rank": 1, + node-admin: [node-osd1][DEBUG ] "weight": 0 + node-admin: [node-osd1][DEBUG ] } + node-admin: [node-osd1][DEBUG ] ] + node-admin: [node-osd1][DEBUG ] }, + node-admin: [node-osd1][DEBUG ] "name": "node-osd1", + node-admin: [node-osd1][DEBUG ] "outside_quorum": [], + node-admin: [node-osd1][DEBUG ] "quorum": [], + node-admin: [node-osd1][DEBUG ] "rank": 1, + node-admin: [node-osd1][DEBUG ] "state": "electing", + node-admin: [node-osd1][DEBUG ] "sync_provider": [] + node-admin: [node-osd1][DEBUG ] } + node-admin: [node-osd1][DEBUG ] ******************************************************************************** + node-admin: [node-osd1][INFO ] monitor: mon.node-osd1 is running node-admin: node-admin: {cephtest} {ceph-install} Add monitor on node-osd2 node-admin: [ceph_deploy.conf][DEBUG ] found configuration file at: /home/ceph-admin/.cephdeploy.conf @@ -2282,50 +3239,568 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [ceph_deploy.cli][INFO ] overwrite_conf : False node-admin: [ceph_deploy.cli][INFO ] subcommand : add node-admin: [ceph_deploy.cli][INFO ] quiet : False - node-admin: [ceph_deploy.cli][INFO ] cd_conf : + node-admin: [ceph_deploy.cli][INFO ] cd_conf : node-admin: [ceph_deploy.cli][INFO ] cluster : ceph node-admin: [ceph_deploy.cli][INFO ] mon : ['node-osd2'] - node-admin: [ceph_deploy.cli][INFO ] func : + node-admin: [ceph_deploy.cli][INFO ] func : node-admin: [ceph_deploy.cli][INFO ] address : None node-admin: [ceph_deploy.cli][INFO ] ceph_conf : None node-admin: [ceph_deploy.cli][INFO ] default_release : False - node-admin: [ceph_deploy][ERROR ] RuntimeError: mon keyring not found; run 'new' to create a new cluster + node-admin: [ceph_deploy.mon][INFO ] ensuring configuration of new mon host: node-osd2 + node-admin: [ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to node-osd2 + node-admin: [node-osd2][DEBUG ] connection detected need for sudo + node-admin: [node-osd2][DEBUG ] connected to host: node-osd2 + node-admin: [node-osd2][DEBUG ] detect platform information from remote host + node-admin: [node-osd2][DEBUG ] detect machine type + node-admin: [node-osd2][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [ceph_deploy.mon][DEBUG ] Adding mon to cluster ceph, host node-osd2 + node-admin: [ceph_deploy.mon][DEBUG ] using mon address by resolving host: 192.168.121.252 + node-admin: [ceph_deploy.mon][DEBUG ] detecting platform for host node-osd2 ... + node-admin: [node-osd2][DEBUG ] connection detected need for sudo + node-admin: [node-osd2][DEBUG ] connected to host: node-osd2 + node-admin: [node-osd2][DEBUG ] detect platform information from remote host + node-admin: [node-osd2][DEBUG ] detect machine type + node-admin: [node-osd2][DEBUG ] find the location of an executable + node-admin: [ceph_deploy.mon][INFO ] distro info: Ubuntu 18.04 bionic + node-admin: [node-osd2][DEBUG ] determining if provided host has same hostname in remote + node-admin: [node-osd2][DEBUG ] get remote short hostname + node-admin: [node-osd2][DEBUG ] adding mon to node-osd2 + node-admin: [node-osd2][DEBUG ] get remote short hostname + node-admin: [node-osd2][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [node-osd2][DEBUG ] create the mon path if it does not exist + node-admin: [node-osd2][DEBUG ] checking for done path: /var/lib/ceph/mon/ceph-node-osd2/done + node-admin: [node-osd2][DEBUG ] done path does not exist: /var/lib/ceph/mon/ceph-node-osd2/done + node-admin: [node-osd2][INFO ] creating keyring file: /var/lib/ceph/tmp/ceph-node-osd2.mon.keyring + node-admin: [node-osd2][DEBUG ] create the monitor keyring file + node-admin: [node-osd2][INFO ] Running command: sudo ceph --cluster ceph mon getmap -o /var/lib/ceph/tmp/ceph.node-osd2.monmap + node-admin: [node-osd2][WARNING] got monmap epoch 2 + node-admin: [node-osd2][INFO ] Running command: sudo ceph-mon --cluster ceph --mkfs -i node-osd2 --monmap /var/lib/ceph/tmp/ceph.node-osd2.monmap --keyring /var/lib/ceph/tmp/ceph-node-osd2.mon.keyring --setuser 64045 --setgroup 64045 + node-admin: [node-osd2][INFO ] unlinking keyring file /var/lib/ceph/tmp/ceph-node-osd2.mon.keyring + node-admin: [node-osd2][DEBUG ] create a done file to avoid re-doing the mon deployment + node-admin: [node-osd2][DEBUG ] create the init path if it does not exist + node-admin: [node-osd2][INFO ] Running command: sudo systemctl enable ceph.target + node-admin: [node-osd2][INFO ] Running command: sudo systemctl enable ceph-mon@node-osd2 + node-admin: [node-osd2][WARNING] Created symlink /etc/systemd/system/ceph-mon.target.wants/ceph-mon@node-osd2.service → /lib/systemd/system/ceph-mon@.service. + node-admin: [node-osd2][INFO ] Running command: sudo systemctl start ceph-mon@node-osd2 + node-admin: [node-osd2][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-osd2.asok mon_status + node-admin: [node-osd2][WARNING] node-osd2 is not defined in `mon initial members` + node-admin: [node-osd2][WARNING] monitor node-osd2 does not exist in monmap + node-admin: [node-osd2][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-osd2.asok mon_status + node-admin: [node-osd2][DEBUG ] ******************************************************************************** + node-admin: [node-osd2][DEBUG ] status for monitor: mon.node-osd2 + node-admin: [node-osd2][DEBUG ] { + node-admin: [node-osd2][DEBUG ] "election_epoch": 1, + node-admin: [node-osd2][DEBUG ] "extra_probe_peers": [ + node-admin: [node-osd2][DEBUG ] { + node-admin: [node-osd2][DEBUG ] "addrvec": [ + node-admin: [node-osd2][DEBUG ] { + node-admin: [node-osd2][DEBUG ] "addr": "192.168.121.124:3300", + node-admin: [node-osd2][DEBUG ] "nonce": 0, + node-admin: [node-osd2][DEBUG ] "type": "v2" + node-admin: [node-osd2][DEBUG ] }, + node-admin: [node-osd2][DEBUG ] { + node-admin: [node-osd2][DEBUG ] "addr": "192.168.121.124:6789", + node-admin: [node-osd2][DEBUG ] "nonce": 0, + node-admin: [node-osd2][DEBUG ] "type": "v1" + node-admin: [node-osd2][DEBUG ] } + node-admin: [node-osd2][DEBUG ] ] + node-admin: [node-osd2][DEBUG ] } + node-admin: [node-osd2][DEBUG ] ], + node-admin: [node-osd2][DEBUG ] "feature_map": { + node-admin: [node-osd2][DEBUG ] "mon": [ + node-admin: [node-osd2][DEBUG ] { + node-admin: [node-osd2][DEBUG ] "features": "0x3f01cfb8ffedffff", + node-admin: [node-osd2][DEBUG ] "num": 1, + node-admin: [node-osd2][DEBUG ] "release": "luminous" + node-admin: [node-osd2][DEBUG ] } + node-admin: [node-osd2][DEBUG ] ] + node-admin: [node-osd2][DEBUG ] }, + node-admin: [node-osd2][DEBUG ] "features": { + node-admin: [node-osd2][DEBUG ] "quorum_con": "0", + node-admin: [node-osd2][DEBUG ] "quorum_mon": [], + node-admin: [node-osd2][DEBUG ] "required_con": "2449958197560098820", + node-admin: [node-osd2][DEBUG ] "required_mon": [ + node-admin: [node-osd2][DEBUG ] "kraken", + node-admin: [node-osd2][DEBUG ] "luminous", + node-admin: [node-osd2][DEBUG ] "mimic", + node-admin: [node-osd2][DEBUG ] "osdmap-prune", + node-admin: [node-osd2][DEBUG ] "nautilus", + node-admin: [node-osd2][DEBUG ] "octopus" + node-admin: [node-osd2][DEBUG ] ] + node-admin: [node-osd2][DEBUG ] }, + node-admin: [node-osd2][DEBUG ] "monmap": { + node-admin: [node-osd2][DEBUG ] "created": "2021-03-23T01:59:18.768135Z", + node-admin: [node-osd2][DEBUG ] "epoch": 3, + node-admin: [node-osd2][DEBUG ] "features": { + node-admin: [node-osd2][DEBUG ] "optional": [], + node-admin: [node-osd2][DEBUG ] "persistent": [ + node-admin: [node-osd2][DEBUG ] "kraken", + node-admin: [node-osd2][DEBUG ] "luminous", + node-admin: [node-osd2][DEBUG ] "mimic", + node-admin: [node-osd2][DEBUG ] "osdmap-prune", + node-admin: [node-osd2][DEBUG ] "nautilus", + node-admin: [node-osd2][DEBUG ] "octopus" + node-admin: [node-osd2][DEBUG ] ] + node-admin: [node-osd2][DEBUG ] }, + node-admin: [node-osd2][DEBUG ] "fsid": "2adbfe38-dcf4-4606-8139-5948a116d429", + node-admin: [node-osd2][DEBUG ] "min_mon_release": 15, + node-admin: [node-osd2][DEBUG ] "min_mon_release_name": "octopus", + node-admin: [node-osd2][DEBUG ] "modified": "2021-03-23T01:59:45.154811Z", + node-admin: [node-osd2][DEBUG ] "mons": [ + node-admin: [node-osd2][DEBUG ] { + node-admin: [node-osd2][DEBUG ] "addr": "192.168.121.38:6789/0", + node-admin: [node-osd2][DEBUG ] "name": "node-admin", + node-admin: [node-osd2][DEBUG ] "priority": 0, + node-admin: [node-osd2][DEBUG ] "public_addr": "192.168.121.38:6789/0", + node-admin: [node-osd2][DEBUG ] "public_addrs": { + node-admin: [node-osd2][DEBUG ] "addrvec": [ + node-admin: [node-osd2][DEBUG ] { + node-admin: [node-osd2][DEBUG ] "addr": "192.168.121.38:3300", + node-admin: [node-osd2][DEBUG ] "nonce": 0, + node-admin: [node-osd2][DEBUG ] "type": "v2" + node-admin: [node-osd2][DEBUG ] }, + node-admin: [node-osd2][DEBUG ] { + node-admin: [node-osd2][DEBUG ] "addr": "192.168.121.38:6789", + node-admin: [node-osd2][DEBUG ] "nonce": 0, + node-admin: [node-osd2][DEBUG ] "type": "v1" + node-admin: [node-osd2][DEBUG ] } + node-admin: [node-osd2][DEBUG ] ] + node-admin: [node-osd2][DEBUG ] }, + node-admin: [node-osd2][DEBUG ] "rank": 0, + node-admin: [node-osd2][DEBUG ] "weight": 0 + node-admin: [node-osd2][DEBUG ] }, + node-admin: [node-osd2][DEBUG ] { + node-admin: [node-osd2][DEBUG ] "addr": "192.168.121.124:6789/0", + node-admin: [node-osd2][DEBUG ] "name": "node-osd1", + node-admin: [node-osd2][DEBUG ] "priority": 0, + node-admin: [node-osd2][DEBUG ] "public_addr": "192.168.121.124:6789/0", + node-admin: [node-osd2][DEBUG ] "public_addrs": { + node-admin: [node-osd2][DEBUG ] "addrvec": [ + node-admin: [node-osd2][DEBUG ] { + node-admin: [node-osd2][DEBUG ] "addr": "192.168.121.124:3300", + node-admin: [node-osd2][DEBUG ] "nonce": 0, + node-admin: [node-osd2][DEBUG ] "type": "v2" + node-admin: [node-osd2][DEBUG ] }, + node-admin: [node-osd2][DEBUG ] { + node-admin: [node-osd2][DEBUG ] "addr": "192.168.121.124:6789", + node-admin: [node-osd2][DEBUG ] "nonce": 0, + node-admin: [node-osd2][DEBUG ] "type": "v1" + node-admin: [node-osd2][DEBUG ] } + node-admin: [node-osd2][DEBUG ] ] + node-admin: [node-osd2][DEBUG ] }, + node-admin: [node-osd2][DEBUG ] "rank": 1, + node-admin: [node-osd2][DEBUG ] "weight": 0 + node-admin: [node-osd2][DEBUG ] }, + node-admin: [node-osd2][DEBUG ] { + node-admin: [node-osd2][DEBUG ] "addr": "192.168.121.252:6789/0", + node-admin: [node-osd2][DEBUG ] "name": "node-osd2", + node-admin: [node-osd2][DEBUG ] "priority": 0, + node-admin: [node-osd2][DEBUG ] "public_addr": "192.168.121.252:6789/0", + node-admin: [node-osd2][DEBUG ] "public_addrs": { + node-admin: [node-osd2][DEBUG ] "addrvec": [ + node-admin: [node-osd2][DEBUG ] { + node-admin: [node-osd2][DEBUG ] "addr": "192.168.121.252:3300", + node-admin: [node-osd2][DEBUG ] "nonce": 0, + node-admin: [node-osd2][DEBUG ] "type": "v2" + node-admin: [node-osd2][DEBUG ] }, + node-admin: [node-osd2][DEBUG ] { + node-admin: [node-osd2][DEBUG ] "addr": "192.168.121.252:6789", + node-admin: [node-osd2][DEBUG ] "nonce": 0, + node-admin: [node-osd2][DEBUG ] "type": "v1" + node-admin: [node-osd2][DEBUG ] } + node-admin: [node-osd2][DEBUG ] ] + node-admin: [node-osd2][DEBUG ] }, + node-admin: [node-osd2][DEBUG ] "rank": 2, + node-admin: [node-osd2][DEBUG ] "weight": 0 + node-admin: [node-osd2][DEBUG ] } + node-admin: [node-osd2][DEBUG ] ] + node-admin: [node-osd2][DEBUG ] }, + node-admin: [node-osd2][DEBUG ] "name": "node-osd2", + node-admin: [node-osd2][DEBUG ] "outside_quorum": [], + node-admin: [node-osd2][DEBUG ] "quorum": [], + node-admin: [node-osd2][DEBUG ] "rank": 2, + node-admin: [node-osd2][DEBUG ] "state": "electing", + node-admin: [node-osd2][DEBUG ] "sync_provider": [] + node-admin: [node-osd2][DEBUG ] } + node-admin: [node-osd2][DEBUG ] ******************************************************************************** + node-admin: [node-osd2][INFO ] monitor: mon.node-osd2 is running + node-admin: + node-admin: {cephtest} {ceph-install} Add monitor on node-osd3 + node-admin: [ceph_deploy.conf][DEBUG ] found configuration file at: /home/ceph-admin/.cephdeploy.conf + node-admin: [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy mon add node-osd3 + node-admin: [ceph_deploy.cli][INFO ] ceph-deploy options: + node-admin: [ceph_deploy.cli][INFO ] username : None + node-admin: [ceph_deploy.cli][INFO ] verbose : False + node-admin: [ceph_deploy.cli][INFO ] overwrite_conf : False + node-admin: [ceph_deploy.cli][INFO ] subcommand : add + node-admin: [ceph_deploy.cli][INFO ] quiet : False + node-admin: [ceph_deploy.cli][INFO ] cd_conf : + node-admin: [ceph_deploy.cli][INFO ] cluster : ceph + node-admin: [ceph_deploy.cli][INFO ] mon : ['node-osd3'] + node-admin: [ceph_deploy.cli][INFO ] func : + node-admin: [ceph_deploy.cli][INFO ] address : None + node-admin: [ceph_deploy.cli][INFO ] ceph_conf : None + node-admin: [ceph_deploy.cli][INFO ] default_release : False + node-admin: [ceph_deploy.mon][INFO ] ensuring configuration of new mon host: node-osd3 + node-admin: [ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to node-osd3 + node-admin: [node-osd3][DEBUG ] connection detected need for sudo + node-admin: [node-osd3][DEBUG ] connected to host: node-osd3 + node-admin: [node-osd3][DEBUG ] detect platform information from remote host + node-admin: [node-osd3][DEBUG ] detect machine type + node-admin: [node-osd3][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [ceph_deploy.mon][DEBUG ] Adding mon to cluster ceph, host node-osd3 + node-admin: [ceph_deploy.mon][DEBUG ] using mon address by resolving host: 192.168.121.236 + node-admin: [ceph_deploy.mon][DEBUG ] detecting platform for host node-osd3 ... + node-admin: [node-osd3][DEBUG ] connection detected need for sudo + node-admin: [node-osd3][DEBUG ] connected to host: node-osd3 + node-admin: [node-osd3][DEBUG ] detect platform information from remote host + node-admin: [node-osd3][DEBUG ] detect machine type + node-admin: [node-osd3][DEBUG ] find the location of an executable + node-admin: [ceph_deploy.mon][INFO ] distro info: Ubuntu 18.04 bionic + node-admin: [node-osd3][DEBUG ] determining if provided host has same hostname in remote + node-admin: [node-osd3][DEBUG ] get remote short hostname + node-admin: [node-osd3][DEBUG ] adding mon to node-osd3 + node-admin: [node-osd3][DEBUG ] get remote short hostname + node-admin: [node-osd3][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [node-osd3][DEBUG ] create the mon path if it does not exist + node-admin: [node-osd3][DEBUG ] checking for done path: /var/lib/ceph/mon/ceph-node-osd3/done + node-admin: [node-osd3][DEBUG ] done path does not exist: /var/lib/ceph/mon/ceph-node-osd3/done + node-admin: [node-osd3][INFO ] creating keyring file: /var/lib/ceph/tmp/ceph-node-osd3.mon.keyring + node-admin: [node-osd3][DEBUG ] create the monitor keyring file + node-admin: [node-osd3][INFO ] Running command: sudo ceph --cluster ceph mon getmap -o /var/lib/ceph/tmp/ceph.node-osd3.monmap + node-admin: [node-osd3][WARNING] got monmap epoch 3 + node-admin: [node-osd3][INFO ] Running command: sudo ceph-mon --cluster ceph --mkfs -i node-osd3 --monmap /var/lib/ceph/tmp/ceph.node-osd3.monmap --keyring /var/lib/ceph/tmp/ceph-node-osd3.mon.keyring --setuser 64045 --setgroup 64045 + node-admin: [node-osd3][INFO ] unlinking keyring file /var/lib/ceph/tmp/ceph-node-osd3.mon.keyring + node-admin: [node-osd3][DEBUG ] create a done file to avoid re-doing the mon deployment + node-admin: [node-osd3][DEBUG ] create the init path if it does not exist + node-admin: [node-osd3][INFO ] Running command: sudo systemctl enable ceph.target + node-admin: [node-osd3][INFO ] Running command: sudo systemctl enable ceph-mon@node-osd3 + node-admin: [node-osd3][WARNING] Created symlink /etc/systemd/system/ceph-mon.target.wants/ceph-mon@node-osd3.service → /lib/systemd/system/ceph-mon@.service. + node-admin: [node-osd3][INFO ] Running command: sudo systemctl start ceph-mon@node-osd3 + node-admin: [node-osd3][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-osd3.asok mon_status + node-admin: [node-osd3][WARNING] node-osd3 is not defined in `mon initial members` + node-admin: [node-osd3][WARNING] monitor node-osd3 does not exist in monmap + node-admin: [node-osd3][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-osd3.asok mon_status + node-admin: [node-osd3][DEBUG ] ******************************************************************************** + node-admin: [node-osd3][DEBUG ] status for monitor: mon.node-osd3 + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "election_epoch": 1, + node-admin: [node-osd3][DEBUG ] "extra_probe_peers": [ + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addrvec": [ + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.124:3300", + node-admin: [node-osd3][DEBUG ] "nonce": 0, + node-admin: [node-osd3][DEBUG ] "type": "v2" + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.124:6789", + node-admin: [node-osd3][DEBUG ] "nonce": 0, + node-admin: [node-osd3][DEBUG ] "type": "v1" + node-admin: [node-osd3][DEBUG ] } + node-admin: [node-osd3][DEBUG ] ] + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addrvec": [ + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.252:3300", + node-admin: [node-osd3][DEBUG ] "nonce": 0, + node-admin: [node-osd3][DEBUG ] "type": "v2" + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.252:6789", + node-admin: [node-osd3][DEBUG ] "nonce": 0, + node-admin: [node-osd3][DEBUG ] "type": "v1" + node-admin: [node-osd3][DEBUG ] } + node-admin: [node-osd3][DEBUG ] ] + node-admin: [node-osd3][DEBUG ] } + node-admin: [node-osd3][DEBUG ] ], + node-admin: [node-osd3][DEBUG ] "feature_map": { + node-admin: [node-osd3][DEBUG ] "mon": [ + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "features": "0x3f01cfb8ffedffff", + node-admin: [node-osd3][DEBUG ] "num": 1, + node-admin: [node-osd3][DEBUG ] "release": "luminous" + node-admin: [node-osd3][DEBUG ] } + node-admin: [node-osd3][DEBUG ] ] + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] "features": { + node-admin: [node-osd3][DEBUG ] "quorum_con": "0", + node-admin: [node-osd3][DEBUG ] "quorum_mon": [], + node-admin: [node-osd3][DEBUG ] "required_con": "2449958197560098820", + node-admin: [node-osd3][DEBUG ] "required_mon": [ + node-admin: [node-osd3][DEBUG ] "kraken", + node-admin: [node-osd3][DEBUG ] "luminous", + node-admin: [node-osd3][DEBUG ] "mimic", + node-admin: [node-osd3][DEBUG ] "osdmap-prune", + node-admin: [node-osd3][DEBUG ] "nautilus", + node-admin: [node-osd3][DEBUG ] "octopus" + node-admin: [node-osd3][DEBUG ] ] + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] "monmap": { + node-admin: [node-osd3][DEBUG ] "created": "2021-03-23T01:59:18.768135Z", + node-admin: [node-osd3][DEBUG ] "epoch": 4, + node-admin: [node-osd3][DEBUG ] "features": { + node-admin: [node-osd3][DEBUG ] "optional": [], + node-admin: [node-osd3][DEBUG ] "persistent": [ + node-admin: [node-osd3][DEBUG ] "kraken", + node-admin: [node-osd3][DEBUG ] "luminous", + node-admin: [node-osd3][DEBUG ] "mimic", + node-admin: [node-osd3][DEBUG ] "osdmap-prune", + node-admin: [node-osd3][DEBUG ] "nautilus", + node-admin: [node-osd3][DEBUG ] "octopus" + node-admin: [node-osd3][DEBUG ] ] + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] "fsid": "2adbfe38-dcf4-4606-8139-5948a116d429", + node-admin: [node-osd3][DEBUG ] "min_mon_release": 15, + node-admin: [node-osd3][DEBUG ] "min_mon_release_name": "octopus", + node-admin: [node-osd3][DEBUG ] "modified": "2021-03-23T01:59:53.113840Z", + node-admin: [node-osd3][DEBUG ] "mons": [ + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.38:6789/0", + node-admin: [node-osd3][DEBUG ] "name": "node-admin", + node-admin: [node-osd3][DEBUG ] "priority": 0, + node-admin: [node-osd3][DEBUG ] "public_addr": "192.168.121.38:6789/0", + node-admin: [node-osd3][DEBUG ] "public_addrs": { + node-admin: [node-osd3][DEBUG ] "addrvec": [ + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.38:3300", + node-admin: [node-osd3][DEBUG ] "nonce": 0, + node-admin: [node-osd3][DEBUG ] "type": "v2" + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.38:6789", + node-admin: [node-osd3][DEBUG ] "nonce": 0, + node-admin: [node-osd3][DEBUG ] "type": "v1" + node-admin: [node-osd3][DEBUG ] } + node-admin: [node-osd3][DEBUG ] ] + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] "rank": 0, + node-admin: [node-osd3][DEBUG ] "weight": 0 + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.124:6789/0", + node-admin: [node-osd3][DEBUG ] "name": "node-osd1", + node-admin: [node-osd3][DEBUG ] "priority": 0, + node-admin: [node-osd3][DEBUG ] "public_addr": "192.168.121.124:6789/0", + node-admin: [node-osd3][DEBUG ] "public_addrs": { + node-admin: [node-osd3][DEBUG ] "addrvec": [ + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.124:3300", + node-admin: [node-osd3][DEBUG ] "nonce": 0, + node-admin: [node-osd3][DEBUG ] "type": "v2" + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.124:6789", + node-admin: [node-osd3][DEBUG ] "nonce": 0, + node-admin: [node-osd3][DEBUG ] "type": "v1" + node-admin: [node-osd3][DEBUG ] } + node-admin: [node-osd3][DEBUG ] ] + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] "rank": 1, + node-admin: [node-osd3][DEBUG ] "weight": 0 + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.252:6789/0", + node-admin: [node-osd3][DEBUG ] "name": "node-osd2", + node-admin: [node-osd3][DEBUG ] "priority": 0, + node-admin: [node-osd3][DEBUG ] "public_addr": "192.168.121.252:6789/0", + node-admin: [node-osd3][DEBUG ] "public_addrs": { + node-admin: [node-osd3][DEBUG ] "addrvec": [ + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.252:3300", + node-admin: [node-osd3][DEBUG ] "nonce": 0, + node-admin: [node-osd3][DEBUG ] "type": "v2" + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.252:6789", + node-admin: [node-osd3][DEBUG ] "nonce": 0, + node-admin: [node-osd3][DEBUG ] "type": "v1" + node-admin: [node-osd3][DEBUG ] } + node-admin: [node-osd3][DEBUG ] ] + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] "rank": 2, + node-admin: [node-osd3][DEBUG ] "weight": 0 + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.236:6789/0", + node-admin: [node-osd3][DEBUG ] "name": "node-osd3", + node-admin: [node-osd3][DEBUG ] "priority": 0, + node-admin: [node-osd3][DEBUG ] "public_addr": "192.168.121.236:6789/0", + node-admin: [node-osd3][DEBUG ] "public_addrs": { + node-admin: [node-osd3][DEBUG ] "addrvec": [ + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.236:3300", + node-admin: [node-osd3][DEBUG ] "nonce": 0, + node-admin: [node-osd3][DEBUG ] "type": "v2" + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] { + node-admin: [node-osd3][DEBUG ] "addr": "192.168.121.236:6789", + node-admin: [node-osd3][DEBUG ] "nonce": 0, + node-admin: [node-osd3][DEBUG ] "type": "v1" + node-admin: [node-osd3][DEBUG ] } + node-admin: [node-osd3][DEBUG ] ] + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] "rank": 3, + node-admin: [node-osd3][DEBUG ] "weight": 0 + node-admin: [node-osd3][DEBUG ] } + node-admin: [node-osd3][DEBUG ] ] + node-admin: [node-osd3][DEBUG ] }, + node-admin: [node-osd3][DEBUG ] "name": "node-osd3", + node-admin: [node-osd3][DEBUG ] "outside_quorum": [], + node-admin: [node-osd3][DEBUG ] "quorum": [], + node-admin: [node-osd3][DEBUG ] "rank": 3, + node-admin: [node-osd3][DEBUG ] "state": "electing", + node-admin: [node-osd3][DEBUG ] "sync_provider": [] + node-admin: [node-osd3][DEBUG ] } + node-admin: [node-osd3][DEBUG ] ******************************************************************************** + node-admin: [node-osd3][INFO ] monitor: mon.node-osd3 is running node-admin: node-admin: {cephtest} {ceph-install} Create manager on all nodes node-admin: [ceph_deploy.conf][DEBUG ] found configuration file at: /home/ceph-admin/.cephdeploy.conf - node-admin: [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy mgr create node-admin node-osd1 node-osd2 + node-admin: [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy mgr create node-admin node-osd1 node-osd2 node-osd3 node-admin: [ceph_deploy.cli][INFO ] ceph-deploy options: node-admin: [ceph_deploy.cli][INFO ] username : None node-admin: [ceph_deploy.cli][INFO ] verbose : False - node-admin: [ceph_deploy.cli][INFO ] mgr : [('node-admin', 'node-admin'), ('node-osd1', 'node-osd1'), ('node-osd2', 'node-osd2')] + node-admin: [ceph_deploy.cli][INFO ] mgr : [('node-admin', 'node-admin'), ('node-osd1', 'node-osd1'), ('node-osd2', 'node-osd2'), ('node-osd3', 'node-osd3')] node-admin: [ceph_deploy.cli][INFO ] overwrite_conf : False node-admin: [ceph_deploy.cli][INFO ] subcommand : create node-admin: [ceph_deploy.cli][INFO ] quiet : False - node-admin: [ceph_deploy.cli][INFO ] cd_conf : + node-admin: [ceph_deploy.cli][INFO ] cd_conf : node-admin: [ceph_deploy.cli][INFO ] cluster : ceph - node-admin: [ceph_deploy.cli][INFO ] func : + node-admin: [ceph_deploy.cli][INFO ] func : node-admin: [ceph_deploy.cli][INFO ] ceph_conf : None node-admin: [ceph_deploy.cli][INFO ] default_release : False - node-admin: [ceph_deploy.mgr][DEBUG ] Deploying mgr, cluster ceph hosts node-admin:node-admin node-osd1:node-osd1 node-osd2:node-osd2 - node-admin: [ceph_deploy][ERROR ] RuntimeError: bootstrap-mgr keyring not found; run 'gatherkeys' + node-admin: [ceph_deploy.mgr][DEBUG ] Deploying mgr, cluster ceph hosts node-admin:node-admin node-osd1:node-osd1 node-osd2:node-osd2 node-osd3:node-osd3 + node-admin: [node-admin][DEBUG ] connection detected need for sudo + node-admin: [node-admin][DEBUG ] connected to host: node-admin + node-admin: [node-admin][DEBUG ] detect platform information from remote host + node-admin: [node-admin][DEBUG ] detect machine type + node-admin: [ceph_deploy.mgr][INFO ] Distro info: Ubuntu 18.04 bionic + node-admin: [ceph_deploy.mgr][DEBUG ] remote host will use systemd + node-admin: [ceph_deploy.mgr][DEBUG ] deploying mgr bootstrap to node-admin + node-admin: [node-admin][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [node-admin][WARNING] mgr keyring does not exist yet, creating one + node-admin: [node-admin][DEBUG ] create a keyring file + node-admin: [node-admin][DEBUG ] create path recursively if it doesn't exist + node-admin: [node-admin][INFO ] Running command: sudo ceph --cluster ceph --name client.bootstrap-mgr --keyring /var/lib/ceph/bootstrap-mgr/ceph.keyring auth get-or-create mgr.node-admin mon allow profile mgr osd allow * mds allow * -o /var/lib/ceph/mgr/ceph-node-admin/keyring + node-admin: [node-admin][INFO ] Running command: sudo systemctl enable ceph-mgr@node-admin + node-admin: [node-admin][WARNING] Created symlink /etc/systemd/system/ceph-mgr.target.wants/ceph-mgr@node-admin.service → /lib/systemd/system/ceph-mgr@.service. + node-admin: [node-admin][INFO ] Running command: sudo systemctl start ceph-mgr@node-admin + node-admin: [node-admin][INFO ] Running command: sudo systemctl enable ceph.target + node-admin: [node-osd1][DEBUG ] connection detected need for sudo + node-admin: [node-osd1][DEBUG ] connected to host: node-osd1 + node-admin: [node-osd1][DEBUG ] detect platform information from remote host + node-admin: [node-osd1][DEBUG ] detect machine type + node-admin: [ceph_deploy.mgr][INFO ] Distro info: Ubuntu 18.04 bionic + node-admin: [ceph_deploy.mgr][DEBUG ] remote host will use systemd + node-admin: [ceph_deploy.mgr][DEBUG ] deploying mgr bootstrap to node-osd1 + node-admin: [node-osd1][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [node-osd1][WARNING] mgr keyring does not exist yet, creating one + node-admin: [node-osd1][DEBUG ] create a keyring file + node-admin: [node-osd1][DEBUG ] create path recursively if it doesn't exist + node-admin: [node-osd1][INFO ] Running command: sudo ceph --cluster ceph --name client.bootstrap-mgr --keyring /var/lib/ceph/bootstrap-mgr/ceph.keyring auth get-or-create mgr.node-osd1 mon allow profile mgr osd allow * mds allow * -o /var/lib/ceph/mgr/ceph-node-osd1/keyring + node-admin: [node-osd1][INFO ] Running command: sudo systemctl enable ceph-mgr@node-osd1 + node-admin: [node-osd1][WARNING] Created symlink /etc/systemd/system/ceph-mgr.target.wants/ceph-mgr@node-osd1.service → /lib/systemd/system/ceph-mgr@.service. + node-admin: [node-osd1][INFO ] Running command: sudo systemctl start ceph-mgr@node-osd1 + node-admin: [node-osd1][INFO ] Running command: sudo systemctl enable ceph.target + node-admin: [node-osd2][DEBUG ] connection detected need for sudo + node-admin: [node-osd2][DEBUG ] connected to host: node-osd2 + node-admin: [node-osd2][DEBUG ] detect platform information from remote host + node-admin: [node-osd2][DEBUG ] detect machine type + node-admin: [ceph_deploy.mgr][INFO ] Distro info: Ubuntu 18.04 bionic + node-admin: [ceph_deploy.mgr][DEBUG ] remote host will use systemd + node-admin: [ceph_deploy.mgr][DEBUG ] deploying mgr bootstrap to node-osd2 + node-admin: [node-osd2][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [node-osd2][WARNING] mgr keyring does not exist yet, creating one + node-admin: [node-osd2][DEBUG ] create a keyring file + node-admin: [node-osd2][DEBUG ] create path recursively if it doesn't exist + node-admin: [node-osd2][INFO ] Running command: sudo ceph --cluster ceph --name client.bootstrap-mgr --keyring /var/lib/ceph/bootstrap-mgr/ceph.keyring auth get-or-create mgr.node-osd2 mon allow profile mgr osd allow * mds allow * -o /var/lib/ceph/mgr/ceph-node-osd2/keyring + node-admin: [node-osd2][INFO ] Running command: sudo systemctl enable ceph-mgr@node-osd2 + node-admin: [node-osd2][WARNING] Created symlink /etc/systemd/system/ceph-mgr.target.wants/ceph-mgr@node-osd2.service → /lib/systemd/system/ceph-mgr@.service. + node-admin: [node-osd2][INFO ] Running command: sudo systemctl start ceph-mgr@node-osd2 + node-admin: [node-osd2][INFO ] Running command: sudo systemctl enable ceph.target + node-admin: [node-osd3][DEBUG ] connection detected need for sudo + node-admin: [node-osd3][DEBUG ] connected to host: node-osd3 + node-admin: [node-osd3][DEBUG ] detect platform information from remote host + node-admin: [node-osd3][DEBUG ] detect machine type + node-admin: [ceph_deploy.mgr][INFO ] Distro info: Ubuntu 18.04 bionic + node-admin: [ceph_deploy.mgr][DEBUG ] remote host will use systemd + node-admin: [ceph_deploy.mgr][DEBUG ] deploying mgr bootstrap to node-osd3 + node-admin: [node-osd3][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [node-osd3][WARNING] mgr keyring does not exist yet, creating one + node-admin: [node-osd3][DEBUG ] create a keyring file + node-admin: [node-osd3][DEBUG ] create path recursively if it doesn't exist + node-admin: [node-osd3][INFO ] Running command: sudo ceph --cluster ceph --name client.bootstrap-mgr --keyring /var/lib/ceph/bootstrap-mgr/ceph.keyring auth get-or-create mgr.node-osd3 mon allow profile mgr osd allow * mds allow * -o /var/lib/ceph/mgr/ceph-node-osd3/keyring + node-admin: [node-osd3][INFO ] Running command: sudo systemctl enable ceph-mgr@node-osd3 + node-admin: [node-osd3][WARNING] Created symlink /etc/systemd/system/ceph-mgr.target.wants/ceph-mgr@node-osd3.service → /lib/systemd/system/ceph-mgr@.service. + node-admin: [node-osd3][INFO ] Running command: sudo systemctl start ceph-mgr@node-osd3 + node-admin: [node-osd3][INFO ] Running command: sudo systemctl enable ceph.target node-admin: node-admin: {cephtest} {ceph-install} Create metadata server on osd nodes node-admin: [ceph_deploy.conf][DEBUG ] found configuration file at: /home/ceph-admin/.cephdeploy.conf - node-admin: [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy mds create node-osd1 node-osd2 + node-admin: [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy mds create node-osd1 node-osd2 node-osd3 node-admin: [ceph_deploy.cli][INFO ] ceph-deploy options: node-admin: [ceph_deploy.cli][INFO ] username : None node-admin: [ceph_deploy.cli][INFO ] verbose : False node-admin: [ceph_deploy.cli][INFO ] overwrite_conf : False node-admin: [ceph_deploy.cli][INFO ] subcommand : create node-admin: [ceph_deploy.cli][INFO ] quiet : False - node-admin: [ceph_deploy.cli][INFO ] cd_conf : + node-admin: [ceph_deploy.cli][INFO ] cd_conf : node-admin: [ceph_deploy.cli][INFO ] cluster : ceph - node-admin: [ceph_deploy.cli][INFO ] func : + node-admin: [ceph_deploy.cli][INFO ] func : node-admin: [ceph_deploy.cli][INFO ] ceph_conf : None - node-admin: [ceph_deploy.cli][INFO ] mds : [('node-osd1', 'node-osd1'), ('node-osd2', 'node-osd2')] + node-admin: [ceph_deploy.cli][INFO ] mds : [('node-osd1', 'node-osd1'), ('node-osd2', 'node-osd2'), ('node-osd3', 'node-osd3')] node-admin: [ceph_deploy.cli][INFO ] default_release : False - node-admin: [ceph_deploy.mds][DEBUG ] Deploying mds, cluster ceph hosts node-osd1:node-osd1 node-osd2:node-osd2 - node-admin: [ceph_deploy][ERROR ] RuntimeError: bootstrap-mds keyring not found; run 'gatherkeys' + node-admin: [ceph_deploy.mds][DEBUG ] Deploying mds, cluster ceph hosts node-osd1:node-osd1 node-osd2:node-osd2 node-osd3:node-osd3 + node-admin: [node-osd1][DEBUG ] connection detected need for sudo + node-admin: [node-osd1][DEBUG ] connected to host: node-osd1 + node-admin: [node-osd1][DEBUG ] detect platform information from remote host + node-admin: [node-osd1][DEBUG ] detect machine type + node-admin: [ceph_deploy.mds][INFO ] Distro info: Ubuntu 18.04 bionic + node-admin: [ceph_deploy.mds][DEBUG ] remote host will use systemd + node-admin: [ceph_deploy.mds][DEBUG ] deploying mds bootstrap to node-osd1 + node-admin: [node-osd1][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [node-osd1][WARNING] mds keyring does not exist yet, creating one + node-admin: [node-osd1][DEBUG ] create a keyring file + node-admin: [node-osd1][DEBUG ] create path if it doesn't exist + node-admin: [node-osd1][INFO ] Running command: sudo ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.node-osd1 osd allow rwx mds allow mon allow profile mds -o /var/lib/ceph/mds/ceph-node-osd1/keyring + node-admin: [node-osd1][INFO ] Running command: sudo systemctl enable ceph-mds@node-osd1 + node-admin: [node-osd1][WARNING] Created symlink /etc/systemd/system/ceph-mds.target.wants/ceph-mds@node-osd1.service → /lib/systemd/system/ceph-mds@.service. + node-admin: [node-osd1][INFO ] Running command: sudo systemctl start ceph-mds@node-osd1 + node-admin: [node-osd1][INFO ] Running command: sudo systemctl enable ceph.target + node-admin: [node-osd2][DEBUG ] connection detected need for sudo + node-admin: [node-osd2][DEBUG ] connected to host: node-osd2 + node-admin: [node-osd2][DEBUG ] detect platform information from remote host + node-admin: [node-osd2][DEBUG ] detect machine type + node-admin: [ceph_deploy.mds][INFO ] Distro info: Ubuntu 18.04 bionic + node-admin: [ceph_deploy.mds][DEBUG ] remote host will use systemd + node-admin: [ceph_deploy.mds][DEBUG ] deploying mds bootstrap to node-osd2 + node-admin: [node-osd2][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [node-osd2][WARNING] mds keyring does not exist yet, creating one + node-admin: [node-osd2][DEBUG ] create a keyring file + node-admin: [node-osd2][DEBUG ] create path if it doesn't exist + node-admin: [node-osd2][INFO ] Running command: sudo ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.node-osd2 osd allow rwx mds allow mon allow profile mds -o /var/lib/ceph/mds/ceph-node-osd2/keyring + node-admin: [node-osd2][INFO ] Running command: sudo systemctl enable ceph-mds@node-osd2 + node-admin: [node-osd2][WARNING] Created symlink /etc/systemd/system/ceph-mds.target.wants/ceph-mds@node-osd2.service → /lib/systemd/system/ceph-mds@.service. + node-admin: [node-osd2][INFO ] Running command: sudo systemctl start ceph-mds@node-osd2 + node-admin: [node-osd2][INFO ] Running command: sudo systemctl enable ceph.target + node-admin: [node-osd3][DEBUG ] connection detected need for sudo + node-admin: [node-osd3][DEBUG ] connected to host: node-osd3 + node-admin: [node-osd3][DEBUG ] detect platform information from remote host + node-admin: [node-osd3][DEBUG ] detect machine type + node-admin: [ceph_deploy.mds][INFO ] Distro info: Ubuntu 18.04 bionic + node-admin: [ceph_deploy.mds][DEBUG ] remote host will use systemd + node-admin: [ceph_deploy.mds][DEBUG ] deploying mds bootstrap to node-osd3 + node-admin: [node-osd3][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf + node-admin: [node-osd3][WARNING] mds keyring does not exist yet, creating one + node-admin: [node-osd3][DEBUG ] create a keyring file + node-admin: [node-osd3][DEBUG ] create path if it doesn't exist + node-admin: [node-osd3][INFO ] Running command: sudo ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.node-osd3 osd allow rwx mds allow mon allow profile mds -o /var/lib/ceph/mds/ceph-node-osd3/keyring + node-admin: [node-osd3][INFO ] Running command: sudo systemctl enable ceph-mds@node-osd3 + node-admin: [node-osd3][WARNING] Created symlink /etc/systemd/system/ceph-mds.target.wants/ceph-mds@node-osd3.service → /lib/systemd/system/ceph-mds@.service. + node-admin: [node-osd3][INFO ] Running command: sudo systemctl start ceph-mds@node-osd3 + node-admin: [node-osd3][INFO ] Running command: sudo systemctl enable ceph.target node-admin: node-admin: {cephtest} {ceph-install} Gather keys on node-osd1 node-admin: [ceph_deploy.conf][DEBUG ] found configuration file at: /home/ceph-admin/.cephdeploy.conf @@ -2335,32 +3810,154 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [ceph_deploy.cli][INFO ] verbose : False node-admin: [ceph_deploy.cli][INFO ] overwrite_conf : False node-admin: [ceph_deploy.cli][INFO ] quiet : False - node-admin: [ceph_deploy.cli][INFO ] cd_conf : + node-admin: [ceph_deploy.cli][INFO ] cd_conf : node-admin: [ceph_deploy.cli][INFO ] cluster : ceph node-admin: [ceph_deploy.cli][INFO ] mon : ['node-admin'] - node-admin: [ceph_deploy.cli][INFO ] func : + node-admin: [ceph_deploy.cli][INFO ] func : node-admin: [ceph_deploy.cli][INFO ] ceph_conf : None node-admin: [ceph_deploy.cli][INFO ] default_release : False - node-admin: [ceph_deploy.gatherkeys][INFO ] Storing keys in temp directory /tmp/tmpjiVKxK - node-admin: Warning: Permanently added 'node-admin,192.168.121.137' (ECDSA) to the list of known hosts. + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing keys in temp directory /tmp/tmphOuYkG + node-admin: Warning: Permanently added 'node-admin,192.168.121.38' (ECDSA) to the list of known hosts. node-admin: [node-admin][DEBUG ] connection detected need for sudo node-admin: [node-admin][DEBUG ] connected to host: node-admin node-admin: [node-admin][DEBUG ] detect platform information from remote host node-admin: [node-admin][DEBUG ] detect machine type node-admin: [node-admin][DEBUG ] get remote short hostname node-admin: [node-admin][DEBUG ] fetch remote file - node-admin: [ceph_deploy.gatherkeys][WARNING] No mon key found in host: node-admin - node-admin: [ceph_deploy.gatherkeys][ERROR ] Failed to connect to host:node-admin - node-admin: [ceph_deploy.gatherkeys][INFO ] Destroy temp directory /tmp/tmpjiVKxK - node-admin: [ceph_deploy][ERROR ] RuntimeError: Failed to connect any mon - node-admin: cp: cannot stat '/home/ceph-admin/ceph.bootstrap-osd.keyring': No such file or directory - node-admin: chown: cannot access '/var/lib/ceph/bootstrap-osd/ceph.keyring': No such file or directory + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --admin-daemon=/var/run/ceph/ceph-mon.node-admin.asok mon_status + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.admin + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-mds + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-mgr + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-osd + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-rgw + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.client.admin.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-mds.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-mgr.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.mon.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-osd.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-rgw.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Destroy temp directory /tmp/tmphOuYkG node-admin: node-admin: {cephtest} {ceph-install} Create OSDs on node-osd1 node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key - node-admin: --> RuntimeError: No valid ceph configuration file was loaded. + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 6ef6ee42-bb65-4327-8a0a-9c7d1f7b716d + node-admin: Running command: /sbin/vgcreate --force --yes ceph-7caabbd4-bd13-43fa-bd04-c55d7b64caa9 /dev/vdb + node-admin: stdout: Physical volume "/dev/vdb" successfully created. + node-admin: stdout: Volume group "ceph-7caabbd4-bd13-43fa-bd04-c55d7b64caa9" successfully created + node-admin: Running command: /sbin/lvcreate --yes -l 2559 -n osd-block-6ef6ee42-bb65-4327-8a0a-9c7d1f7b716d ceph-7caabbd4-bd13-43fa-bd04-c55d7b64caa9 + node-admin: stdout: Logical volume "osd-block-6ef6ee42-bb65-4327-8a0a-9c7d1f7b716d" created. node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key - node-admin: --> RuntimeError: No valid ceph configuration file was loaded. + node-admin: Running command: /bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-0 + node-admin: --> Executable selinuxenabled not in PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin + node-admin: Running command: /bin/chown -h ceph:ceph /dev/ceph-7caabbd4-bd13-43fa-bd04-c55d7b64caa9/osd-block-6ef6ee42-bb65-4327-8a0a-9c7d1f7b716d + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-0 + node-admin: Running command: /bin/ln -s /dev/ceph-7caabbd4-bd13-43fa-bd04-c55d7b64caa9/osd-block-6ef6ee42-bb65-4327-8a0a-9c7d1f7b716d /var/lib/ceph/osd/ceph-0/block + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-0/activate.monmap + node-admin: stderr: 2021-03-23T02:00:29.706+0000 7fb87fec3700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory + node-admin: 2021-03-23T02:00:29.706+0000 7fb87fec3700 -1 AuthRegistry(0x7fb878059880) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx + node-admin: stderr: got monmap epoch 4 + node-admin: Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-0/keyring --create-keyring --name osd.0 --add-key AQC8S1lg8w5uIBAAbpqMjOilmKmiDQRvlDfbkg== + node-admin: stdout: creating /var/lib/ceph/osd/ceph-0/keyring + node-admin: added entity osd.0 auth(key=AQC8S1lg8w5uIBAAbpqMjOilmKmiDQRvlDfbkg==) + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/keyring + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/ + node-admin: Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 0 --monmap /var/lib/ceph/osd/ceph-0/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-0/ --osd-uuid 6ef6ee42-bb65-4327-8a0a-9c7d1f7b716d --setuser ceph --setgroup ceph + node-admin: stderr: 2021-03-23T02:00:30.578+0000 7fb88bd38d80 -1 bluestore(/var/lib/ceph/osd/ceph-0/) _read_fsid unparsable uuid + node-admin: stderr: 2021-03-23T02:00:30.626+0000 7fb88bd38d80 -1 freelist read_size_meta_from_db missing size meta in DB + node-admin: --> ceph-volume lvm prepare successful for: /dev/vdb + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 + node-admin: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-7caabbd4-bd13-43fa-bd04-c55d7b64caa9/osd-block-6ef6ee42-bb65-4327-8a0a-9c7d1f7b716d --path /var/lib/ceph/osd/ceph-0 --no-mon-config + node-admin: Running command: /bin/ln -snf /dev/ceph-7caabbd4-bd13-43fa-bd04-c55d7b64caa9/osd-block-6ef6ee42-bb65-4327-8a0a-9c7d1f7b716d /var/lib/ceph/osd/ceph-0/block + node-admin: Running command: /bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-0 + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 + node-admin: Running command: /bin/systemctl enable ceph-volume@lvm-0-6ef6ee42-bb65-4327-8a0a-9c7d1f7b716d + node-admin: stderr: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-0-6ef6ee42-bb65-4327-8a0a-9c7d1f7b716d.service → /lib/systemd/system/ceph-volume@.service. + node-admin: Running command: /bin/systemctl enable --runtime ceph-osd@0 + node-admin: stderr: Created symlink /run/systemd/system/ceph-osd.target.wants/ceph-osd@0.service → /lib/systemd/system/ceph-osd@.service. + node-admin: Running command: /bin/systemctl start ceph-osd@0 + node-admin: --> ceph-volume lvm activate successful for osd ID: 0 + node-admin: --> ceph-volume lvm create successful for: /dev/vdb + node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new dc01a43c-377d-41f0-82ee-0dc90c438200 + node-admin: Running command: /sbin/vgcreate --force --yes ceph-7eced4ab-7e60-4537-abf4-b13faaae86d1 /dev/vdc + node-admin: stdout: Physical volume "/dev/vdc" successfully created. + node-admin: stdout: Volume group "ceph-7eced4ab-7e60-4537-abf4-b13faaae86d1" successfully created + node-admin: Running command: /sbin/lvcreate --yes -l 2559 -n osd-block-dc01a43c-377d-41f0-82ee-0dc90c438200 ceph-7eced4ab-7e60-4537-abf4-b13faaae86d1 + node-admin: stdout: Logical volume "osd-block-dc01a43c-377d-41f0-82ee-0dc90c438200" created. + node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key + node-admin: Running command: /bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-1 + node-admin: --> Executable selinuxenabled not in PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin + node-admin: Running command: /bin/chown -h ceph:ceph /dev/ceph-7eced4ab-7e60-4537-abf4-b13faaae86d1/osd-block-dc01a43c-377d-41f0-82ee-0dc90c438200 + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-1 + node-admin: Running command: /bin/ln -s /dev/ceph-7eced4ab-7e60-4537-abf4-b13faaae86d1/osd-block-dc01a43c-377d-41f0-82ee-0dc90c438200 /var/lib/ceph/osd/ceph-1/block + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-1/activate.monmap + node-admin: stderr: 2021-03-23T02:00:35.948+0000 7f793f95e700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory + node-admin: 2021-03-23T02:00:35.948+0000 7f793f95e700 -1 AuthRegistry(0x7f7938059880) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx + node-admin: stderr: got monmap epoch 4 + node-admin: Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-1/keyring --create-keyring --name osd.1 --add-key AQDCS1lg5LPULxAADSEBBnz3x0UBgDu/TcO4nQ== + node-admin: stdout: creating /var/lib/ceph/osd/ceph-1/keyring + node-admin: stdout: added entity osd.1 auth(key=AQDCS1lg5LPULxAADSEBBnz3x0UBgDu/TcO4nQ==) + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1/keyring + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1/ + node-admin: Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 1 --monmap /var/lib/ceph/osd/ceph-1/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-1/ --osd-uuid dc01a43c-377d-41f0-82ee-0dc90c438200 --setuser ceph --setgroup ceph + node-admin: stderr: 2021-03-23T02:00:36.400+0000 7f8501768d80 -1 bluestore(/var/lib/ceph/osd/ceph-1/) _read_fsid unparsable uuid + node-admin: stderr: 2021-03-23T02:00:36.444+0000 7f8501768d80 -1 freelist read_size_meta_from_db missing size meta in DB + node-admin: --> ceph-volume lvm prepare successful for: /dev/vdc + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 + node-admin: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-7eced4ab-7e60-4537-abf4-b13faaae86d1/osd-block-dc01a43c-377d-41f0-82ee-0dc90c438200 --path /var/lib/ceph/osd/ceph-1 --no-mon-config + node-admin: Running command: /bin/ln -snf /dev/ceph-7eced4ab-7e60-4537-abf4-b13faaae86d1/osd-block-dc01a43c-377d-41f0-82ee-0dc90c438200 /var/lib/ceph/osd/ceph-1/block + node-admin: Running command: /bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-1 + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 + node-admin: Running command: /bin/systemctl enable ceph-volume@lvm-1-dc01a43c-377d-41f0-82ee-0dc90c438200 + node-admin: stderr: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-1-dc01a43c-377d-41f0-82ee-0dc90c438200.service → /lib/systemd/system/ceph-volume@.service. + node-admin: Running command: /bin/systemctl enable --runtime ceph-osd@1 + node-admin: stderr: Created symlink /run/systemd/system/ceph-osd.target.wants/ceph-osd@1.service → /lib/systemd/system/ceph-osd@.service. + node-admin: Running command: /bin/systemctl start ceph-osd@1 + node-admin: --> ceph-volume lvm activate successful for osd ID: 1 + node-admin: --> ceph-volume lvm create successful for: /dev/vdc + node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new af0d3899-3e37-4ce4-bdb5-ec99b5754474 + node-admin: Running command: /sbin/vgcreate --force --yes ceph-80c8a741-ccab-45f3-b1ee-8e633a161c78 /dev/vdd + node-admin: stdout: Physical volume "/dev/vdd" successfully created. + node-admin: stdout: Volume group "ceph-80c8a741-ccab-45f3-b1ee-8e633a161c78" successfully created + node-admin: Running command: /sbin/lvcreate --yes -l 2559 -n osd-block-af0d3899-3e37-4ce4-bdb5-ec99b5754474 ceph-80c8a741-ccab-45f3-b1ee-8e633a161c78 + node-admin: stdout: Logical volume "osd-block-af0d3899-3e37-4ce4-bdb5-ec99b5754474" created. + node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key + node-admin: Running command: /bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-2 + node-admin: --> Executable selinuxenabled not in PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin + node-admin: Running command: /bin/chown -h ceph:ceph /dev/ceph-80c8a741-ccab-45f3-b1ee-8e633a161c78/osd-block-af0d3899-3e37-4ce4-bdb5-ec99b5754474 + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-2 + node-admin: Running command: /bin/ln -s /dev/ceph-80c8a741-ccab-45f3-b1ee-8e633a161c78/osd-block-af0d3899-3e37-4ce4-bdb5-ec99b5754474 /var/lib/ceph/osd/ceph-2/block + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-2/activate.monmap + node-admin: stderr: 2021-03-23T02:00:41.746+0000 7f998aeaa700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory + node-admin: stderr: + node-admin: stderr: 2021-03-23T02:00:41.746+0000 7f998aeaa700 -1 AuthRegistry(0x7f9984059880) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx + node-admin: stderr: + node-admin: stderr: got monmap epoch 4 + node-admin: Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-2/keyring --create-keyring --name osd.2 --add-key AQDIS1lgrMOZIxAAU/0/ZU0LMULJCWp1pycXKQ== + node-admin: stdout: creating /var/lib/ceph/osd/ceph-2/keyring + node-admin: added entity osd.2 auth(key=AQDIS1lgrMOZIxAAU/0/ZU0LMULJCWp1pycXKQ==) + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2/keyring + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2/ + node-admin: Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 2 --monmap /var/lib/ceph/osd/ceph-2/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-2/ --osd-uuid af0d3899-3e37-4ce4-bdb5-ec99b5754474 --setuser ceph --setgroup ceph + node-admin: stderr: 2021-03-23T02:00:42.214+0000 7f5772cbfd80 -1 bluestore(/var/lib/ceph/osd/ceph-2/) _read_fsid unparsable uuid + node-admin: stderr: 2021-03-23T02:00:42.270+0000 7f5772cbfd80 -1 freelist read_size_meta_from_db missing size meta in DB + node-admin: --> ceph-volume lvm prepare successful for: /dev/vdd + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 + node-admin: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-80c8a741-ccab-45f3-b1ee-8e633a161c78/osd-block-af0d3899-3e37-4ce4-bdb5-ec99b5754474 --path /var/lib/ceph/osd/ceph-2 --no-mon-config + node-admin: Running command: /bin/ln -snf /dev/ceph-80c8a741-ccab-45f3-b1ee-8e633a161c78/osd-block-af0d3899-3e37-4ce4-bdb5-ec99b5754474 /var/lib/ceph/osd/ceph-2/block + node-admin: Running command: /bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-2 + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 + node-admin: Running command: /bin/systemctl enable ceph-volume@lvm-2-af0d3899-3e37-4ce4-bdb5-ec99b5754474 + node-admin: stderr: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-2-af0d3899-3e37-4ce4-bdb5-ec99b5754474.service → /lib/systemd/system/ceph-volume@.service. + node-admin: Running command: /bin/systemctl enable --runtime ceph-osd@2 + node-admin: stderr: Created symlink /run/systemd/system/ceph-osd.target.wants/ceph-osd@2.service → /lib/systemd/system/ceph-osd@.service. + node-admin: Running command: /bin/systemctl start ceph-osd@2 + node-admin: --> ceph-volume lvm activate successful for osd ID: 2 + node-admin: --> ceph-volume lvm create successful for: /dev/vdd node-admin: node-admin: {cephtest} {ceph-install} Gather keys on node-osd2 node-admin: [ceph_deploy.conf][DEBUG ] found configuration file at: /home/ceph-admin/.cephdeploy.conf @@ -2370,34 +3967,332 @@ Bringing machine 'node-osd2' up with 'libvirt' provider... node-admin: [ceph_deploy.cli][INFO ] verbose : False node-admin: [ceph_deploy.cli][INFO ] overwrite_conf : False node-admin: [ceph_deploy.cli][INFO ] quiet : False - node-admin: [ceph_deploy.cli][INFO ] cd_conf : + node-admin: [ceph_deploy.cli][INFO ] cd_conf : node-admin: [ceph_deploy.cli][INFO ] cluster : ceph node-admin: [ceph_deploy.cli][INFO ] mon : ['node-admin'] - node-admin: [ceph_deploy.cli][INFO ] func : + node-admin: [ceph_deploy.cli][INFO ] func : node-admin: [ceph_deploy.cli][INFO ] ceph_conf : None node-admin: [ceph_deploy.cli][INFO ] default_release : False - node-admin: [ceph_deploy.gatherkeys][INFO ] Storing keys in temp directory /tmp/tmpRJKK0I - node-admin: Warning: Permanently added 'node-admin,192.168.121.137' (ECDSA) to the list of known hosts. + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing keys in temp directory /tmp/tmpMim2c8 + node-admin: Warning: Permanently added 'node-admin,192.168.121.38' (ECDSA) to the list of known hosts. node-admin: [node-admin][DEBUG ] connection detected need for sudo node-admin: [node-admin][DEBUG ] connected to host: node-admin node-admin: [node-admin][DEBUG ] detect platform information from remote host node-admin: [node-admin][DEBUG ] detect machine type node-admin: [node-admin][DEBUG ] get remote short hostname node-admin: [node-admin][DEBUG ] fetch remote file - node-admin: [ceph_deploy.gatherkeys][WARNING] No mon key found in host: node-admin - node-admin: [ceph_deploy.gatherkeys][ERROR ] Failed to connect to host:node-admin - node-admin: [ceph_deploy.gatherkeys][INFO ] Destroy temp directory /tmp/tmpRJKK0I - node-admin: [ceph_deploy][ERROR ] RuntimeError: Failed to connect any mon - node-admin: cp: cannot stat '/home/ceph-admin/ceph.bootstrap-osd.keyring': No such file or directory - node-admin: chown: cannot access '/var/lib/ceph/bootstrap-osd/ceph.keyring': No such file or directory + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --admin-daemon=/var/run/ceph/ceph-mon.node-admin.asok mon_status + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.admin + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-mds + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-mgr + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-osd + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-rgw + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.client.admin.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-mds.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-mgr.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.mon.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-osd.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-rgw.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Destroy temp directory /tmp/tmpMim2c8 node-admin: node-admin: {cephtest} {ceph-install} Create OSDs on node-osd2 node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key - node-admin: --> RuntimeError: No valid ceph configuration file was loaded. + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 777cf962-9931-4edc-85af-0ca4bdf976bd + node-admin: Running command: /sbin/vgcreate --force --yes ceph-0de329ad-07c8-4b6e-b3ea-ef88b88d69cc /dev/vdb + node-admin: stdout: Physical volume "/dev/vdb" successfully created. + node-admin: stdout: Volume group "ceph-0de329ad-07c8-4b6e-b3ea-ef88b88d69cc" successfully created + node-admin: Running command: /sbin/lvcreate --yes -l 2559 -n osd-block-777cf962-9931-4edc-85af-0ca4bdf976bd ceph-0de329ad-07c8-4b6e-b3ea-ef88b88d69cc + node-admin: stdout: Logical volume "osd-block-777cf962-9931-4edc-85af-0ca4bdf976bd" created. node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key - node-admin: --> RuntimeError: No valid ceph configuration file was loaded. - node-admin: unable to get monitor info from DNS SRV with service name: ceph-mon - node-admin: [errno 2] RADOS object not found (error connecting to the cluster) + node-admin: Running command: /bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-3 + node-admin: --> Executable selinuxenabled not in PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin + node-admin: Running command: /bin/chown -h ceph:ceph /dev/ceph-0de329ad-07c8-4b6e-b3ea-ef88b88d69cc/osd-block-777cf962-9931-4edc-85af-0ca4bdf976bd + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-0 + node-admin: Running command: /bin/ln -s /dev/ceph-0de329ad-07c8-4b6e-b3ea-ef88b88d69cc/osd-block-777cf962-9931-4edc-85af-0ca4bdf976bd /var/lib/ceph/osd/ceph-3/block + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-3/activate.monmap + node-admin: stderr: 2021-03-23T02:00:56.227+0000 7faee2bb0700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory + node-admin: stderr: + node-admin: stderr: 2021-03-23T02:00:56.227+0000 7faee2bb0700 -1 AuthRegistry(0x7faedc059880) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx + node-admin: stderr: + node-admin: stderr: got monmap epoch 4 + node-admin: Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-3/keyring --create-keyring --name osd.3 --add-key AQDXS1lg4QGoARAAZ0H+zzN8pl6MichK/jSeGA== + node-admin: stdout: creating /var/lib/ceph/osd/ceph-3/keyring + node-admin: stdout: added entity osd.3 auth(key=AQDXS1lg4QGoARAAZ0H+zzN8pl6MichK/jSeGA==) + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3/keyring + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3/ + node-admin: Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 3 --monmap /var/lib/ceph/osd/ceph-3/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-3/ --osd-uuid 777cf962-9931-4edc-85af-0ca4bdf976bd --setuser ceph --setgroup ceph + node-admin: stderr: 2021-03-23T02:00:56.711+0000 7f2cef0a4d80 -1 bluestore(/var/lib/ceph/osd/ceph-3/) _read_fsid unparsable uuid + node-admin: stderr: 2021-03-23T02:00:56.787+0000 7f2cef0a4d80 -1 freelist read_size_meta_from_db missing size meta in DB + node-admin: --> ceph-volume lvm prepare successful for: /dev/vdb + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 + node-admin: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-0de329ad-07c8-4b6e-b3ea-ef88b88d69cc/osd-block-777cf962-9931-4edc-85af-0ca4bdf976bd --path /var/lib/ceph/osd/ceph-3 --no-mon-config + node-admin: Running command: /bin/ln -snf /dev/ceph-0de329ad-07c8-4b6e-b3ea-ef88b88d69cc/osd-block-777cf962-9931-4edc-85af-0ca4bdf976bd /var/lib/ceph/osd/ceph-3/block + node-admin: Running command: /bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-0 + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 + node-admin: Running command: /bin/systemctl enable ceph-volume@lvm-3-777cf962-9931-4edc-85af-0ca4bdf976bd + node-admin: stderr: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-3-777cf962-9931-4edc-85af-0ca4bdf976bd.service → /lib/systemd/system/ceph-volume@.service. + node-admin: Running command: /bin/systemctl enable --runtime ceph-osd@3 + node-admin: stderr: Created symlink /run/systemd/system/ceph-osd.target.wants/ceph-osd@3.service → /lib/systemd/system/ceph-osd@.service. + node-admin: Running command: /bin/systemctl start ceph-osd@3 + node-admin: --> ceph-volume lvm activate successful for osd ID: 3 + node-admin: --> ceph-volume lvm create successful for: /dev/vdb + node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new b454e036-8aba-4f01-b59d-2866627275bc + node-admin: Running command: /sbin/vgcreate --force --yes ceph-e5da5ae1-1198-4203-8746-900380b863e5 /dev/vdc + node-admin: stdout: Physical volume "/dev/vdc" successfully created. + node-admin: stdout: Volume group "ceph-e5da5ae1-1198-4203-8746-900380b863e5" successfully created + node-admin: Running command: /sbin/lvcreate --yes -l 2559 -n osd-block-b454e036-8aba-4f01-b59d-2866627275bc ceph-e5da5ae1-1198-4203-8746-900380b863e5 + node-admin: stdout: Logical volume "osd-block-b454e036-8aba-4f01-b59d-2866627275bc" created. + node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key + node-admin: Running command: /bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-4 + node-admin: --> Executable selinuxenabled not in PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin + node-admin: Running command: /bin/chown -h ceph:ceph /dev/ceph-e5da5ae1-1198-4203-8746-900380b863e5/osd-block-b454e036-8aba-4f01-b59d-2866627275bc + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-1 + node-admin: Running command: /bin/ln -s /dev/ceph-e5da5ae1-1198-4203-8746-900380b863e5/osd-block-b454e036-8aba-4f01-b59d-2866627275bc /var/lib/ceph/osd/ceph-4/block + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-4/activate.monmap + node-admin: stderr: 2021-03-23T02:01:02.087+0000 7f99d240b700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory + node-admin: stderr: + node-admin: stderr: 2021-03-23T02:01:02.087+0000 7f99d240b700 -1 AuthRegistry(0x7f99cc059880) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx + node-admin: stderr: + node-admin: stderr: got monmap epoch 4 + node-admin: Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-4/keyring --create-keyring --name osd.4 --add-key AQDcS1lgSfmJNxAA4sqbepQVNfu+JF4a3havUg== + node-admin: stdout: creating /var/lib/ceph/osd/ceph-4/keyring + node-admin: added entity osd.4 auth(key=AQDcS1lgSfmJNxAA4sqbepQVNfu+JF4a3havUg==) + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4/keyring + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4/ + node-admin: Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 4 --monmap /var/lib/ceph/osd/ceph-4/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-4/ --osd-uuid b454e036-8aba-4f01-b59d-2866627275bc --setuser ceph --setgroup ceph + node-admin: stderr: 2021-03-23T02:01:02.595+0000 7f9702caed80 -1 bluestore(/var/lib/ceph/osd/ceph-4/) _read_fsid unparsable uuid + node-admin: stderr: 2021-03-23T02:01:02.639+0000 7f9702caed80 -1 freelist read_size_meta_from_db missing size meta in DB + node-admin: --> ceph-volume lvm prepare successful for: /dev/vdc + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 + node-admin: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-e5da5ae1-1198-4203-8746-900380b863e5/osd-block-b454e036-8aba-4f01-b59d-2866627275bc --path /var/lib/ceph/osd/ceph-4 --no-mon-config + node-admin: Running command: /bin/ln -snf /dev/ceph-e5da5ae1-1198-4203-8746-900380b863e5/osd-block-b454e036-8aba-4f01-b59d-2866627275bc /var/lib/ceph/osd/ceph-4/block + node-admin: Running command: /bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-1 + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 + node-admin: Running command: /bin/systemctl enable ceph-volume@lvm-4-b454e036-8aba-4f01-b59d-2866627275bc + node-admin: stderr: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-4-b454e036-8aba-4f01-b59d-2866627275bc.service → /lib/systemd/system/ceph-volume@.service. + node-admin: Running command: /bin/systemctl enable --runtime ceph-osd@4 + node-admin: stderr: Created symlink /run/systemd/system/ceph-osd.target.wants/ceph-osd@4.service → /lib/systemd/system/ceph-osd@.service. + node-admin: Running command: /bin/systemctl start ceph-osd@4 + node-admin: --> ceph-volume lvm activate successful for osd ID: 4 + node-admin: --> ceph-volume lvm create successful for: /dev/vdc + node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 38484789-259d-4574-aa67-a9afd0cc48b0 + node-admin: Running command: /sbin/vgcreate --force --yes ceph-edd7af76-56e2-42bf-a9d2-6b64cc22d684 /dev/vdd + node-admin: stdout: Physical volume "/dev/vdd" successfully created. + node-admin: stdout: Volume group "ceph-edd7af76-56e2-42bf-a9d2-6b64cc22d684" successfully created + node-admin: Running command: /sbin/lvcreate --yes -l 2559 -n osd-block-38484789-259d-4574-aa67-a9afd0cc48b0 ceph-edd7af76-56e2-42bf-a9d2-6b64cc22d684 + node-admin: stdout: Logical volume "osd-block-38484789-259d-4574-aa67-a9afd0cc48b0" created. + node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key + node-admin: Running command: /bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-5 + node-admin: --> Executable selinuxenabled not in PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin + node-admin: Running command: /bin/chown -h ceph:ceph /dev/ceph-edd7af76-56e2-42bf-a9d2-6b64cc22d684/osd-block-38484789-259d-4574-aa67-a9afd0cc48b0 + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-2 + node-admin: Running command: /bin/ln -s /dev/ceph-edd7af76-56e2-42bf-a9d2-6b64cc22d684/osd-block-38484789-259d-4574-aa67-a9afd0cc48b0 /var/lib/ceph/osd/ceph-5/block + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-5/activate.monmap + node-admin: stderr: 2021-03-23T02:01:08.051+0000 7f82c65a4700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory + node-admin: 2021-03-23T02:01:08.051+0000 7f82c65a4700 -1 AuthRegistry(0x7f82c0059880) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx + node-admin: stderr: got monmap epoch 4 + node-admin: Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-5/keyring --create-keyring --name osd.5 --add-key AQDiS1lgR+FoMRAAL4EMqcAK9iSsz42DT7k62Q== + node-admin: stdout: creating /var/lib/ceph/osd/ceph-5/keyring + node-admin: stdout: added entity osd.5 auth(key=AQDiS1lgR+FoMRAAL4EMqcAK9iSsz42DT7k62Q==) + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5/keyring + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5/ + node-admin: Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 5 --monmap /var/lib/ceph/osd/ceph-5/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-5/ --osd-uuid 38484789-259d-4574-aa67-a9afd0cc48b0 --setuser ceph --setgroup ceph + node-admin: stderr: 2021-03-23T02:01:08.527+0000 7f3abed4dd80 -1 bluestore(/var/lib/ceph/osd/ceph-5/) _read_fsid unparsable uuid + node-admin: stderr: 2021-03-23T02:01:08.571+0000 7f3abed4dd80 -1 freelist read_size_meta_from_db missing size meta in DB + node-admin: --> ceph-volume lvm prepare successful for: /dev/vdd + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 + node-admin: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-edd7af76-56e2-42bf-a9d2-6b64cc22d684/osd-block-38484789-259d-4574-aa67-a9afd0cc48b0 --path /var/lib/ceph/osd/ceph-5 --no-mon-config + node-admin: Running command: /bin/ln -snf /dev/ceph-edd7af76-56e2-42bf-a9d2-6b64cc22d684/osd-block-38484789-259d-4574-aa67-a9afd0cc48b0 /var/lib/ceph/osd/ceph-5/block + node-admin: Running command: /bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-2 + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 + node-admin: Running command: /bin/systemctl enable ceph-volume@lvm-5-38484789-259d-4574-aa67-a9afd0cc48b0 + node-admin: stderr: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-5-38484789-259d-4574-aa67-a9afd0cc48b0.service → /lib/systemd/system/ceph-volume@.service. + node-admin: Running command: /bin/systemctl enable --runtime ceph-osd@5 + node-admin: stderr: Created symlink /run/systemd/system/ceph-osd.target.wants/ceph-osd@5.service → /lib/systemd/system/ceph-osd@.service. + node-admin: Running command: /bin/systemctl start ceph-osd@5 + node-admin: --> ceph-volume lvm activate successful for osd ID: 5 + node-admin: --> ceph-volume lvm create successful for: /dev/vdd + node-admin: + node-admin: {cephtest} {ceph-install} Gather keys on node-osd3 + node-admin: [ceph_deploy.conf][DEBUG ] found configuration file at: /home/ceph-admin/.cephdeploy.conf + node-admin: [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy gatherkeys node-admin + node-admin: [ceph_deploy.cli][INFO ] ceph-deploy options: + node-admin: [ceph_deploy.cli][INFO ] username : None + node-admin: [ceph_deploy.cli][INFO ] verbose : False + node-admin: [ceph_deploy.cli][INFO ] overwrite_conf : False + node-admin: [ceph_deploy.cli][INFO ] quiet : False + node-admin: [ceph_deploy.cli][INFO ] cd_conf : + node-admin: [ceph_deploy.cli][INFO ] cluster : ceph + node-admin: [ceph_deploy.cli][INFO ] mon : ['node-admin'] + node-admin: [ceph_deploy.cli][INFO ] func : + node-admin: [ceph_deploy.cli][INFO ] ceph_conf : None + node-admin: [ceph_deploy.cli][INFO ] default_release : False + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing keys in temp directory /tmp/tmpFglECH + node-admin: Warning: Permanently added 'node-admin,192.168.121.38' (ECDSA) to the list of known hosts. + node-admin: [node-admin][DEBUG ] connection detected need for sudo + node-admin: [node-admin][DEBUG ] connected to host: node-admin + node-admin: [node-admin][DEBUG ] detect platform information from remote host + node-admin: [node-admin][DEBUG ] detect machine type + node-admin: [node-admin][DEBUG ] get remote short hostname + node-admin: [node-admin][DEBUG ] fetch remote file + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --admin-daemon=/var/run/ceph/ceph-mon.node-admin.asok mon_status + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.admin + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-mds + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-mgr + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-osd + node-admin: [node-admin][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-node-admin/keyring auth get client.bootstrap-rgw + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.client.admin.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-mds.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-mgr.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.mon.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-osd.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-rgw.keyring + node-admin: [ceph_deploy.gatherkeys][INFO ] Destroy temp directory /tmp/tmpFglECH + node-admin: + node-admin: {cephtest} {ceph-install} Create OSDs on node-osd3 + node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 1e85154a-b1a8-4a63-b567-2ba58b819cf8 + node-admin: Running command: /sbin/vgcreate --force --yes ceph-4ed63a0c-b859-4444-875c-d2eb687daec6 /dev/vdb + node-admin: stdout: Physical volume "/dev/vdb" successfully created. + node-admin: stdout: Volume group "ceph-4ed63a0c-b859-4444-875c-d2eb687daec6" successfully created + node-admin: Running command: /sbin/lvcreate --yes -l 2559 -n osd-block-1e85154a-b1a8-4a63-b567-2ba58b819cf8 ceph-4ed63a0c-b859-4444-875c-d2eb687daec6 + node-admin: stdout: Logical volume "osd-block-1e85154a-b1a8-4a63-b567-2ba58b819cf8" created. + node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key + node-admin: Running command: /bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-6 + node-admin: --> Executable selinuxenabled not in PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin + node-admin: Running command: /bin/chown -h ceph:ceph /dev/ceph-4ed63a0c-b859-4444-875c-d2eb687daec6/osd-block-1e85154a-b1a8-4a63-b567-2ba58b819cf8 + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-0 + node-admin: Running command: /bin/ln -s /dev/ceph-4ed63a0c-b859-4444-875c-d2eb687daec6/osd-block-1e85154a-b1a8-4a63-b567-2ba58b819cf8 /var/lib/ceph/osd/ceph-6/block + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-6/activate.monmap + node-admin: stderr: 2021-03-23T02:01:22.433+0000 7f622e3ab700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory + node-admin: stderr: + node-admin: stderr: 2021-03-23T02:01:22.433+0000 7f622e3ab700 -1 AuthRegistry(0x7f6228059880) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx + node-admin: stderr: + node-admin: stderr: got monmap epoch 4 + node-admin: Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-6/keyring --create-keyring --name osd.6 --add-key AQDxS1lg0wyjERAAMz9AWVJDXhWZq7IHeyVE0w== + node-admin: stdout: creating /var/lib/ceph/osd/ceph-6/keyring + node-admin: stdout: added entity osd.6 auth(key=AQDxS1lg0wyjERAAMz9AWVJDXhWZq7IHeyVE0w==) + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6/keyring + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6/ + node-admin: Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 6 --monmap /var/lib/ceph/osd/ceph-6/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-6/ --osd-uuid 1e85154a-b1a8-4a63-b567-2ba58b819cf8 --setuser ceph --setgroup ceph + node-admin: stderr: 2021-03-23T02:01:22.881+0000 7f01fe957d80 -1 bluestore(/var/lib/ceph/osd/ceph-6/) _read_fsid unparsable uuid + node-admin: stderr: 2021-03-23T02:01:22.933+0000 7f01fe957d80 -1 freelist read_size_meta_from_db missing size meta in DB + node-admin: --> ceph-volume lvm prepare successful for: /dev/vdb + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 + node-admin: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-4ed63a0c-b859-4444-875c-d2eb687daec6/osd-block-1e85154a-b1a8-4a63-b567-2ba58b819cf8 --path /var/lib/ceph/osd/ceph-6 --no-mon-config + node-admin: Running command: /bin/ln -snf /dev/ceph-4ed63a0c-b859-4444-875c-d2eb687daec6/osd-block-1e85154a-b1a8-4a63-b567-2ba58b819cf8 /var/lib/ceph/osd/ceph-6/block + node-admin: Running command: /bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-0 + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 + node-admin: Running command: /bin/systemctl enable ceph-volume@lvm-6-1e85154a-b1a8-4a63-b567-2ba58b819cf8 + node-admin: stderr: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-6-1e85154a-b1a8-4a63-b567-2ba58b819cf8.service → /lib/systemd/system/ceph-volume@.service. + node-admin: Running command: /bin/systemctl enable --runtime ceph-osd@6 + node-admin: stderr: Created symlink /run/systemd/system/ceph-osd.target.wants/ceph-osd@6.service → /lib/systemd/system/ceph-osd@.service. + node-admin: Running command: /bin/systemctl start ceph-osd@6 + node-admin: --> ceph-volume lvm activate successful for osd ID: 6 + node-admin: --> ceph-volume lvm create successful for: /dev/vdb + node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new abe09057-ea16-4e43-b0d4-2461f1a70cc0 + node-admin: Running command: /sbin/vgcreate --force --yes ceph-15c36edd-a2e3-48f1-96b9-0ad8a92167f0 /dev/vdc + node-admin: stdout: Physical volume "/dev/vdc" successfully created. + node-admin: stdout: Volume group "ceph-15c36edd-a2e3-48f1-96b9-0ad8a92167f0" successfully created + node-admin: Running command: /sbin/lvcreate --yes -l 2559 -n osd-block-abe09057-ea16-4e43-b0d4-2461f1a70cc0 ceph-15c36edd-a2e3-48f1-96b9-0ad8a92167f0 + node-admin: stdout: Logical volume "osd-block-abe09057-ea16-4e43-b0d4-2461f1a70cc0" created. + node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key + node-admin: Running command: /bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-7 + node-admin: --> Executable selinuxenabled not in PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin + node-admin: Running command: /bin/chown -h ceph:ceph /dev/ceph-15c36edd-a2e3-48f1-96b9-0ad8a92167f0/osd-block-abe09057-ea16-4e43-b0d4-2461f1a70cc0 + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-1 + node-admin: Running command: /bin/ln -s /dev/ceph-15c36edd-a2e3-48f1-96b9-0ad8a92167f0/osd-block-abe09057-ea16-4e43-b0d4-2461f1a70cc0 /var/lib/ceph/osd/ceph-7/block + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-7/activate.monmap + node-admin: stderr: 2021-03-23T02:01:28.127+0000 7f5ea8878700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory + node-admin: 2021-03-23T02:01:28.127+0000 7f5ea8878700 -1 AuthRegistry(0x7f5ea4059880) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx + node-admin: stderr: got monmap epoch 4 + node-admin: Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-7/keyring --create-keyring --name osd.7 --add-key AQD2S1lgwabQOhAA1q9v2ZGF2PpQpVin5v2mLA== + node-admin: stdout: creating /var/lib/ceph/osd/ceph-7/keyring + node-admin: stdout: added entity osd.7 auth(key=AQD2S1lgwabQOhAA1q9v2ZGF2PpQpVin5v2mLA==) + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7/keyring + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7/ + node-admin: Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 7 --monmap /var/lib/ceph/osd/ceph-7/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-7/ --osd-uuid abe09057-ea16-4e43-b0d4-2461f1a70cc0 --setuser ceph --setgroup ceph + node-admin: stderr: 2021-03-23T02:01:28.631+0000 7f2cebbcdd80 -1 bluestore(/var/lib/ceph/osd/ceph-7/) _read_fsid unparsable uuid + node-admin: stderr: 2021-03-23T02:01:28.667+0000 7f2cebbcdd80 -1 freelist read_size_meta_from_db missing size meta in DB + node-admin: --> ceph-volume lvm prepare successful for: /dev/vdc + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 + node-admin: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-15c36edd-a2e3-48f1-96b9-0ad8a92167f0/osd-block-abe09057-ea16-4e43-b0d4-2461f1a70cc0 --path /var/lib/ceph/osd/ceph-7 --no-mon-config + node-admin: Running command: /bin/ln -snf /dev/ceph-15c36edd-a2e3-48f1-96b9-0ad8a92167f0/osd-block-abe09057-ea16-4e43-b0d4-2461f1a70cc0 /var/lib/ceph/osd/ceph-7/block + node-admin: Running command: /bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-1 + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 + node-admin: Running command: /bin/systemctl enable ceph-volume@lvm-7-abe09057-ea16-4e43-b0d4-2461f1a70cc0 + node-admin: stderr: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-7-abe09057-ea16-4e43-b0d4-2461f1a70cc0.service → /lib/systemd/system/ceph-volume@.service. + node-admin: Running command: /bin/systemctl enable --runtime ceph-osd@7 + node-admin: stderr: Created symlink /run/systemd/system/ceph-osd.target.wants/ceph-osd@7.service → /lib/systemd/system/ceph-osd@.service. + node-admin: Running command: /bin/systemctl start ceph-osd@7 + node-admin: --> ceph-volume lvm activate successful for osd ID: 7 + node-admin: --> ceph-volume lvm create successful for: /dev/vdc + node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 77562a54-b08f-4773-b76e-9fa2d399cb11 + node-admin: Running command: /sbin/vgcreate --force --yes ceph-0da4bcf3-bb18-4cba-bd31-4eb0889befb2 /dev/vdd + node-admin: stdout: Physical volume "/dev/vdd" successfully created. + node-admin: stdout: Volume group "ceph-0da4bcf3-bb18-4cba-bd31-4eb0889befb2" successfully created + node-admin: Running command: /sbin/lvcreate --yes -l 2559 -n osd-block-77562a54-b08f-4773-b76e-9fa2d399cb11 ceph-0da4bcf3-bb18-4cba-bd31-4eb0889befb2 + node-admin: stdout: Logical volume "osd-block-77562a54-b08f-4773-b76e-9fa2d399cb11" created. + node-admin: Running command: /usr/bin/ceph-authtool --gen-print-key + node-admin: Running command: /bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-8 + node-admin: --> Executable selinuxenabled not in PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin + node-admin: Running command: /bin/chown -h ceph:ceph /dev/ceph-0da4bcf3-bb18-4cba-bd31-4eb0889befb2/osd-block-77562a54-b08f-4773-b76e-9fa2d399cb11 + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-2 + node-admin: Running command: /bin/ln -s /dev/ceph-0da4bcf3-bb18-4cba-bd31-4eb0889befb2/osd-block-77562a54-b08f-4773-b76e-9fa2d399cb11 /var/lib/ceph/osd/ceph-8/block + node-admin: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-8/activate.monmap + node-admin: stderr: 2021-03-23T02:01:33.929+0000 7fbefd5d6700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory + node-admin: 2021-03-23T02:01:33.929+0000 7fbefd5d6700 -1 AuthRegistry(0x7fbef8059880) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx + node-admin: stderr: got monmap epoch 4 + node-admin: Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-8/keyring --create-keyring --name osd.8 --add-key AQD8S1lgQ1aBLhAANjhDWlCt/Ka0Pg/lCKTbFQ== + node-admin: stdout: creating /var/lib/ceph/osd/ceph-8/keyring + node-admin: stdout: added entity osd.8 auth(key=AQD8S1lgQ1aBLhAANjhDWlCt/Ka0Pg/lCKTbFQ==) + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-8/keyring + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-8/ + node-admin: Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 8 --monmap /var/lib/ceph/osd/ceph-8/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-8/ --osd-uuid 77562a54-b08f-4773-b76e-9fa2d399cb11 --setuser ceph --setgroup ceph + node-admin: stderr: 2021-03-23T02:01:34.397+0000 7f8662808d80 -1 bluestore(/var/lib/ceph/osd/ceph-8/) _read_fsid unparsable uuid + node-admin: stderr: 2021-03-23T02:01:34.449+0000 7f8662808d80 -1 freelist read_size_meta_from_db missing size meta in DB + node-admin: --> ceph-volume lvm prepare successful for: /dev/vdd + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-8 + node-admin: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-0da4bcf3-bb18-4cba-bd31-4eb0889befb2/osd-block-77562a54-b08f-4773-b76e-9fa2d399cb11 --path /var/lib/ceph/osd/ceph-8 --no-mon-config + node-admin: Running command: /bin/ln -snf /dev/ceph-0da4bcf3-bb18-4cba-bd31-4eb0889befb2/osd-block-77562a54-b08f-4773-b76e-9fa2d399cb11 /var/lib/ceph/osd/ceph-8/block + node-admin: Running command: /bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-8/block + node-admin: Running command: /bin/chown -R ceph:ceph /dev/dm-2 + node-admin: Running command: /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-8 + node-admin: Running command: /bin/systemctl enable ceph-volume@lvm-8-77562a54-b08f-4773-b76e-9fa2d399cb11 + node-admin: stderr: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-8-77562a54-b08f-4773-b76e-9fa2d399cb11.service → /lib/systemd/system/ceph-volume@.service. + node-admin: Running command: /bin/systemctl enable --runtime ceph-osd@8 + node-admin: stderr: Created symlink /run/systemd/system/ceph-osd.target.wants/ceph-osd@8.service → /lib/systemd/system/ceph-osd@.service. + node-admin: Running command: /bin/systemctl start ceph-osd@8 + node-admin: --> ceph-volume lvm activate successful for osd ID: 8 + node-admin: --> ceph-volume lvm create successful for: /dev/vdd + node-admin: cluster: + node-admin: id: 2adbfe38-dcf4-4606-8139-5948a116d429 + node-admin: health: HEALTH_WARN + node-admin: clock skew detected on mon.node-osd1, mon.node-osd3 + node-admin: + node-admin: services: + node-admin: mon: 4 daemons, quorum node-admin,node-osd1,node-osd2,node-osd3 (age 108s) + node-admin: mgr: node-admin(active, since 103s), standbys: node-osd1, node-osd2, node-osd3 + node-admin: mds: 3 up:standby + node-admin: osd: 9 osds: 9 up (since 7s), 9 in (since 7s) + node-admin: + node-admin: task status: + node-admin: + node-admin: data: + node-admin: pools: 1 pools, 1 pgs + node-admin: objects: 0 objects, 0 B + node-admin: usage: 9.1 GiB used, 81 GiB / 90 GiB avail + node-admin: pgs: 1 active+clean + node-admin: node-admin: node-admin: {cephtest} {ceph-install} Ceph cluster is now ready for pool creation and other activities. node-admin: Thanks for waiting. Enjoy! diff --git a/scripts/ceph-install.sh b/scripts/ceph-install.sh index 27e4b26..8682dc7 100755 --- a/scripts/ceph-install.sh +++ b/scripts/ceph-install.sh @@ -102,8 +102,9 @@ for NODE in $OSD_NODES; do ssh $NODE sudo cp /home/$CEPH_ADMIN_USER/ceph.bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring ssh $NODE sudo chown ceph:ceph /var/lib/ceph/bootstrap-osd/ceph.keyring $OUTPUT_LOG "Create OSDs on $NODE" - ssh $NODE sudo ceph-volume lvm create --filestore --data /dev/vdb --journal /dev/vda1 - ssh $NODE sudo ceph-volume lvm create --filestore --data /dev/vdc --journal /dev/vda2 + ssh $NODE sudo ceph-volume lvm create --bluestore --data /dev/vdb + ssh $NODE sudo ceph-volume lvm create --bluestore --data /dev/vdc + ssh $NODE sudo ceph-volume lvm create --bluestore --data /dev/vdd done; # wait 10 seconds and get cluster status diff --git a/scripts/cephtest-utils.sh b/scripts/cephtest-utils.sh index ca9fd0d..35a1308 100755 --- a/scripts/cephtest-utils.sh +++ b/scripts/cephtest-utils.sh @@ -38,7 +38,7 @@ CEPH_ADMIN_USER="ceph-admin" # Nodes ADMIN_NODE="node-admin" -OSD_NODES="node-osd1 node-osd2" +OSD_NODES="node-osd1 node-osd2 node-osd3" NODES="$ADMIN_NODE $OSD_NODES" # Networks