switch-from-non-containerized-to-containerized: mask unit files

We must mask the image so we are sure that even if the system reboots
then the OSDs won't start.

Also remove Ceph udev rules if found on the system prior to deploy
containers. If we don't do this we are exposed to conflicts between udev
rules and sytemd unit files.

Also add the CI will now test the migration from a non-containerized cluster to a
containerized cluster.

Signed-off-by: Sébastien Han <seb@redhat.com>

diff --git a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml
index 6203ca29b717985029529423ceb6d6458f25aaea..b7a6e16d70f6549cd46a145df11cdb8611ab2830 100644 (file)
--- a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml
+++ b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml
@@ -32,6 +32,7 @@
     - "{{ mon_group_name|default('mons') }}"
     - "{{ osd_group_name|default('osds') }}"
     - "{{ mds_group_name|default('mdss') }}"
+    - "{{ mgr_group_name|default('mgrs') }}"
     - "{{ rgw_group_name|default('rgws') }}"
     - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
     - "{{ nfs_group_name|default('nfss') }}"
@@ -39,32 +40,6 @@
   become: true
 
   tasks:
-    - name: install docker and dependencies for the docker module
-      package:
-        name: "{{ item }}"
-        state: present
-      with_items:
-        - python-docker-py
-        - python-urllib3
-        - docker
-      when: ansible_os_family == 'RedHat'
-
-    - name: install docker-py for the docker module
-      package:
-        name: "{{ item }}"
-        state: present
-      with_items:
-        - docker-py
-        - python-urllib3
-        - docker
-      when: ansible_os_family == 'Debian'
-
-    - name: start docker service
-      service:
-        name: docker
-        state: started
-        enabled: yes
-
     - name: check if selinux is enabled
       command: getenforce
       register: sestatus
@@ -81,6 +56,12 @@
         - sestatus.stdout != 'Disabled'
         - ansible_os_family == 'RedHat'
 
+    - name: gather and delegate facts
+      setup:
+      delegate_to: "{{ item }}"
+      delegate_facts: True
+      with_items: "{{ groups['all'] }}"
+
 - name: switching from non-containerized to containerized ceph mon
 
   vars:
@@ -143,9 +124,11 @@
     - ceph-mon
 
   post_tasks:
+    # We don't do a container test by running 'docker exec ...' since not all the monitors have switched to containers yet.
+    # Thus, we continue to use the 'ceph' binary from the host, there is no issue with that.
     - name: waiting for the containerized monitor to join the quorum...
       shell: |
-        docker exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["quorum_names"])'
+        ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["quorum_names"])'
       register: result
       until: "{{ ansible_hostname in result.stdout }}"
       retries: "{{ health_mon_check_retries }}"
@@ -182,10 +165,11 @@
       changed_when: false
 
     - name: stop non-containerized ceph osd(s)
-      service:
+      systemd:
         name: "{{ item }}"
         state: stopped
         enabled: no
+        masked: yes
       with_items: "{{ running_osds.stdout_lines }}"
 
     - set_fact:
@@ -259,6 +243,10 @@
   hosts:
     - "{{ mds_group_name|default('mdss') }}"
 
+  vars:
+    containerized_deployment: true
+    mds_group_name: mdss
+
   serial: 1
   become: true
 
@@ -297,13 +285,17 @@
   hosts:
     - "{{ rgw_group_name|default('rgws') }}"
 
+  vars:
+    containerized_deployment: true
+    rgw_group_name: rgws
+
   serial: 1
   become: true
 
   pre_tasks:
     - name: stop non-containerized ceph rgw(s)
       service:
-        name: "ceph-rgw@{{ ansible_hostname }}"
+        name: "ceph-radosgw@rgw.{{ ansible_hostname }}"
         state: stopped
         enabled: no
 
@@ -335,13 +327,17 @@
   hosts:
     - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
 
+  vars:
+    containerized_deployment: true
+    rbdmirror_group_name: rbdmirrors
+
   serial: 1
   become: true
 
   pre_tasks:
     - name: stop non-containerized ceph rbd mirror(s)
       service:
-        name: "ceph-rbd-mirror@{{ ansible_hostname }}"
+        name: "ceph-rbd-mirror@rbd-mirror.{{ ansible_hostname }}"
         state: stopped
         enabled: no
 
@@ -373,6 +369,10 @@
   hosts:
     - "{{ nfs_group_name|default('nfss') }}"
 
+  vars:
+    containerized_deployment: true
+    nfs_group_name: nfss
+
   serial: 1
   become: true
 

diff --git a/roles/ceph-docker-common/tasks/pre_requisites/debian_prerequisites.yml b/roles/ceph-docker-common/tasks/pre_requisites/debian_prerequisites.yml
index effe2a3463401935fc1a880873f0a996b8e4732a..e1655c1a2ed52a1aaa41f3a83ac1ec17c451f860 100644 (file)
--- a/roles/ceph-docker-common/tasks/pre_requisites/debian_prerequisites.yml
+++ b/roles/ceph-docker-common/tasks/pre_requisites/debian_prerequisites.yml
@@ -1,5 +1,4 @@
 ---
-# To install docker on debian
 - name: allow apt to use a repository over https (debian)
   package:
     name: "{{ item }}"
@@ -10,15 +9,11 @@
     - ca-certificates
     - curl
     - software-properties-common
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
 
 - name: add docker's gpg key
   apt_key:
     url: https://apt.dockerproject.org/gpg
     state: present
-  when: ansible_distribution == 'Debian'
 
 - name: add docker and debian testing repository
   apt_repository:
@@ -27,7 +22,6 @@
   with_items:
     - "deb https://apt.dockerproject.org/repo/ debian-{{ ansible_distribution_release }} main"
     - "deb http://http.us.debian.org/debian/ testing contrib main"
-  when: ansible_distribution == 'Debian'
 
 - name: install pip from testing on debian
   package:
@@ -35,26 +29,17 @@
     state: present
     default_release: testing
     update_cache: yes
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
 
 - name: install docker-py via pip for debian
   pip:
     name: docker-py
     state: latest
-  tags:
-    with_pkg
-  when: ansible_distribution == 'Debian'
 
 - name: install docker on debian
   package:
     name: docker-engine
     state: present
     update_cache: yes
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
 
 # NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
 # re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
@@ -62,6 +47,3 @@
   pip:
     name: six
     version: 1.9.0
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg

diff --git a/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml b/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml
index cb0951c86a9bb8f1f3d98047dbe336d0be5a0d21..4296d55653d647cff2949857c2932635e9bbc455 100644 (file)
--- a/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml
+++ b/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml
@@ -1,7 +1,10 @@
 ---
-# Manage debian in a separate file because of specificities
+- include: remove_ceph_udev_rules.yml
+
 - include: debian_prerequisites.yml
   when: ansible_distribution == 'Debian'
+  tags:
+    with_pkg
 
 - name: install docker on ubuntu
   package:

diff --git a/roles/ceph-docker-common/tasks/pre_requisites/remove_ceph_udev_rules.yml b/roles/ceph-docker-common/tasks/pre_requisites/remove_ceph_udev_rules.yml
new file mode 100644 (file)
index 0000000..3e4ed3c
--- /dev/null
+++ b/roles/ceph-docker-common/tasks/pre_requisites/remove_ceph_udev_rules.yml
@@ -0,0 +1,8 @@
+---
+- name: remove ceph udev rules
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - /usr/lib/udev/rules.d/95-ceph-osd.rules
+    - /usr/lib/udev/rules.d/60-ceph-by-parttypeuuid.rules

diff --git a/roles/ceph-docker-common/tasks/stat_ceph_files.yml b/roles/ceph-docker-common/tasks/stat_ceph_files.yml
index 4492843c2c5590dc5d95114bcac85b9c8aec0fe3..01573183c2b53b47c610e2dec1ee98a1325457a8 100644 (file)
--- a/roles/ceph-docker-common/tasks/stat_ceph_files.yml
+++ b/roles/ceph-docker-common/tasks/stat_ceph_files.yml
@@ -25,17 +25,17 @@
     tmp_ceph_mgr_keys: "{{ ceph_conf_key_directory }}/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring"
   with_items: "{{ groups.get(mgr_group_name, []) }}"
   register: tmp_ceph_mgr_keys_result
-  when: "{{ groups.get(mgr_group_name, []) | length > 0 }}"
+  when: groups.get(mgr_group_name, []) | length > 0
 
 - name: convert mgr keys to an array
   set_fact:
     ceph_mgr_keys: "{{ tmp_ceph_mgr_keys_result.results | map(attribute='ansible_facts.tmp_ceph_mgr_keys') | list }}"
-  when: "{{ groups.get(mgr_group_name, []) | length > 0 }}"
+  when: groups.get(mgr_group_name, []) | length > 0
 
 - name: merge mgr keys to config and keys paths
   set_fact:
     ceph_config_keys: "{{ ceph_config_keys + ceph_mgr_keys }}"
-  when: "{{ groups.get(mgr_group_name, []) | length > 0 }}"
+  when: groups.get(mgr_group_name, []) | length > 0
 
 - name: stat for ceph config and keys
   local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}

diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml
index 626ecbc21d1aa3d9ad53650e326ac6518caf09db..b3c46f55cc75c92ebf93b31a623bc9f61fcb5008 100644 (file)
--- a/roles/ceph-mon/tasks/docker/main.yml
+++ b/roles/ceph-mon/tasks/docker/main.yml
@@ -8,25 +8,65 @@
   delay: 15
   until: monitor_socket.rc == 0
 
-- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup
-  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[item]['ansible_' + monitor_interface].ipv4.address }}
-  with_items: "{{ groups[mon_group_name] }}"
+- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup - monitor_interface
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[groups[mon_group_name][0]]['ansible_' + monitor_interface].ipv4.address }}
   changed_when: false
   failed_when: false
   when:
-    - inventory_hostname == groups[mon_group_name][0]
     - not containerized_deployment_with_kv
     - ip_version == 'ipv4'
+    - hostvars[groups[mon_group_name][0]]['monitor_interface'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_interface'] != 'interface'
 
-- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup
-  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[item]['ansible_' + monitor_interface].ipv6[0].address }}]
-  with_items: "{{ groups[mon_group_name] }}"
+- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[groups[mon_group_name][0]]['monitor_address'] }}
+  changed_when: false
+  failed_when: false
+  when:
+    - not containerized_deployment_with_kv
+    - ip_version == 'ipv4'
+    - hostvars[groups[mon_group_name][0]]['monitor_address'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_address'] != '0.0.0.0'
+
+- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address_block
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[groups[mon_group_name][0]]['ansible_all_' + ip_version + '_addresses'] | ipaddr(monitor_address_block) | first }}
+  changed_when: false
+  failed_when: false
+  when:
+    - not containerized_deployment_with_kv
+    - ip_version == 'ipv4'
+    - hostvars[groups[mon_group_name][0]]['monitor_address_block'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_address_block'] | length > 0
+
+- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup - monitor_interface
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[groups[mon_group_name][0]]['ansible_' + monitor_interface].ipv6[0].address }}]
+  changed_when: false
+  failed_when: false
+  when:
+    - not containerized_deployment_with_kv
+    - ip_version == 'ipv6'
+    - hostvars[groups[mon_group_name][0]]['monitor_interface'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_interface'] != 'interface'
+
+- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[groups[mon_group_name][0]]['monitor_address'] }}]
+  changed_when: false
+  failed_when: false
+  when:
+    - not containerized_deployment_with_kv
+    - ip_version == 'ipv6'
+    - hostvars[groups[mon_group_name][0]]['monitor_address'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_address'] != '0.0.0.0'
+
+- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address_block
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[groups[mon_group_name][0]]['ansible_all_' + ip_version + '_addresses'] | ipaddr(monitor_address_block) | first }}]
   changed_when: false
   failed_when: false
   when:
-    - inventory_hostname == groups[mon_group_name][0]
     - not containerized_deployment_with_kv
     - ip_version == 'ipv6'
+    - hostvars[groups[mon_group_name][0]]['monitor_address_block'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_address_block'] | length > 0
 
 - include: copy_configs.yml
   when: not containerized_deployment_with_kv

diff --git a/site-docker.yml.sample b/site-docker.yml.sample
index 238997aeadd199b8e9472f5c6d8cf254b69e7acb..29272223c9be65b5984700a925e995dbbdacd5fb 100644 (file)
--- a/site-docker.yml.sample
+++ b/site-docker.yml.sample
@@ -14,6 +14,8 @@
   - iscsigws
   - mgrs
 
+  gather_facts: false
+
   tasks:
     - name: gather and delegate facts
       setup:
@@ -33,6 +35,7 @@
 
 - hosts: osds
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -41,6 +44,7 @@
 
 - hosts: mdss
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -49,6 +53,7 @@
 
 - hosts: rgws
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -57,6 +62,7 @@
 
 - hosts: nfss
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -65,6 +71,7 @@
 
 - hosts: rbd_mirrors
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -73,6 +80,7 @@
 
 - hosts: restapis
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -81,6 +89,7 @@
 
 - hosts: mgrs
   become: True
+  gather_facts: false
   roles:
     - { role: ceph-defaults, when: "ceph_release_num.{{ ceph_stable_release }} > ceph_release_num.jewel" }
     - { role: ceph-docker-common, when: "ceph_release_num.{{ ceph_stable_release }} > ceph_release_num.jewel" }
@@ -89,6 +98,7 @@
 
 - hosts: clients
   become: True
+  gather_facts: false
   roles:
   - ceph-defaults
   - ceph-common

diff --git a/tox.ini b/tox.ini
index 6b0c0deacdb903d6d83136e4073acd88bac10482..057c012c07ca7b5e4e8c3517fb7190fe7d9cf0da 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -1,6 +1,6 @@
 [tox]
 envlist = {dev,jewel,luminous,rhcs}-{ansible2.2,ansible2.3}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,docker_dmcrypt_journal_collocation,update_dmcrypt,update_cluster,cluster,purge_docker_cluster,update_docker_cluster}
-  {dev,luminous}-{ansible2.2,ansible2.3}-{bluestore_journal_collocation,bluestore_cluster,bluestore_dmcrypt_journal,bluestore_dmcrypt_journal_collocation,bluestore_docker_cluster,bluestore_docker_dedicated_journal,bluestore_docker_dmcrypt_journal_collocation,lvm_osds,purge_lvm_osds,shrink_mon,shrink_osd}
+  {dev,luminous}-{ansible2.2,ansible2.3}-{bluestore_journal_collocation,bluestore_cluster,bluestore_dmcrypt_journal,bluestore_dmcrypt_journal_collocation,bluestore_docker_cluster,bluestore_docker_dedicated_journal,bluestore_docker_dmcrypt_journal_collocation,lvm_osds,purge_lvm_osds,shrink_mon,shrink_osd,switch_to_containers}
 
 skipsdist = True
 
@@ -96,12 +96,29 @@ commands=
       osd_to_kill=0 \
   "
 
+[switch-to-containers]
+commands=
+  cp {toxinidir}/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml {toxinidir}/switch-from-non-containerized-to-containerized-ceph-daemons.yml
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/switch-from-non-containerized-to-containerized-ceph-daemons.yml --extra-vars "\
+      ireallymeanit=yes \
+      fetch_directory={env:FETCH_DIRECTORY:{changedir}/fetch} \
+      ceph_docker_registry={env:CEPH_DOCKER_REGISTRY:docker.io} \
+      ceph_docker_image={env:UPDATE_CEPH_DOCKER_IMAGE:ceph/daemon} \
+      ceph_docker_image_tag={env:UPDATE_CEPH_DOCKER_IMAGE_TAG:latest} \
+      ceph_dev_branch={env:UPDATE_CEPH_DEV_BRANCH:master} \
+      ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest} \
+  "
+
+  echo "docker: True" >> {changedir}/group_vars/all
+  testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
+
 [testenv]
 whitelist_externals =
     vagrant
     bash
     pip
     cp
+    echo
 passenv=*
 sitepackages=True
 setenv=
@@ -157,6 +174,7 @@ changedir=
   purge_dmcrypt: {toxinidir}/tests/functional/centos/7/crypt-ded-jrn
   update_dmcrypt: {toxinidir}/tests/functional/centos/7/crypt-ded-jrn
   update_cluster: {toxinidir}/tests/functional/centos/7/cluster
+  switch_to_containers: {toxinidir}/tests/functional/centos/7/cluster
   bluestore_journal_collocation: {toxinidir}/tests/functional/centos/7/bs-jrn-col
   bluestore_cluster: {toxinidir}/tests/functional/centos/7/bluestore
   bluestore_dmcrypt_journal: {toxinidir}/tests/functional/centos/7/bs-crypt-ded-jrn
@@ -197,6 +215,7 @@ commands=
   purge_lvm_osds: {[purge-lvm]commands}
   purge_dmcrypt: {[purge]commands}
   purge_docker_cluster: {[purge]commands}
+  switch_to_containers: {[switch-to-containers]commands}
   update_dmcrypt: {[update]commands}
   update_cluster: {[update]commands}
   update_docker_cluster: {[update]commands}