mgr: wait for all mgr to be available

before managing mgr modules, we must ensure all mgr are available
otherwise we can hit failure like following:

```
stdout:Error ENOENT: all mgr daemons do not support module 'restful', pass --force to force enablement
```

It happens because all mgr are not yet available when trying to manage
with mgr modules.

This should have been cherry-picked from
41f7518c1ba0fea74a61e4207999ec622a052270 but there's too much changes.

Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>

diff --git a/roles/ceph-mgr/tasks/main.yml b/roles/ceph-mgr/tasks/main.yml
index a2eda822d1fa3942f0ca01fce325a2fdb2e4465f..12a31576434711b2f15a77c973075460b086627f 100644 (file)
--- a/roles/ceph-mgr/tasks/main.yml
+++ b/roles/ceph-mgr/tasks/main.yml
@@ -16,40 +16,52 @@
   include: docker/main.yml
   when: containerized_deployment
 
-- name: get enabled modules from ceph-mgr
-  command: "{{ docker_exec_cmd_mgr | default('') }} ceph --cluster {{ cluster }} --format json mgr module ls"
-  register: _ceph_mgr_modules
-  delegate_to: "{{ groups[mon_group_name][0] }}"
-  when:
-    - ceph_release_num[ceph_release] >= ceph_release_num['luminous']
+- block:
+    - name: wait for all mgr to be up
+      shell: "{{ docker_exec_cmd_mgr | default('') }} ceph --cluster {{ cluster }} mgr dump -f json | python -c 'import sys, json; print(json.load(sys.stdin)[\"available\"])'"
+      register: mgr_dump
+      retries: 30
+      delay: 5
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+      until:
+        - mgr_dump.rc == 0
+        - mgr_dump.stdout | bool
 
-- name: set _ceph_mgr_modules fact (convert _ceph_mgr_modules.stdout to a dict)
-  set_fact:
-    _ceph_mgr_modules: "{{ _ceph_mgr_modules.get('stdout', '{}') | from_json }}"
-  when:
-    - ceph_release_num[ceph_release] >= ceph_release_num['luminous']
+    - name: get enabled modules from ceph-mgr
+      command: "{{ docker_exec_cmd_mgr | default('') }} ceph --cluster {{ cluster }} --format json mgr module ls"
+      register: _ceph_mgr_modules
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+      when:
+        - ceph_release_num[ceph_release] >= ceph_release_num['luminous']
 
-- name: set _disabled_ceph_mgr_modules fact
-  set_fact:
-    _disabled_ceph_mgr_modules: "{% if _ceph_mgr_modules.disabled_modules | length == 0 %}[]{% elif _ceph_mgr_modules.disabled_modules[0] | type_debug != 'dict' %}{{ _ceph_mgr_modules['disabled_modules'] }}{% else %}{{ _ceph_mgr_modules['disabled_modules'] | map(attribute='name') | list }}{% endif %}"
-  when:
-    - ceph_release_num[ceph_release] >= ceph_release_num['luminous']
-
-- name: disable ceph mgr enabled modules
-  command: "{{ docker_exec_cmd_mgr | default('') }} ceph --cluster {{ cluster }} mgr module disable {{ item }}"
-  # When ceph release is jewel, ceph-mgr role is skipped. It means, the enabled_ceph_mgr_modules doesn't contain 'stdout' attribute.
-  # Therefore, we need to get a default value which can be used up by from_json filter.
-  with_items: "{{ _ceph_mgr_modules.get('enabled_modules', []) }}"
-  delegate_to: "{{ groups[mon_group_name][0] }}"
-  when:
-    - item not in ceph_mgr_modules
-    - not _ceph_mgr_modules.get('skipped')
-    - ceph_release_num[ceph_release] >= ceph_release_num['luminous']
-
-- name: add modules to ceph-mgr
-  command: "{{ docker_exec_cmd_mgr | default('') }} ceph --cluster {{ cluster }} mgr module enable {{ item }}"
-  with_items: "{{ ceph_mgr_modules }}"
-  delegate_to: "{{ groups[mon_group_name][0] }}"
-  when:
-    - (item in _disabled_ceph_mgr_modules or _disabled_ceph_mgr_modules == [])
-    - ceph_release_num[ceph_release] >= ceph_release_num['luminous']
+    - name: set _ceph_mgr_modules fact (convert _ceph_mgr_modules.stdout to a dict)
+      set_fact:
+        _ceph_mgr_modules: "{{ _ceph_mgr_modules.get('stdout', '{}') | from_json }}"
+      when:
+        - ceph_release_num[ceph_release] >= ceph_release_num['luminous']
+
+    - name: set _disabled_ceph_mgr_modules fact
+      set_fact:
+        _disabled_ceph_mgr_modules: "{% if _ceph_mgr_modules.disabled_modules | length == 0 %}[]{% elif _ceph_mgr_modules.disabled_modules[0] | type_debug != 'dict' %}{{ _ceph_mgr_modules['disabled_modules'] }}{% else %}{{ _ceph_mgr_modules['disabled_modules'] | map(attribute='name') | list }}{% endif %}"
+      when:
+        - ceph_release_num[ceph_release] >= ceph_release_num['luminous']
+
+    - name: disable ceph mgr enabled modules
+      command: "{{ docker_exec_cmd_mgr | default('') }} ceph --cluster {{ cluster }} mgr module disable {{ item }}"
+      # When ceph release is jewel, ceph-mgr role is skipped. It means, the enabled_ceph_mgr_modules doesn't contain 'stdout' attribute.
+      # Therefore, we need to get a default value which can be used up by from_json filter.
+      with_items: "{{ _ceph_mgr_modules.get('enabled_modules', []) }}"
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+      when:
+        - item not in ceph_mgr_modules
+        - not _ceph_mgr_modules.get('skipped')
+        - ceph_release_num[ceph_release] >= ceph_release_num['luminous']
+
+    - name: add modules to ceph-mgr
+      command: "{{ docker_exec_cmd_mgr | default('') }} ceph --cluster {{ cluster }} mgr module enable {{ item }}"
+      with_items: "{{ ceph_mgr_modules }}"
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+      when:
+        - (item in _disabled_ceph_mgr_modules or _disabled_ceph_mgr_modules == [])
+        - ceph_release_num[ceph_release] >= ceph_release_num['luminous']
+  when: inventory_hostname == groups[mgr_group_name] | last