mon: fix osd_pool_default_crush_rule persistence and effectiveness

Running the last portion (insert new default and add new default crush
tasks) of crush_rules.yml only on the last monitor is
wrong since ceph CLI calls usually end up on the master having the
quorum, which is by default the one with the lower IP.
So if we run the  command and end up on another mon the creation will
happen on the default crush rule because the particular mon hasn't been
updated.
To fix this we remove the |last on the include and use run_once: true on
 certain tasks, then we let the final two tasks run on all the monitors.

Signed-off-by: Sébastien Han <seb@redhat.com>

diff --git a/roles/ceph-mon/tasks/crush_rules.yml b/roles/ceph-mon/tasks/crush_rules.yml
index 8e508953054fec07217d1b58f8777db69ba2e4e4..96b449dda7727df500602e7912c18159f548a5e6 100644 (file)
--- a/roles/ceph-mon/tasks/crush_rules.yml
+++ b/roles/ceph-mon/tasks/crush_rules.yml
@@ -4,9 +4,10 @@
     cluster: "{{ cluster }}"
     location: "{{ hostvars[item]['osd_crush_location'] }}"
     containerized: "{{ docker_exec_cmd }}"
-  run_once: true
   with_items: "{{ groups[osd_group_name] }}"
+  register: config_crush_hierarchy
   when:
+    - inventory_hostname == groups.get(mon_group_name) | last
     - create_crush_tree
     - hostvars[item]['osd_crush_location'] is defined
 
@@ -14,32 +15,41 @@
   command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd crush rule create-simple {{ item.name }} {{ item.root }} {{ item.type }}"
   with_items: "{{ crush_rules | unique }}"
   changed_when: false
-  run_once: true
+  when:
+    - inventory_hostname == groups.get(mon_group_name) | last
 
 - name: get id for new default crush rule
   command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd -f json crush rule dump {{ item.name }}"
   register: info_ceph_default_crush_rule
   changed_when: false
   with_items: "{{ crush_rules }}"
-  when: item.default
+  when:
+    - inventory_hostname == groups.get(mon_group_name) | last
+    - item.default
 
 # If multiple rules are set as default (should not be) then the last one is taken as actual default.
+# the with_items statement overrides each iteration with the new one.
 # NOTE(leseb): we should actually fail if multiple rules are set as default
 - name: set_fact info_ceph_default_crush_rule_yaml
   set_fact:
-    info_ceph_default_crush_rule_yaml: "{{ item.stdout|from_json() }}"
+    info_ceph_default_crush_rule_yaml: "{{ item.stdout | from_json() }}"
   with_items: "{{ info_ceph_default_crush_rule.results }}"
-  when: not item.get('skipped', false)
+  when:
+    - inventory_hostname == groups.get(mon_group_name) | last
+    - not item.get('skipped', false)
 
 - name: set_fact osd_pool_default_crush_rule to osd_pool_default_crush_replicated_ruleset if release < luminous else osd_pool_default_crush_rule
   set_fact:
     osd_pool_default_crush_rule: "{{ 'osd_pool_default_crush_replicated_ruleset' if ceph_release_num[ceph_release] < ceph_release_num.luminous else 'osd_pool_default_crush_rule' }}"
 
 - name: insert new default crush rule into daemon to prevent restart
-  command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} daemon mon.{{ monitor_name }} config set {{ osd_pool_default_crush_rule }} {{ info_ceph_default_crush_rule_yaml.rule_id }}"
+  command: "{{ hostvars[item]['docker_exec_cmd'] | default('') }} ceph --cluster {{ cluster }} daemon mon.{{ hostvars[item]['monitor_name'] }} config set {{ osd_pool_default_crush_rule }} {{ info_ceph_default_crush_rule_yaml.rule_id }}"
   changed_when: false
+  delegate_to: "{{ item }}"
+  with_items: "{{ groups[mon_group_name] }}"
   when:
-    - info_ceph_default_crush_rule_yaml|default('')|length > 0
+    - not config_crush_hierarchy.get('skipped', false)
+    - info_ceph_default_crush_rule_yaml | default('') | length > 0
 
 - name: "add new default crush rule to {{ cluster }}.conf"
   ini_file:
@@ -47,4 +57,8 @@
     section: "global"
     option: "{{ osd_pool_default_crush_rule }}"
     value: "{{ info_ceph_default_crush_rule_yaml.rule_id }}"
-  when: info_ceph_default_crush_rule_yaml|default('')|length > 0
+  delegate_to: "{{ item }}"
+  with_items: "{{ groups[mon_group_name] }}"
+  when:
+    - not config_crush_hierarchy.get('skipped', false)
+    - info_ceph_default_crush_rule_yaml | default('') | length > 0

diff --git a/roles/ceph-mon/tasks/main.yml b/roles/ceph-mon/tasks/main.yml
index fab74a43d53c51dda948d8c033c1ead4c0cb61d1..3d70229d98b549baa5e513a25613b3031b22393a 100644 (file)
--- a/roles/ceph-mon/tasks/main.yml
+++ b/roles/ceph-mon/tasks/main.yml
@@ -36,7 +36,6 @@
   include: crush_rules.yml
   when:
     - crush_rule_config
-    - inventory_hostname == groups.get(mon_group_name) | last
 
 - name: include set_osd_pool_default_pg_num.yml
   include: set_osd_pool_default_pg_num.yml