From a52ed430935e454bcfc20dacea06f87551f88723 Mon Sep 17 00:00:00 2001 From: =?utf8?q?S=C3=A9bastien=20Han?= Date: Fri, 2 Mar 2018 15:50:01 +0100 Subject: [PATCH] mon: fix osd_pool_default_crush_rule persistence and effectiveness MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Running the last portion (insert new default and add new default crush tasks) of crush_rules.yml only on the last monitor is wrong since ceph CLI calls usually end up on the master having the quorum, which is by default the one with the lower IP. So if we run the command and end up on another mon the creation will happen on the default crush rule because the particular mon hasn't been updated. To fix this we remove the |last on the include and use run_once: true on certain tasks, then we let the final two tasks run on all the monitors. Signed-off-by: Sébastien Han --- roles/ceph-mon/tasks/crush_rules.yml | 30 ++++++++++++++++++++-------- roles/ceph-mon/tasks/main.yml | 1 - 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/roles/ceph-mon/tasks/crush_rules.yml b/roles/ceph-mon/tasks/crush_rules.yml index 8e5089530..96b449dda 100644 --- a/roles/ceph-mon/tasks/crush_rules.yml +++ b/roles/ceph-mon/tasks/crush_rules.yml @@ -4,9 +4,10 @@ cluster: "{{ cluster }}" location: "{{ hostvars[item]['osd_crush_location'] }}" containerized: "{{ docker_exec_cmd }}" - run_once: true with_items: "{{ groups[osd_group_name] }}" + register: config_crush_hierarchy when: + - inventory_hostname == groups.get(mon_group_name) | last - create_crush_tree - hostvars[item]['osd_crush_location'] is defined @@ -14,32 +15,41 @@ command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd crush rule create-simple {{ item.name }} {{ item.root }} {{ item.type }}" with_items: "{{ crush_rules | unique }}" changed_when: false - run_once: true + when: + - inventory_hostname == groups.get(mon_group_name) | last - name: get id for new default crush rule command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd -f json crush rule dump {{ item.name }}" register: info_ceph_default_crush_rule changed_when: false with_items: "{{ crush_rules }}" - when: item.default + when: + - inventory_hostname == groups.get(mon_group_name) | last + - item.default # If multiple rules are set as default (should not be) then the last one is taken as actual default. +# the with_items statement overrides each iteration with the new one. # NOTE(leseb): we should actually fail if multiple rules are set as default - name: set_fact info_ceph_default_crush_rule_yaml set_fact: - info_ceph_default_crush_rule_yaml: "{{ item.stdout|from_json() }}" + info_ceph_default_crush_rule_yaml: "{{ item.stdout | from_json() }}" with_items: "{{ info_ceph_default_crush_rule.results }}" - when: not item.get('skipped', false) + when: + - inventory_hostname == groups.get(mon_group_name) | last + - not item.get('skipped', false) - name: set_fact osd_pool_default_crush_rule to osd_pool_default_crush_replicated_ruleset if release < luminous else osd_pool_default_crush_rule set_fact: osd_pool_default_crush_rule: "{{ 'osd_pool_default_crush_replicated_ruleset' if ceph_release_num[ceph_release] < ceph_release_num.luminous else 'osd_pool_default_crush_rule' }}" - name: insert new default crush rule into daemon to prevent restart - command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} daemon mon.{{ monitor_name }} config set {{ osd_pool_default_crush_rule }} {{ info_ceph_default_crush_rule_yaml.rule_id }}" + command: "{{ hostvars[item]['docker_exec_cmd'] | default('') }} ceph --cluster {{ cluster }} daemon mon.{{ hostvars[item]['monitor_name'] }} config set {{ osd_pool_default_crush_rule }} {{ info_ceph_default_crush_rule_yaml.rule_id }}" changed_when: false + delegate_to: "{{ item }}" + with_items: "{{ groups[mon_group_name] }}" when: - - info_ceph_default_crush_rule_yaml|default('')|length > 0 + - not config_crush_hierarchy.get('skipped', false) + - info_ceph_default_crush_rule_yaml | default('') | length > 0 - name: "add new default crush rule to {{ cluster }}.conf" ini_file: @@ -47,4 +57,8 @@ section: "global" option: "{{ osd_pool_default_crush_rule }}" value: "{{ info_ceph_default_crush_rule_yaml.rule_id }}" - when: info_ceph_default_crush_rule_yaml|default('')|length > 0 + delegate_to: "{{ item }}" + with_items: "{{ groups[mon_group_name] }}" + when: + - not config_crush_hierarchy.get('skipped', false) + - info_ceph_default_crush_rule_yaml | default('') | length > 0 diff --git a/roles/ceph-mon/tasks/main.yml b/roles/ceph-mon/tasks/main.yml index fab74a43d..3d70229d9 100644 --- a/roles/ceph-mon/tasks/main.yml +++ b/roles/ceph-mon/tasks/main.yml @@ -36,7 +36,6 @@ include: crush_rules.yml when: - crush_rule_config - - inventory_hostname == groups.get(mon_group_name) | last - name: include set_osd_pool_default_pg_num.yml include: set_osd_pool_default_pg_num.yml -- 2.39.5