From 0050f8e6efdf3e8532072f08dde25f7d862808b2 Mon Sep 17 00:00:00 2001 From: Christian Zunker Date: Mon, 12 Jun 2017 08:36:29 +0000 Subject: [PATCH] Create OpenStack pools with crush rule Add an extra variable to the openstack pools, which creates them with defined rules. This will allow to place different pools on e.g. different type of disks. This commit will also set a new default rule when defined and move the rbd pool to the new rule. --- group_vars/mons.yml.sample | 26 ++++++++++++ group_vars/rgws.yml.sample | 2 +- roles/ceph-mon/defaults/main.yml | 26 ++++++++++++ roles/ceph-mon/tasks/ceph_keys.yml | 4 ++ roles/ceph-mon/tasks/crush_rules.yml | 48 +++++++++++++++++++++++ roles/ceph-mon/tasks/openstack_config.yml | 2 +- 6 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 roles/ceph-mon/tasks/crush_rules.yml diff --git a/group_vars/mons.yml.sample b/group_vars/mons.yml.sample index 8244d650e..688d01f57 100644 --- a/group_vars/mons.yml.sample +++ b/group_vars/mons.yml.sample @@ -44,6 +44,27 @@ dummy: # Enable debugging for Calamari #calamari_debug: false + +############# +# crush rules +############# +#crush_rule_config: false + +#crush_rule_hdd: +# name: HDD +# root: HDD +# type: rack +# default: true + +#crush_rule_ssd: +# name: SSD +# root: SSD +# type: host +# default: false + +#crush_rules: +# - "{{ crush_rule_hdd }}" +# - "{{ crush_rule_ssd }}" ############# # OPENSTACK # ############# @@ -52,18 +73,23 @@ dummy: #openstack_glance_pool: # name: images # pg_num: "{{ osd_pool_default_pg_num }}" +# rule_name: "" #openstack_cinder_pool: # name: volumes # pg_num: "{{ osd_pool_default_pg_num }}" +# rule_name: "" #openstack_nova_pool: # name: vms # pg_num: "{{ osd_pool_default_pg_num }}" +# rule_name: "" #openstack_cinder_backup_pool: # name: backups # pg_num: "{{ osd_pool_default_pg_num }}" +# rule_name: "" #openstack_gnocchi_pool: # name: metrics # pg_num: "{{ osd_pool_default_pg_num }}" +# rule_name: "" #openstack_pools: # - "{{ openstack_glance_pool }}" diff --git a/group_vars/rgws.yml.sample b/group_vars/rgws.yml.sample index 22e2eeb04..46c9150ea 100644 --- a/group_vars/rgws.yml.sample +++ b/group_vars/rgws.yml.sample @@ -33,7 +33,7 @@ dummy: # important to split them into shards. We suggest about 100K # objects per shard as a conservative maximum. #rgw_override_bucket_index_max_shards: 16 -# + # Consider setting a quota on buckets so that exceeding this # limit will require admin intervention. #rgw_bucket_default_quota_max_objects: 1638400 # i.e., 100K * 16 diff --git a/roles/ceph-mon/defaults/main.yml b/roles/ceph-mon/defaults/main.yml index 629b8b924..73bfc3241 100644 --- a/roles/ceph-mon/defaults/main.yml +++ b/roles/ceph-mon/defaults/main.yml @@ -36,6 +36,27 @@ calamari: false # Enable debugging for Calamari calamari_debug: false + +############# +# crush rules +############# +crush_rule_config: false + +crush_rule_hdd: + name: HDD + root: HDD + type: rack + default: true + +crush_rule_ssd: + name: SSD + root: SSD + type: host + default: false + +crush_rules: + - "{{ crush_rule_hdd }}" + - "{{ crush_rule_ssd }}" ############# # OPENSTACK # ############# @@ -44,18 +65,23 @@ openstack_config: false openstack_glance_pool: name: images pg_num: "{{ osd_pool_default_pg_num }}" + rule_name: "" openstack_cinder_pool: name: volumes pg_num: "{{ osd_pool_default_pg_num }}" + rule_name: "" openstack_nova_pool: name: vms pg_num: "{{ osd_pool_default_pg_num }}" + rule_name: "" openstack_cinder_backup_pool: name: backups pg_num: "{{ osd_pool_default_pg_num }}" + rule_name: "" openstack_gnocchi_pool: name: metrics pg_num: "{{ osd_pool_default_pg_num }}" + rule_name: "" openstack_pools: - "{{ openstack_glance_pool }}" diff --git a/roles/ceph-mon/tasks/ceph_keys.yml b/roles/ceph-mon/tasks/ceph_keys.yml index 84f42477a..256426f85 100644 --- a/roles/ceph-mon/tasks/ceph_keys.yml +++ b/roles/ceph-mon/tasks/ceph_keys.yml @@ -58,6 +58,10 @@ - include: set_osd_pool_default_pg_num.yml +- include: crush_rules.yml + when: + - crush_rule_config + - name: test if rbd exists shell: | ceph --cluster {{ cluster }} osd pool ls | grep -sq rbd diff --git a/roles/ceph-mon/tasks/crush_rules.yml b/roles/ceph-mon/tasks/crush_rules.yml new file mode 100644 index 000000000..2c5d3c5a7 --- /dev/null +++ b/roles/ceph-mon/tasks/crush_rules.yml @@ -0,0 +1,48 @@ +--- +- name: create roots needed for configured crush rules + command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd crush add-bucket {{ item.root }} root" + with_items: "{{ crush_rules | unique }}" + changed_when: false + failed_when: false + run_once: true + +- name: create configured crush rules + command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd crush rule create-simple {{ item.name }} {{ item.root }} {{ item.type }}" + with_items: "{{ crush_rules | unique }}" + changed_when: false + failed_when: false + run_once: true + +- name: get id for new default crush rule + command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd -f json crush rule dump {{ item.name }}" + register: info_ceph_default_crush_rule + changed_when: false + failed_when: false + with_items: "{{ crush_rules }}" + when: item.default + +- name: set crush rule info as fact + set_fact: + info_ceph_default_crush_rule_yaml: "{{ info_ceph_default_crush_rule.results[0].stdout|from_json() }}" + when: info_ceph_default_crush_rule.results|length > 0 + +- name: insert new default crush rule into daemon to prevent restart + command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} daemon mon.{{ monitor_name }} config set osd_pool_default_crush_replicated_ruleset {{ info_ceph_default_crush_rule_yaml.rule_id }}" + changed_when: false + failed_when: false + when: info_ceph_default_crush_rule.results|length > 0 + +- name: add new default crush rule to ceph.conf + ini_file: + dest: "/etc/ceph/{{ cluster }}.conf" + section: "global" + option: "osd pool default crush replicated ruleset" + value: "{{ info_ceph_default_crush_rule_yaml.rule_id }}" + when: info_ceph_default_crush_rule.results|length > 0 + +- name: move rbd pool to new default root + command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd pool set rbd crush_ruleset {{ info_ceph_default_crush_rule_yaml.rule_id }}" + changed_when: false + failed_when: false + run_once: true + when: info_ceph_default_crush_rule.results|length > 0 diff --git a/roles/ceph-mon/tasks/openstack_config.yml b/roles/ceph-mon/tasks/openstack_config.yml index c4dc9b946..32e246966 100644 --- a/roles/ceph-mon/tasks/openstack_config.yml +++ b/roles/ceph-mon/tasks/openstack_config.yml @@ -1,6 +1,6 @@ --- - name: create openstack pool(s) - command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd pool create {{ item.name }} {{ item.pg_num }}" + command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd pool create {{ item.name }} {{ item.pg_num }} {{ item.rule_name }}" with_items: "{{ openstack_pools | unique }}" changed_when: false failed_when: false -- 2.47.3