Create OpenStack pools with crush rule

Add an extra variable to the openstack pools, which creates them with
defined rules. This will allow to place different pools on e.g.
different type of disks.

This commit will also set a new default rule when defined and move
the rbd pool to the new rule.

diff --git a/group_vars/mons.yml.sample b/group_vars/mons.yml.sample
index 8244d650e1f5cfd300cac6eb8d3d9dd9ca7c8e62..688d01f57771289f940a761a5e807535d0c6cc13 100644 (file)
--- a/group_vars/mons.yml.sample
+++ b/group_vars/mons.yml.sample
@@ -44,6 +44,27 @@ dummy:
 # Enable debugging for Calamari
 #calamari_debug: false
 
+
+#############
+# crush rules
+#############
+#crush_rule_config: false
+
+#crush_rule_hdd:
+#  name: HDD
+#  root: HDD
+#  type: rack
+#  default: true
+
+#crush_rule_ssd:
+#  name: SSD
+#  root: SSD
+#  type: host
+#  default: false
+
+#crush_rules:
+#  - "{{ crush_rule_hdd }}"
+#  - "{{ crush_rule_ssd }}"
 #############
 # OPENSTACK #
 #############
@@ -52,18 +73,23 @@ dummy:
 #openstack_glance_pool:
 #  name: images
 #  pg_num: "{{ osd_pool_default_pg_num }}"
+#  rule_name: ""
 #openstack_cinder_pool:
 #  name: volumes
 #  pg_num: "{{ osd_pool_default_pg_num }}"
+#  rule_name: ""
 #openstack_nova_pool:
 #  name: vms
 #  pg_num: "{{ osd_pool_default_pg_num }}"
+#  rule_name: ""
 #openstack_cinder_backup_pool:
 #  name: backups
 #  pg_num: "{{ osd_pool_default_pg_num }}"
+#  rule_name: ""
 #openstack_gnocchi_pool:
 #  name: metrics
 #  pg_num: "{{ osd_pool_default_pg_num }}"
+#  rule_name: ""
 
 #openstack_pools:
 #  - "{{ openstack_glance_pool }}"

diff --git a/group_vars/rgws.yml.sample b/group_vars/rgws.yml.sample
index 22e2eeb043e9aff74dc7e6590cdeb17550bf976c..46c9150ea63924c30cf0b09ad1a187bd0fe98413 100644 (file)
--- a/group_vars/rgws.yml.sample
+++ b/group_vars/rgws.yml.sample
@@ -33,7 +33,7 @@ dummy:
 # important to split them into shards. We suggest about 100K
 # objects per shard as a conservative maximum.
 #rgw_override_bucket_index_max_shards: 16
-# 
+ 
 # Consider setting a quota on buckets so that exceeding this
 # limit will require admin intervention.
 #rgw_bucket_default_quota_max_objects: 1638400 # i.e., 100K * 16

diff --git a/roles/ceph-mon/defaults/main.yml b/roles/ceph-mon/defaults/main.yml
index 629b8b9248d8354e340d10afb92a8f56d3c7ba94..73bfc3241bb9d30938c2df5eb12ef0eb27e73005 100644 (file)
--- a/roles/ceph-mon/defaults/main.yml
+++ b/roles/ceph-mon/defaults/main.yml
@@ -36,6 +36,27 @@ calamari: false
 # Enable debugging for Calamari
 calamari_debug: false
 
+
+#############
+# crush rules
+#############
+crush_rule_config: false
+
+crush_rule_hdd:
+  name: HDD
+  root: HDD
+  type: rack
+  default: true
+
+crush_rule_ssd:
+  name: SSD
+  root: SSD
+  type: host
+  default: false
+
+crush_rules:
+  - "{{ crush_rule_hdd }}"
+  - "{{ crush_rule_ssd }}"
 #############
 # OPENSTACK #
 #############
@@ -44,18 +65,23 @@ openstack_config: false
 openstack_glance_pool:
   name: images
   pg_num: "{{ osd_pool_default_pg_num }}"
+  rule_name: ""
 openstack_cinder_pool:
   name: volumes
   pg_num: "{{ osd_pool_default_pg_num }}"
+  rule_name: ""
 openstack_nova_pool:
   name: vms
   pg_num: "{{ osd_pool_default_pg_num }}"
+  rule_name: ""
 openstack_cinder_backup_pool:
   name: backups
   pg_num: "{{ osd_pool_default_pg_num }}"
+  rule_name: ""
 openstack_gnocchi_pool:
   name: metrics
   pg_num: "{{ osd_pool_default_pg_num }}"
+  rule_name: ""
 
 openstack_pools:
   - "{{ openstack_glance_pool }}"

diff --git a/roles/ceph-mon/tasks/ceph_keys.yml b/roles/ceph-mon/tasks/ceph_keys.yml
index 84f42477a7094fea4476882bf934938bc274de8e..256426f852683cb82eb338d9b810bd0139c9770a 100644 (file)
--- a/roles/ceph-mon/tasks/ceph_keys.yml
+++ b/roles/ceph-mon/tasks/ceph_keys.yml
@@ -58,6 +58,10 @@
 
 - include: set_osd_pool_default_pg_num.yml
 
+- include: crush_rules.yml
+  when:
+    - crush_rule_config
+
 - name: test if rbd exists
   shell: |
     ceph --cluster {{ cluster }} osd pool ls | grep -sq rbd

diff --git a/roles/ceph-mon/tasks/crush_rules.yml b/roles/ceph-mon/tasks/crush_rules.yml
new file mode 100644 (file)
index 0000000..2c5d3c5
--- /dev/null
+++ b/roles/ceph-mon/tasks/crush_rules.yml
@@ -0,0 +1,48 @@
+---
+- name: create roots needed for configured crush rules
+  command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd crush add-bucket {{ item.root }} root"
+  with_items: "{{ crush_rules | unique }}"
+  changed_when: false
+  failed_when: false
+  run_once: true
+
+- name: create configured crush rules
+  command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd crush rule create-simple {{ item.name }} {{ item.root }} {{ item.type }}"
+  with_items: "{{ crush_rules | unique }}"
+  changed_when: false
+  failed_when: false
+  run_once: true
+
+- name: get id for new default crush rule
+  command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd -f json crush rule dump {{ item.name }}"
+  register: info_ceph_default_crush_rule
+  changed_when: false
+  failed_when: false
+  with_items: "{{ crush_rules }}"
+  when: item.default
+
+- name: set crush rule info as fact
+  set_fact:
+    info_ceph_default_crush_rule_yaml: "{{ info_ceph_default_crush_rule.results[0].stdout|from_json() }}"
+  when: info_ceph_default_crush_rule.results|length > 0
+
+- name: insert new default crush rule into daemon to prevent restart
+  command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} daemon mon.{{ monitor_name }} config set osd_pool_default_crush_replicated_ruleset {{ info_ceph_default_crush_rule_yaml.rule_id }}"
+  changed_when: false
+  failed_when: false
+  when: info_ceph_default_crush_rule.results|length > 0
+
+- name: add new default crush rule to ceph.conf
+  ini_file:
+    dest: "/etc/ceph/{{ cluster }}.conf"
+    section: "global"
+    option: "osd pool default crush replicated ruleset"
+    value: "{{ info_ceph_default_crush_rule_yaml.rule_id }}"
+  when: info_ceph_default_crush_rule.results|length > 0
+
+- name: move rbd pool to new default root
+  command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd pool set rbd crush_ruleset {{ info_ceph_default_crush_rule_yaml.rule_id }}"
+  changed_when: false
+  failed_when: false
+  run_once: true
+  when: info_ceph_default_crush_rule.results|length > 0

diff --git a/roles/ceph-mon/tasks/openstack_config.yml b/roles/ceph-mon/tasks/openstack_config.yml
index c4dc9b946929a83713f8f602bae9d188664ce7e7..32e246966aa1af08a422052d69100f80c6d7a549 100644 (file)
--- a/roles/ceph-mon/tasks/openstack_config.yml
+++ b/roles/ceph-mon/tasks/openstack_config.yml
@@ -1,6 +1,6 @@
 ---
 - name: create openstack pool(s)
-  command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd pool create {{ item.name }} {{ item.pg_num }}"
+  command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd pool create {{ item.name }} {{ item.pg_num }} {{ item.rule_name }}"
   with_items: "{{ openstack_pools | unique }}"
   changed_when: false
   failed_when: false