From 788b75efec287a555dcb9617942f6aa19f3b32ec Mon Sep 17 00:00:00 2001 From: =?utf8?q?S=C3=A9bastien=20Han?= Date: Tue, 19 Jul 2016 17:55:57 +0200 Subject: [PATCH] ceph-osd: re-arrange osd scenario numbers MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Signed-off-by: Sébastien Han --- group_vars/osds.sample | 25 ++++++++------ .../tasks/checks/check_mandatory_vars.yml | 15 +++++--- roles/ceph-osd/defaults/main.yml | 25 ++++++++------ roles/ceph-osd/tasks/activate_osds.yml | 34 ++++++++++++++++++- roles/ceph-osd/tasks/main.yml | 9 +++-- roles/ceph-osd/tasks/pre_requisite.yml | 6 ++++ .../scenarios/dmcrypt-dedicated-journal.yml | 29 ++++++++++++++++ ...ypt.yml => dmcrypt-journal-colocation.yml} | 9 +++-- 8 files changed, 117 insertions(+), 35 deletions(-) create mode 100644 roles/ceph-osd/tasks/scenarios/dmcrypt-dedicated-journal.yml rename roles/ceph-osd/tasks/scenarios/{dmcrypt.yml => dmcrypt-journal-colocation.yml} (85%) diff --git a/group_vars/osds.sample b/group_vars/osds.sample index 458300e20..b2241ba04 100644 --- a/group_vars/osds.sample +++ b/group_vars/osds.sample @@ -85,13 +85,7 @@ dummy: #journal_collocation: false -# II. Second scenario: single journal device for N OSDs -# Use 'true' to enable this scenario - -# deprecated, please use scenario III with a single raw_journal_device - - -# III. Third scenario: N journal devices for N OSDs +# II. N journal devices for N OSDs # Use 'true' to enable this scenario # # In the following example: @@ -109,7 +103,7 @@ dummy: # - /dev/sdc -# IV. Fourth scenario: use directory instead of disk for OSDs +# III. Use directory instead of disk for OSDs # Use 'true' to enable this scenario #osd_directory: false @@ -118,15 +112,24 @@ dummy: # - /var/lib/ceph/osd/mydir2 -# V. Fith scenario: this will partition disks for BlueStore +# IV. This will partition disks for BlueStore # Use 'true' to enable this scenario #bluestore: false -# VI. Sixth scenario: encrypt osd data and/or journal devices with dm-crypt. +# V. Encrypt osd data and/or journal devices with dm-crypt. +# Keys are stored into the monitors k/v store +# Use 'true' to enable this scenario +# Both journal and data are stored on the same dm-crypt encrypted device +#dmcrypt_journal_colocation: false + + +# VI. Encrypt osd data and/or journal devices with dm-crypt. # Keys are stored into the monitors k/v store # Use 'true' to enable this scenario -#dmcrypt: false +# Journal and osd data are separated, each with their own dm-crypt device +# You must use raw_journal_devices and set your journal devices +#dmcrypt_dedicated_journal: false ########## diff --git a/roles/ceph-common/tasks/checks/check_mandatory_vars.yml b/roles/ceph-common/tasks/checks/check_mandatory_vars.yml index 43be97554..8f7cfa1af 100644 --- a/roles/ceph-common/tasks/checks/check_mandatory_vars.yml +++ b/roles/ceph-common/tasks/checks/check_mandatory_vars.yml @@ -72,7 +72,8 @@ - not raw_multi_journal - not osd_directory - not bluestore - - not dmcrypt + - not dmcrypt_journal_colocation + - not dmcrypt_dedicated_journal - name: verify only one osd scenario was chosen fail: @@ -86,10 +87,14 @@ or (raw_multi_journal and osd_directory) or (raw_multi_journal and bluestore) or (osd_directory and bluestore) - or (dmcrypt and journal_collocation) - or (dmcrypt and raw_multi_journal) - or (dmcrypt and osd_directory) - or (dmcrypt and bluestore) + or (dmcrypt_journal_colocation and journal_collocation) + or (dmcrypt_journal_colocation and raw_multi_journal) + or (dmcrypt_journal_colocation and osd_directory) + or (dmcrypt_journal_colocation and bluestore) + or (dmcrypt_dedicated_journal and journal_collocation) + or (dmcrypt_dedicated_journal and raw_multi_journal) + or (dmcrypt_dedicated_journal and osd_directory) + or (dmcrypt_dedicated_journal and bluestore) - name: verify devices have been provided fail: diff --git a/roles/ceph-osd/defaults/main.yml b/roles/ceph-osd/defaults/main.yml index b53f7e8cf..f2927b56e 100644 --- a/roles/ceph-osd/defaults/main.yml +++ b/roles/ceph-osd/defaults/main.yml @@ -77,13 +77,7 @@ osd_auto_discovery: false journal_collocation: false -# II. Second scenario: single journal device for N OSDs -# Use 'true' to enable this scenario - -# deprecated, please use scenario III with a single raw_journal_device - - -# III. Third scenario: N journal devices for N OSDs +# II. N journal devices for N OSDs # Use 'true' to enable this scenario # # In the following example: @@ -101,7 +95,7 @@ raw_multi_journal: false # - /dev/sdc -# IV. Fourth scenario: use directory instead of disk for OSDs +# III. Use directory instead of disk for OSDs # Use 'true' to enable this scenario osd_directory: false @@ -110,15 +104,24 @@ osd_directory: false # - /var/lib/ceph/osd/mydir2 -# V. Fith scenario: this will partition disks for BlueStore +# IV. This will partition disks for BlueStore # Use 'true' to enable this scenario bluestore: false -# VI. Sixth scenario: encrypt osd data and/or journal devices with dm-crypt. +# V. Encrypt osd data and/or journal devices with dm-crypt. +# Keys are stored into the monitors k/v store +# Use 'true' to enable this scenario +# Both journal and data are stored on the same dm-crypt encrypted device +dmcrypt_journal_colocation: false + + +# VI. Encrypt osd data and/or journal devices with dm-crypt. # Keys are stored into the monitors k/v store # Use 'true' to enable this scenario -dmcrypt: false +# Journal and osd data are separated, each with their own dm-crypt device +# You must use raw_journal_devices and set your journal devices +dmcrypt_dedicated_journal: false ########## diff --git a/roles/ceph-osd/tasks/activate_osds.yml b/roles/ceph-osd/tasks/activate_osds.yml index 34dde8f5e..4219e005f 100644 --- a/roles/ceph-osd/tasks/activate_osds.yml +++ b/roles/ceph-osd/tasks/activate_osds.yml @@ -25,13 +25,45 @@ - not item.0.get("skipped") - item.0.get("rc", 0) != 0 - not osd_auto_discovery + - raw_multi_journal + +- name: automatically activate osd disk(s) without partitions (dmcrypt) + command: ceph-disk activate --dmcrypt "/dev/{{ item.key }}" + ignore_errors: true + with_dict: ansible_devices + when: + - ansible_devices is defined + - item.value.removable == "0" + - item.value.partitions|count == 0 + - osd_auto_discovery + - dmcrypt_journal_colocation + +- name: activate osd(s) when device is a disk (dmcrypt) + command: ceph-disk activate --dmcrypt {{ item.1 | regex_replace('^(\/dev\/cciss\/c[0-9]{1}d[0-9]{1})$', '\\1p') }}1 + with_together: + - combined_ispartition_results.results + - devices + changed_when: false + failed_when: false + register: activate_osd_disk_dmcrypt + when: + - not item.0.get("skipped") + - item.0.get("rc", 0) != 0 + - not osd_auto_discovery + - dmcrypt_dedicated_journal + +# NOTE (leseb): we must do this because of +# https://github.com/ansible/ansible/issues/4297 +- name: combine ispartition results + set_fact: + combined_activate_osd_disk_results: "{{ activate_osd_disk if not dmcrypt_journal_colocation else activate_osd_disk_dmcrypt }}" - name: fail if ceph-disk cannot create an OSD fail: msg: "ceph-disk failed to create an OSD" when: " 'ceph-disk: Error: ceph osd create failed' in item.get('stderr', '') " - with_items: "{{activate_osd_disk.results}}" + with_items: "{{combined_activate_osd_disk_results.results}}" # NOTE (leseb): this task is for partitions because we don't explicitly use a partition. - name: activate osd(s) when device is a partition diff --git a/roles/ceph-osd/tasks/main.yml b/roles/ceph-osd/tasks/main.yml index 76bc44cdb..d965a558c 100644 --- a/roles/ceph-osd/tasks/main.yml +++ b/roles/ceph-osd/tasks/main.yml @@ -22,9 +22,14 @@ - osd_objectstore == 'bluestore' - not osd_containerized_deployment -- include: ./scenarios/dmcrypt.yml +- include: ./scenarios/dmcrypt-journal-colocation.yml when: - - dmcrypt + - dmcrypt_journal_colocation + - not osd_containerized_deployment + +- include: ./scenarios/dmcrypt-dedicated-journal.yml + when: + - dmcrypt_dedicated_journal - not osd_containerized_deployment - include: ./docker/main.yml diff --git a/roles/ceph-osd/tasks/pre_requisite.yml b/roles/ceph-osd/tasks/pre_requisite.yml index 64430f0f8..9e2afda17 100644 --- a/roles/ceph-osd/tasks/pre_requisite.yml +++ b/roles/ceph-osd/tasks/pre_requisite.yml @@ -31,6 +31,12 @@ when: cephx +- name: copy ceph admin key when using dmcrypt + set_fact: + copy_admin_key: true + when: + - dmcrypt_journal_colocation or dmcrypt_dedicated_journal + - name: copy osd bootstrap key copy: src: "{{ fetch_directory }}/{{ fsid }}{{ item.name }}" diff --git a/roles/ceph-osd/tasks/scenarios/dmcrypt-dedicated-journal.yml b/roles/ceph-osd/tasks/scenarios/dmcrypt-dedicated-journal.yml new file mode 100644 index 000000000..effc39d98 --- /dev/null +++ b/roles/ceph-osd/tasks/scenarios/dmcrypt-dedicated-journal.yml @@ -0,0 +1,29 @@ +--- +## SCENARIO 6: DMCRYPT N JOURNAL DEVICES FOR N OSDS + +- include: ../check_devices.yml + +# NOTE (leseb): the prepare process must be parallelized somehow... +# if you have 64 disks with 4TB each, this will take a while +# since Ansible will sequential process the loop + +# NOTE (alahouze): if the device is a partition, the parted command below has +# failed, this is why we check if the device is a partition too. +- name: prepare osd disk(s) + command: "ceph-disk prepare --dmcrypt --cluster {{ cluster }} {{ item.2 }} {{ item.3 }}" + with_together: + - combined_parted_results.results + - combined_ispartition_results.results + - devices + - raw_journal_devices + changed_when: false + when: + - not item.0.get("skipped") + - not item.1.get("skipped") + - item.0.get("rc", 0) != 0 + - item.1.get("rc", 0) != 0 + - raw_multi_journal + - not osd_auto_discovery + - dmcrypt_dedicated_journal + +- include: ../activate_osds.yml diff --git a/roles/ceph-osd/tasks/scenarios/dmcrypt.yml b/roles/ceph-osd/tasks/scenarios/dmcrypt-journal-colocation.yml similarity index 85% rename from roles/ceph-osd/tasks/scenarios/dmcrypt.yml rename to roles/ceph-osd/tasks/scenarios/dmcrypt-journal-colocation.yml index 835b7e4f5..534fd3c57 100644 --- a/roles/ceph-osd/tasks/scenarios/dmcrypt.yml +++ b/roles/ceph-osd/tasks/scenarios/dmcrypt-journal-colocation.yml @@ -9,18 +9,17 @@ # NOTE (alahouze): if the device is a partition, the parted command below has # failed, this is why we check if the device is a partition too. -- name: automatic prepare osd disk(s) without partitions +- name: automatic prepare osd disk(s) without partitions (dmcrypt) command: ceph-disk prepare --dmcrypt --cluster "{{ cluster }}" "/dev/{{ item.key }}" - register: prepared_osds with_dict: ansible_devices when: - ansible_devices is defined - item.value.removable == "0" - item.value.partitions|count == 0 - - bluestore + - dmcrypt_journal_colocation - osd_auto_discovery -- name: manually prepare osd disk(s) +- name: manually prepare osd disk(s) (dmcrypt) command: ceph-disk prepare --dmcrypt --cluster "{{ cluster }}" "{{ item.2 }}" with_together: - combined_parted_results.results @@ -31,7 +30,7 @@ - not item.1.get("skipped") - item.0.get("rc", 0) != 0 - item.1.get("rc", 0) != 0 - - bluestore + - dmcrypt_journal_colocation - not osd_auto_discovery - include: ../activate_osds.yml -- 2.39.5