From cb125fa4c83e0dcea87fe42f28909b7e307c4405 Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Mon, 20 May 2019 15:58:10 +0200 Subject: [PATCH] nfs: support internal Ganesha with external ceph cluster This commits allows to deploy an internal ganesha with an external ceph cluster. This requires to define `external_cluster_mon_ips` with a comma separated list of external monitors. Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1710358 Signed-off-by: Guillaume Abrioux (cherry picked from commit 6a6785b719d126cf54ebad8b2a22c97d90afd05e) --- group_vars/all.yml.sample | 2 +- group_vars/clients.yml.sample | 1 + group_vars/rhcs.yml.sample | 2 +- roles/ceph-client/defaults/main.yml | 2 + roles/ceph-client/tasks/create_users_keys.yml | 4 +- roles/ceph-config/templates/ceph.conf.j2 | 2 +- roles/ceph-defaults/defaults/main.yml | 2 +- roles/ceph-nfs/tasks/common.yml | 1 + roles/ceph-nfs/tasks/main.yml | 13 +++ .../tasks/pre_requisite_container.yml | 96 ++++++++++--------- .../tasks/pre_requisite_non_container.yml | 4 +- roles/ceph-nfs/tasks/start_nfs.yml | 42 ++++---- 12 files changed, 99 insertions(+), 72 deletions(-) diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample index 72e672f51..8c3dec90e 100644 --- a/group_vars/all.yml.sample +++ b/group_vars/all.yml.sample @@ -474,7 +474,7 @@ dummy: # Set this to true to enable File access via NFS. Requires an MDS role. #nfs_file_gw: false # Set this to true to enable Object access via NFS. Requires an RGW role. -#nfs_obj_gw: true +#nfs_obj_gw: "{{ False if groups.get(mon_group_name, []) | length == 0 else True }}" ############# diff --git a/group_vars/clients.yml.sample b/group_vars/clients.yml.sample index f1ae17b30..f1c8132a8 100644 --- a/group_vars/clients.yml.sample +++ b/group_vars/clients.yml.sample @@ -57,3 +57,4 @@ dummy: # - { name: client.test, caps: { mon: "allow r", osd: "allow class-read object_prefix rbd_children, allow rwx pool=test" }, mode: "{{ ceph_keyring_permissions }}" } # - { name: client.test2, caps: { mon: "allow r", osd: "allow class-read object_prefix rbd_children, allow rwx pool=test2" }, mode: "{{ ceph_keyring_permissions }}" } +#ceph_nfs_ceph_user: { name: client.rgw.nfs0, key: 'SECRET==', caps: { mon: "allow rw", osd: "allow rwx" }, mode: "{{ ceph_keyring_permissions }}" } diff --git a/group_vars/rhcs.yml.sample b/group_vars/rhcs.yml.sample index 9511571bd..4cd9c45b6 100644 --- a/group_vars/rhcs.yml.sample +++ b/group_vars/rhcs.yml.sample @@ -474,7 +474,7 @@ ceph_rhcs_version: 4 # Set this to true to enable File access via NFS. Requires an MDS role. #nfs_file_gw: false # Set this to true to enable Object access via NFS. Requires an RGW role. -#nfs_obj_gw: true +#nfs_obj_gw: "{{ False if groups.get(mon_group_name, []) | length == 0 else True }}" ############# diff --git a/roles/ceph-client/defaults/main.yml b/roles/ceph-client/defaults/main.yml index 6f664db82..d8e893b72 100644 --- a/roles/ceph-client/defaults/main.yml +++ b/roles/ceph-client/defaults/main.yml @@ -48,3 +48,5 @@ pools: keys: - { name: client.test, caps: { mon: "allow r", osd: "allow class-read object_prefix rbd_children, allow rwx pool=test" }, mode: "{{ ceph_keyring_permissions }}" } - { name: client.test2, caps: { mon: "allow r", osd: "allow class-read object_prefix rbd_children, allow rwx pool=test2" }, mode: "{{ ceph_keyring_permissions }}" } + +#ceph_nfs_ceph_user: { name: client.rgw.nfs0, key: 'SECRET==', caps: { mon: "allow rw", osd: "allow rwx" }, mode: "{{ ceph_keyring_permissions }}" } \ No newline at end of file diff --git a/roles/ceph-client/tasks/create_users_keys.yml b/roles/ceph-client/tasks/create_users_keys.yml index bd1389584..c8ddff3bf 100644 --- a/roles/ceph-client/tasks/create_users_keys.yml +++ b/roles/ceph-client/tasks/create_users_keys.yml @@ -45,7 +45,9 @@ environment: CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}" CEPH_CONTAINER_BINARY: "{{ container_binary }}" - with_items: "{{ keys }}" + with_items: + - "{{ keys }}" + - "{{ ceph_nfs_ceph_user | default([]) }}" delegate_to: "{{ delegated_node }}" when: - cephx diff --git a/roles/ceph-config/templates/ceph.conf.j2 b/roles/ceph-config/templates/ceph.conf.j2 index 52c3e9a6b..1e5821052 100644 --- a/roles/ceph-config/templates/ceph.conf.j2 +++ b/roles/ceph-config/templates/ceph.conf.j2 @@ -56,7 +56,7 @@ mon host = {% if nb_mon > 0 %} {%- endif %} {%- if not loop.last -%},{%- endif %} {%- endfor %} -{% elif nb_mon == 0 and inventory_hostname in groups.get(client_group_name, []) %} +{% elif nb_mon == 0 %} {{ external_cluster_mon_ips }} {% endif %} diff --git a/roles/ceph-defaults/defaults/main.yml b/roles/ceph-defaults/defaults/main.yml index 967f45385..03b0e175b 100644 --- a/roles/ceph-defaults/defaults/main.yml +++ b/roles/ceph-defaults/defaults/main.yml @@ -466,7 +466,7 @@ handler_health_mgr_check_delay: 10 # Set this to true to enable File access via NFS. Requires an MDS role. nfs_file_gw: false # Set this to true to enable Object access via NFS. Requires an RGW role. -nfs_obj_gw: true +nfs_obj_gw: "{{ False if groups.get(mon_group_name, []) | length == 0 else True }}" ############# diff --git a/roles/ceph-nfs/tasks/common.yml b/roles/ceph-nfs/tasks/common.yml index c086f8a98..6794ccd60 100644 --- a/roles/ceph-nfs/tasks/common.yml +++ b/roles/ceph-nfs/tasks/common.yml @@ -12,3 +12,4 @@ when: - cephx - item.copy_key|bool + - groups.get(mon_group_name, []) | length > 0 diff --git a/roles/ceph-nfs/tasks/main.yml b/roles/ceph-nfs/tasks/main.yml index e1c71978f..09766e68e 100644 --- a/roles/ceph-nfs/tasks/main.yml +++ b/roles/ceph-nfs/tasks/main.yml @@ -17,6 +17,7 @@ - name: include create_rgw_nfs_user.yml import_tasks: create_rgw_nfs_user.yml + when: groups.get(mon_group_name, []) | length > 0 # NOTE (leseb): workaround for issues with ganesha and librgw - name: include ganesha_selinux_fix.yml @@ -25,5 +26,17 @@ - not containerized_deployment - ansible_os_family == 'RedHat' +- name: copy rgw keyring when deploying internal ganesha with external ceph cluster + copy: + src: "/etc/ceph/{{ cluster }}.{{ ceph_nfs_ceph_user.name }}.keyring" + dest: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}/keyring" + mode: '0600' + owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}" + group: "{{ ceph_uid if containerized_deployment else 'ceph' }}" + remote_src: true + when: + - groups.get(mon_group_name, []) | length == 0 + - ceph_nfs_ceph_user is defined + - name: include start_nfs.yml import_tasks: start_nfs.yml diff --git a/roles/ceph-nfs/tasks/pre_requisite_container.yml b/roles/ceph-nfs/tasks/pre_requisite_container.yml index 5206c1285..ead1e19f6 100644 --- a/roles/ceph-nfs/tasks/pre_requisite_container.yml +++ b/roles/ceph-nfs/tasks/pre_requisite_container.yml @@ -1,54 +1,58 @@ --- -- name: set_fact admin_keyring - set_fact: - admin_keyring: - - "/etc/ceph/{{ cluster }}.client.admin.keyring" - when: copy_admin_key +- name: keyring related tasks + block: + - name: set_fact admin_keyring + set_fact: + admin_keyring: + - "/etc/ceph/{{ cluster }}.client.admin.keyring" + when: copy_admin_key -- name: set_fact ceph_config_keys - set_fact: - ceph_config_keys: - - /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring + - name: set_fact ceph_config_keys + set_fact: + ceph_config_keys: + - /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring -- name: merge ceph_config_keys and admin_keyring - set_fact: - ceph_config_keys: "{{ ceph_config_keys + admin_keyring }}" - when: copy_admin_key + - name: merge ceph_config_keys and admin_keyring + set_fact: + ceph_config_keys: "{{ ceph_config_keys + admin_keyring }}" + when: copy_admin_key -- name: stat for config and keys - stat: - path: "{{ fetch_directory }}/{{ fsid }}/{{ item }}" - delegate_to: localhost - with_items: "{{ ceph_config_keys }}" - changed_when: false - become: false - failed_when: false - check_mode: no - register: statconfig + - name: stat for config and keys + stat: + path: "{{ fetch_directory }}/{{ fsid }}/{{ item }}" + delegate_to: localhost + with_items: "{{ ceph_config_keys }}" + changed_when: false + become: false + failed_when: false + check_mode: no + register: statconfig -- name: try to fetch config and keys - copy: - src: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}" - dest: "{{ item.0 }}" - owner: "64045" - group: "64045" - mode: 0644 - changed_when: false - with_together: - - "{{ ceph_config_keys }}" - - "{{ statconfig.results }}" - when: item.1.stat.exists + - name: try to fetch config and keys + copy: + src: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}" + dest: "{{ item.0 }}" + owner: "64045" + group: "64045" + mode: 0644 + changed_when: false + with_together: + - "{{ ceph_config_keys }}" + - "{{ statconfig.results }}" + when: item.1.stat.exists + when: groups.get(mon_group_name, []) | length > 0 -- name: create dbus service file - become: true - copy: - src: "org.ganesha.nfsd.conf" - dest: /etc/dbus-1/system.d/org.ganesha.nfsd.conf - owner: "root" - group: "root" - mode: "0644" - when: ceph_nfs_dynamic_exports +- name: dbus related tasks + block: + - name: create dbus service file + become: true + copy: + src: "org.ganesha.nfsd.conf" + dest: /etc/dbus-1/system.d/org.ganesha.nfsd.conf + owner: "root" + group: "root" + mode: "0644" -- name: reload dbus configuration - command: "killall -SIGHUP dbus-daemon" + - name: reload dbus configuration + command: "killall -SIGHUP dbus-daemon" when: ceph_nfs_dynamic_exports diff --git a/roles/ceph-nfs/tasks/pre_requisite_non_container.yml b/roles/ceph-nfs/tasks/pre_requisite_non_container.yml index f88f67ffd..1b1868a2c 100644 --- a/roles/ceph-nfs/tasks/pre_requisite_non_container.yml +++ b/roles/ceph-nfs/tasks/pre_requisite_non_container.yml @@ -42,7 +42,9 @@ when: item.create|bool - name: cephx related tasks - when: cephx + when: + - cephx + - groups.get(mon_group_name, []) | length > 0 block: - name: copy bootstrap cephx keys copy: diff --git a/roles/ceph-nfs/tasks/start_nfs.yml b/roles/ceph-nfs/tasks/start_nfs.yml index 28a5ea255..0e7aa959a 100644 --- a/roles/ceph-nfs/tasks/start_nfs.yml +++ b/roles/ceph-nfs/tasks/start_nfs.yml @@ -1,26 +1,28 @@ --- -- name: set_fact container_exec_cmd_nfs - set_fact: - container_exec_cmd_nfs: "{{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}" - when: containerized_deployment +- block: + - name: set_fact container_exec_cmd_nfs + set_fact: + container_exec_cmd_nfs: "{{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}" + when: containerized_deployment -- name: check if rados index object exists - shell: "{{ container_exec_cmd_nfs | default('') }} rados -p {{ cephfs_data }} --cluster {{ cluster }} ls|grep {{ ceph_nfs_rados_export_index }}" - changed_when: false - failed_when: false - register: rados_index_exists - check_mode: no - when: ceph_nfs_rados_backend - delegate_to: "{{ groups[mon_group_name][0] }}" - run_once: true + - name: check if rados index object exists + shell: "{{ container_exec_cmd_nfs | default('') }} rados -p {{ cephfs_data }} --cluster {{ cluster }} ls|grep {{ ceph_nfs_rados_export_index }}" + changed_when: false + failed_when: false + register: rados_index_exists + check_mode: no + when: ceph_nfs_rados_backend + delegate_to: "{{ groups[mon_group_name][0] }}" + run_once: true -- name: create an empty rados index object - command: "{{ container_exec_cmd_nfs | default('') }} rados -p {{ cephfs_data }} --cluster {{ cluster }} put {{ ceph_nfs_rados_export_index }} /dev/null" - when: - - ceph_nfs_rados_backend - - rados_index_exists.rc != 0 - delegate_to: "{{ groups[mon_group_name][0] }}" - run_once: true + - name: create an empty rados index object + command: "{{ container_exec_cmd_nfs | default('') }} rados -p {{ cephfs_data }} --cluster {{ cluster }} put {{ ceph_nfs_rados_export_index }} /dev/null" + when: + - ceph_nfs_rados_backend + - rados_index_exists.rc != 0 + delegate_to: "{{ groups[mon_group_name][0] }}" + run_once: true + when: groups.get(mon_group_name, []) | length > 0 - name: create /etc/ganesha file: -- 2.39.5