From 52efe92a879a75248537803a57c345f528190c94 Mon Sep 17 00:00:00 2001 From: Ali Maredia Date: Sun, 10 Sep 2017 23:34:36 -0400 Subject: [PATCH] nfs: configure RGW FSAL to start up correctly - Add RGW keyring to nfs node - Add RGW section to ganesha.conf - Add RGW section to ceph.conf onf nfs node Signed-off-by: Ali Maredia --- group_vars/nfss.yml.sample | 4 +- roles/ceph-config/templates/ceph.conf.j2 | 36 ++++++++++++++ roles/ceph-nfs/defaults/main.yml | 4 +- roles/ceph-nfs/tasks/pre_requisite.yml | 47 +++++++++++++++++++ roles/ceph-nfs/templates/ganesha.conf.j2 | 7 +++ .../functional/tests/nfs/test_nfs_ganesha.py | 9 ++++ 6 files changed, 105 insertions(+), 2 deletions(-) diff --git a/group_vars/nfss.yml.sample b/group_vars/nfss.yml.sample index 556501c2d..a8151cb2b 100644 --- a/group_vars/nfss.yml.sample +++ b/group_vars/nfss.yml.sample @@ -35,7 +35,7 @@ dummy: #ceph_nfs_pseudo_path: "/cephfile" #ceph_nfs_protocols: "3,4" #ceph_nfs_access_type: "RW" -#ceph_nfs_log_file: "/var/log/ganesha.log" +#ceph_nfs_log_file: "/var/log/ganesha/ganesha.log" #################### # FSAL Ceph Config # @@ -57,6 +57,7 @@ dummy: # they must be configered. #ceph_nfs_rgw_access_key: "QFAMEDSJP5DEKJO0DDXY" #ceph_nfs_rgw_secret_key: "iaSFLDVvDdQt6lkNzHyW4fPLZugBAI1g17LO0+87[MAC[M#C" +#rgw_client_name: client.rgw.{{ ansible_hostname }} ################### # CONFIG OVERRIDE # @@ -74,6 +75,7 @@ dummy: # #ganesha_ceph_export_overrides: #ganesha_rgw_export_overrides: +#ganesha_rgw_section_overrides: #ganesha_log_overrides: #ganesha_conf_overrides: | # CACHEINODE { diff --git a/roles/ceph-config/templates/ceph.conf.j2 b/roles/ceph-config/templates/ceph.conf.j2 index 1282580cd..9cf939fa0 100644 --- a/roles/ceph-config/templates/ceph.conf.j2 +++ b/roles/ceph-config/templates/ceph.conf.j2 @@ -178,6 +178,42 @@ rgw data = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ hostvars[host]['ansible_ho {% endif %} {% endif %} +{% if groups[nfs_group_name] is defined %} +{% if nfs_group_name in group_names %} +{% for host in groups[nfs_group_name] %} +{% if nfs_obj_gw %} +{% if hostvars[host]['ansible_hostname'] is defined %} +[client.rgw.{{ hostvars[host]['ansible_hostname'] }}] +host = {{ hostvars[host]['ansible_hostname'] }} +keyring = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ hostvars[host]['ansible_hostname'] }}/keyring +log file = /var/log/ceph/{{ cluster }}-rgw-{{ hostvars[host]['ansible_hostname'] }}.log +rgw data = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ hostvars[host]['ansible_hostname'] }} +{% if radosgw_address_block | length > 0 %} + {% if ip_version == 'ipv4' -%} + rgw frontends = civetweb port={{ hostvars[host]['ansible_all_' + ip_version + '_addresses'] | ipaddr(radosgw_address_block) | first }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }} + {%- elif ip_version == 'ipv6' -%} + rgw frontends = civetweb port=[{{ hostvars[host]['ansible_all_' + ip_version + '_addresses'] | ipaddr(radosgw_address_block) | first }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }} + {%- endif %} +{% elif hostvars[host]['radosgw_address'] is defined and hostvars[host]['radosgw_address'] != '0.0.0.0' -%} + {% if ip_version == 'ipv4' -%} + rgw frontends = civetweb port={{ hostvars[host]['radosgw_address'] }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }} + {%- elif ip_version == 'ipv6' -%} + rgw frontends = civetweb port=[{{ hostvars[host]['radosgw_address'] }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }} + {% endif %} +{%- else -%} + {% set interface = ["ansible_",radosgw_interface]|join %} + {% if ip_version == 'ipv6' -%} + rgw frontends = civetweb port=[{{ hostvars[host][interface][ip_version][0]['address'] }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }} + {%- elif ip_version == 'ipv4' -%} + rgw frontends = civetweb port={{ hostvars[host][interface][ip_version]['address'] }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }} + {% endif %} +{%- endif %} +{% endif %} +{% endif %} +{% endfor %} +{% endif %} +{% endif %} + {% if groups[restapi_group_name] is defined %} {% if restapi_group_name in group_names %} [client.restapi] diff --git a/roles/ceph-nfs/defaults/main.yml b/roles/ceph-nfs/defaults/main.yml index 2e4c8bfff..d311a28e1 100644 --- a/roles/ceph-nfs/defaults/main.yml +++ b/roles/ceph-nfs/defaults/main.yml @@ -27,7 +27,7 @@ ceph_nfs_export_id: 20134 ceph_nfs_pseudo_path: "/cephfile" ceph_nfs_protocols: "3,4" ceph_nfs_access_type: "RW" -ceph_nfs_log_file: "/var/log/ganesha.log" +ceph_nfs_log_file: "/var/log/ganesha/ganesha.log" #################### # FSAL Ceph Config # @@ -49,6 +49,7 @@ ceph_nfs_rgw_user: "cephnfs" # they must be configered. #ceph_nfs_rgw_access_key: "QFAMEDSJP5DEKJO0DDXY" #ceph_nfs_rgw_secret_key: "iaSFLDVvDdQt6lkNzHyW4fPLZugBAI1g17LO0+87[MAC[M#C" +rgw_client_name: client.rgw.{{ ansible_hostname }} ################### # CONFIG OVERRIDE # @@ -66,6 +67,7 @@ ceph_nfs_rgw_user: "cephnfs" # #ganesha_ceph_export_overrides: #ganesha_rgw_export_overrides: +#ganesha_rgw_section_overrides: #ganesha_log_overrides: #ganesha_conf_overrides: | # CACHEINODE { diff --git a/roles/ceph-nfs/tasks/pre_requisite.yml b/roles/ceph-nfs/tasks/pre_requisite.yml index 32570ce5b..665345097 100644 --- a/roles/ceph-nfs/tasks/pre_requisite.yml +++ b/roles/ceph-nfs/tasks/pre_requisite.yml @@ -1,4 +1,51 @@ --- +- name: create rados gateway directories + file: + path: "{{ item }}" + state: directory + owner: "ceph" + group: "ceph" + mode: "0755" + with_items: + - /var/lib/ceph/bootstrap-rgw + - /var/lib/ceph/radosgw + - /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }} + - "{{ rbd_client_admin_socket_path }}" + when: + - nfs_obj_gw + +- name: copy rados gateway bootstrap key + copy: + src: "{{ fetch_directory }}/{{ fsid }}{{ item.name }}" + dest: "{{ item.name }}" + owner: "ceph" + group: "ceph" + mode: "0600" + with_items: + - { name: "/var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring", copy_key: true } + - { name: "/etc/ceph/{{ cluster }}.client.admin.keyring", copy_key: "{{ copy_admin_key }}" } + when: + - nfs_obj_gw + - cephx + - item.copy_key|bool + +- name: create rados gateway keyring + command: ceph --cluster {{ cluster }} --name client.bootstrap-rgw --keyring /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring auth get-or-create client.rgw.{{ ansible_hostname }} osd 'allow rwx' mon 'allow rw' -o /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}/keyring + args: + creates: /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}/keyring + changed_when: false + when: + - nfs_obj_gw + - cephx + +- name: set rados gateway key permissions + file: + path: /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}/keyring + owner: "ceph" + group: "ceph" + mode: "0600" + when: cephx + - name: create rgw nfs user command: radosgw-admin --cluster {{ cluster }} user create --uid={{ ceph_nfs_rgw_user }} --display-name="RGW NFS User" register: rgwuser diff --git a/roles/ceph-nfs/templates/ganesha.conf.j2 b/roles/ceph-nfs/templates/ganesha.conf.j2 index dee5a3e83..2c88c9d45 100644 --- a/roles/ceph-nfs/templates/ganesha.conf.j2 +++ b/roles/ceph-nfs/templates/ganesha.conf.j2 @@ -52,6 +52,13 @@ EXPORT {{ ganesha_rgw_export_overrides | default(None) }} } + +RGW { + ceph_conf = "/etc/ceph/{{ cluster }}.conf"; + cluster = "{{ cluster }}"; + name = "{{ rgw_client_name }}"; + {{ ganesha_rgw_section_overrides | default(None) }} +} {% endif %} LOG { diff --git a/tests/functional/tests/nfs/test_nfs_ganesha.py b/tests/functional/tests/nfs/test_nfs_ganesha.py index c9b1310a8..e943a3338 100644 --- a/tests/functional/tests/nfs/test_nfs_ganesha.py +++ b/tests/functional/tests/nfs/test_nfs_ganesha.py @@ -21,3 +21,12 @@ class TestNFSs(object): @pytest.mark.no_docker def test_nfs_config_override(self, node, host): assert host.file("/etc/ganesha/ganesha.conf").contains("Entries_HWMark") + + @pytest.mark.no_docker + def test_nfs_rgw_fsal_export(self, node, host): + if(host.mount_point("/mnt").exists): + cmd = host.run("sudo umount /mnt") + assert cmd.rc == 0 + cmd = host.run("sudo mount.nfs localhost:/ceph /mnt/") + assert cmd.rc == 0 + assert host.mount_point("/mnt").exists -- 2.39.5