ceph-nfs: allow disabling ganesha caching

Add a variable, ceph_nfs_disable_caching, that if set to true
disables ganesha's directory and attribute caching as much as
possible.

Also, disable caching done by ganesha, when 'nfs_file_gw'
variable is true, i.e., when Ganesha is used as CephFS's gateway.
This is the recommended Ganesha setting as libcephfs already caches
information. And doing so helps avoid cache incoherency issues
especially with clustered ganesha over CephFS.

Fixes: https://tracker.ceph.com/issues/23393
Signed-off-by: Ramana Raja <rraja@redhat.com>

diff --git a/roles/ceph-nfs/defaults/main.yml b/roles/ceph-nfs/defaults/main.yml
index 84723d1dca37456f167f4610f0096dd8a1127648..19dbb799a1671e64e13e399459d99cc25934159a 100644 (file)
--- a/roles/ceph-nfs/defaults/main.yml
+++ b/roles/ceph-nfs/defaults/main.yml
@@ -27,6 +27,10 @@ ceph_nfs_enable_service: true
 #######################
 # These are currently in ceph-common defaults because nfs_obj_gw shared with ceph-rgw
 # Enable NFS File access
+# If set to true, then ganesha is set up to export the root of the
+# Ceph filesystem, and ganesha's attribute and directory caching is disabled
+# as much as possible since libcephfs clients also caches the same
+# information.
 nfs_file_gw: false
 # Enable NFS Object access
 nfs_obj_gw: true
@@ -49,6 +53,14 @@ ceph_nfs_rados_export_index: "ganesha-export-index"
 # this bug: https://github.com/nfs-ganesha/nfs-ganesha/issues/217)
 # ceph_nfs_bind_addr: 0.0.0.0
 
+# If set to true, then ganesha's attribute and directory caching is disabled
+# as much as possible. Currently, ganesha caches by default.
+# When using ganesha as CephFS's gateway, it is recommended to turn off
+# ganesha's caching as the libcephfs clients also cache the same information.
+# Note: Irrespective of this option's setting, ganesha's caching is disabled
+# when setting 'nfs_file_gw' option as true.
+ceph_nfs_disable_caching: false
+
 ####################
 # FSAL Ceph Config #
 ####################

diff --git a/roles/ceph-nfs/templates/ganesha.conf.j2 b/roles/ceph-nfs/templates/ganesha.conf.j2
index 8e05f793debf2adf02066899feda695a82f03f4e..464e959909fe295eee4d7e834203f265f44edf43 100644 (file)
--- a/roles/ceph-nfs/templates/ganesha.conf.j2
+++ b/roles/ceph-nfs/templates/ganesha.conf.j2
@@ -12,6 +12,18 @@ NFS_Core_Param
 {% endif %}
 }
 
+{% if ceph_nfs_disable_caching or nfs_file_gw %}
+CACHEINODE {
+       Dir_Max = 1;
+       Dir_Chunk = 0;
+
+       Cache_FDs = false;
+
+       NParts = 1;
+       Cache_Size = 1;
+}
+{% endif %}
+
 {% if ceph_nfs_rados_backend %}
 RADOS_URLS {
    ceph_conf = '/etc/ceph/{{ cluster }}.conf';
@@ -46,6 +58,8 @@ EXPORT
 
        SecType = sys,krb5,krb5i,krb5p;
 
+       Attr_Expiration_Time = 0;
+
        FSAL {
                Name = CEPH;
                User_Id = "{{ ceph_nfs_ceph_user }}";