]> git.apps.os.sepia.ceph.com Git - ceph-ci.git/commitdiff
qa/cephfs: introduce nfs-ganesha tests
authorVenky Shankar <vshankar@redhat.com>
Fri, 1 Aug 2025 10:39:46 +0000 (10:39 +0000)
committerVenky Shankar <vshankar@redhat.com>
Wed, 15 Oct 2025 04:53:11 +0000 (04:53 +0000)
Fixes: http://tracker.ceph.com/issues/73172
Signed-off-by: Venky Shankar <vshankar@redhat.com>
36 files changed:
qa/suites/fs/nfs-ganesha/% [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/.qa [new symlink]
qa/suites/fs/nfs-ganesha/begin/+ [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/begin/.qa [new symlink]
qa/suites/fs/nfs-ganesha/begin/0-install.yaml [new symlink]
qa/suites/fs/nfs-ganesha/begin/2-logrotate.yaml [new symlink]
qa/suites/fs/nfs-ganesha/centos_9.stream.yaml [new symlink]
qa/suites/fs/nfs-ganesha/clusters/.qa [new symlink]
qa/suites/fs/nfs-ganesha/clusters/1a3s-mds-1c.yaml [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/conf [new symlink]
qa/suites/fs/nfs-ganesha/overrides/+ [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/overrides/.qa [new symlink]
qa/suites/fs/nfs-ganesha/overrides/ignorelist_health.yaml [new symlink]
qa/suites/fs/nfs-ganesha/overrides/pg_health.yaml [new symlink]
qa/suites/fs/nfs-ganesha/tasks/% [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/tasks/.qa [new symlink]
qa/suites/fs/nfs-ganesha/tasks/0-create-export.yaml [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/tasks/1-apply-config/% [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/tasks/1-apply-config/.qa [new symlink]
qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/.qa [new symlink]
qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/no.yaml [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/yes.yaml [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/tasks/1-apply-config/ganesha.yaml [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/.qa [new symlink]
qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/no.yaml [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/yes.yaml [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/tasks/2-mount/% [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/tasks/2-mount/.qa [new symlink]
qa/suites/fs/nfs-ganesha/tasks/2-mount/ganesha-client.yaml [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/.qa [new symlink]
qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/4.1.yaml [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/4.2.yaml [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/latest.yaml [new file with mode: 0644]
qa/suites/fs/nfs-ganesha/tasks/3-workload/iogen.yaml [new file with mode: 0644]
qa/tasks/ganesha_client.py [new file with mode: 0644]
qa/tasks/ganesha_reconf.py [new file with mode: 0644]

diff --git a/qa/suites/fs/nfs-ganesha/% b/qa/suites/fs/nfs-ganesha/%
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/qa/suites/fs/nfs-ganesha/.qa b/qa/suites/fs/nfs-ganesha/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/begin/+ b/qa/suites/fs/nfs-ganesha/begin/+
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/qa/suites/fs/nfs-ganesha/begin/.qa b/qa/suites/fs/nfs-ganesha/begin/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/begin/0-install.yaml b/qa/suites/fs/nfs-ganesha/begin/0-install.yaml
new file mode 120000 (symlink)
index 0000000..3b18529
--- /dev/null
@@ -0,0 +1 @@
+.qa/cephfs/begin/0-install.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/begin/2-logrotate.yaml b/qa/suites/fs/nfs-ganesha/begin/2-logrotate.yaml
new file mode 120000 (symlink)
index 0000000..9d6e7ba
--- /dev/null
@@ -0,0 +1 @@
+.qa/cephfs/begin/2-logrotate.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/centos_9.stream.yaml b/qa/suites/fs/nfs-ganesha/centos_9.stream.yaml
new file mode 120000 (symlink)
index 0000000..dca92dd
--- /dev/null
@@ -0,0 +1 @@
+.qa/distros/podman/centos_9.stream.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/clusters/.qa b/qa/suites/fs/nfs-ganesha/clusters/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/clusters/1a3s-mds-1c.yaml b/qa/suites/fs/nfs-ganesha/clusters/1a3s-mds-1c.yaml
new file mode 100644 (file)
index 0000000..ff9e3c9
--- /dev/null
@@ -0,0 +1,9 @@
+roles:
+- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3, client.0]
+- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7]
+openstack:
+- volumes: # attached to each instance
+    count: 4
+    size: 20 # GB
+- machine:
+    disk: 200 # GB
diff --git a/qa/suites/fs/nfs-ganesha/conf b/qa/suites/fs/nfs-ganesha/conf
new file mode 120000 (symlink)
index 0000000..16e8cc4
--- /dev/null
@@ -0,0 +1 @@
+.qa/cephfs/conf
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/overrides/+ b/qa/suites/fs/nfs-ganesha/overrides/+
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/qa/suites/fs/nfs-ganesha/overrides/.qa b/qa/suites/fs/nfs-ganesha/overrides/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/overrides/ignorelist_health.yaml b/qa/suites/fs/nfs-ganesha/overrides/ignorelist_health.yaml
new file mode 120000 (symlink)
index 0000000..5cb891a
--- /dev/null
@@ -0,0 +1 @@
+.qa/cephfs/overrides/ignorelist_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/overrides/pg_health.yaml b/qa/suites/fs/nfs-ganesha/overrides/pg_health.yaml
new file mode 120000 (symlink)
index 0000000..5b6be3a
--- /dev/null
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/% b/qa/suites/fs/nfs-ganesha/tasks/%
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/qa/suites/fs/nfs-ganesha/tasks/.qa b/qa/suites/fs/nfs-ganesha/tasks/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/0-create-export.yaml b/qa/suites/fs/nfs-ganesha/tasks/0-create-export.yaml
new file mode 100644 (file)
index 0000000..43b9fbf
--- /dev/null
@@ -0,0 +1,50 @@
+overrides:
+  ceph:
+    conf:
+      osd:
+        osd shutdown pgref assert: true
+tasks:
+- cephadm:
+    roleless: false
+- cephadm.shell:
+    mon.a:
+      - ceph orch status
+      - ceph orch ps
+      - ceph orch ls
+      - ceph orch host ls
+      - ceph orch device ls
+- cephadm.shell:
+    mon.a:
+      - cmd: ceph nfs cluster create nfs-ganesha-test
+      - cmd: ceph nfs export apply nfs-ganesha-test -i /dev/stdin
+        stdin: |
+          {
+              "export_id": 1,
+              "path": "/",
+              "cluster_id": "nfs-ganesha-test",
+              "pseudo": "/nfsganesha",
+              "access_type": "RW",
+              "squash": "none",
+              "security_label": true,
+              "protocols": [
+                  4
+              ],
+              "transports": [
+                  "TCP"
+              ],
+              "fsal": {
+                  "name": "CEPH",
+                  "user_id": "nfs.nfs-ganesha-test.cephfs.a4cd9f65",
+                  "fs_name": "cephfs",
+                  "cmount_path": "/"
+              },
+              "clients": []
+          }
+      # for debug
+      - cmd: ceph nfs export info nfs-ganesha-test --pseudo_path=/nfsganesha
+      # for debug
+      - cmd: ceph orch ls --service-name nfs.nfs-ganesha-test --export
+      # sleep a bit
+      - cmd: sleep 60
+      # more debug
+      - cmd: ceph orch ps
diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/% b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/%
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/.qa b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/.qa b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/no.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/no.yaml
new file mode 100644 (file)
index 0000000..d45f84a
--- /dev/null
@@ -0,0 +1,3 @@
+overrides:
+  ganesha-reconf:
+      async: no
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/yes.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/yes.yaml
new file mode 100644 (file)
index 0000000..12a9518
--- /dev/null
@@ -0,0 +1,3 @@
+overrides:
+  ganesha-reconf:
+      async: true
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/ganesha.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/ganesha.yaml
new file mode 100644 (file)
index 0000000..5a1b69b
--- /dev/null
@@ -0,0 +1,11 @@
+tasks:
+- ganesha-reconf:
+    cluster_id: 'nfs-ganesha-test'
+    pseudo_path: '/nfsganesha'
+
+- cephadm.shell:
+    mon.a:
+      # sleep a bit
+      - cmd: sleep 30
+      # more debug
+      - cmd: ceph orch ps
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/.qa b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/no.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/no.yaml
new file mode 100644 (file)
index 0000000..9bf12e5
--- /dev/null
@@ -0,0 +1,3 @@
+overrides:
+  ganesha-reconf:
+      zerocopy: no
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/yes.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/yes.yaml
new file mode 100644 (file)
index 0000000..4f6ede0
--- /dev/null
@@ -0,0 +1,3 @@
+overrides:
+  ganesha-reconf:
+      zerocopy: true
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/% b/qa/suites/fs/nfs-ganesha/tasks/2-mount/%
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/.qa b/qa/suites/fs/nfs-ganesha/tasks/2-mount/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/ganesha-client.yaml b/qa/suites/fs/nfs-ganesha/tasks/2-mount/ganesha-client.yaml
new file mode 100644 (file)
index 0000000..2839d86
--- /dev/null
@@ -0,0 +1,5 @@
+tasks:
+- ganesha-client:
+    client.0:
+      cluster_id: nfs-ganesha-test
+      pseudo_path: /nfsganesha
diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/.qa b/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/4.1.yaml b/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/4.1.yaml
new file mode 100644 (file)
index 0000000..5aae179
--- /dev/null
@@ -0,0 +1,3 @@
+overrides:
+  ganesha-client:
+      version: 4.1
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/4.2.yaml b/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/4.2.yaml
new file mode 100644 (file)
index 0000000..56e48f5
--- /dev/null
@@ -0,0 +1,3 @@
+overrides:
+  ganesha-client:
+      version: 4.2
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/latest.yaml b/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/latest.yaml
new file mode 100644 (file)
index 0000000..de4a002
--- /dev/null
@@ -0,0 +1,3 @@
+overrides:
+  ganesha-client:
+      version: latest
\ No newline at end of file
diff --git a/qa/suites/fs/nfs-ganesha/tasks/3-workload/iogen.yaml b/qa/suites/fs/nfs-ganesha/tasks/3-workload/iogen.yaml
new file mode 100644 (file)
index 0000000..98d3b52
--- /dev/null
@@ -0,0 +1,5 @@
+tasks:
+- workunit:
+    clients:
+      client.0:
+        - suites/iogen.sh
\ No newline at end of file
diff --git a/qa/tasks/ganesha_client.py b/qa/tasks/ganesha_client.py
new file mode 100644 (file)
index 0000000..03f550d
--- /dev/null
@@ -0,0 +1,108 @@
+"""
+mount a ganesha client
+"""
+
+import os
+import json
+import logging
+from io import StringIO
+
+from teuthology.misc import deep_merge
+from teuthology.task import Task
+from teuthology import misc
+
+log = logging.getLogger(__name__)
+
+class GaneshaClient(Task):
+    def __init__(self, ctx, config):
+        super(GaneshaClient, self).__init__(ctx, config)
+        self.log = log
+        self.mounts = {}
+
+    def setup(self):
+        super(GaneshaClient, self).setup()
+
+    def begin(self):
+        super(GaneshaClient, self).begin()
+        log.info('mounting ganesha client(s)')
+
+        if self.config is None:
+            ids = misc.all_roles_of_type(self.ctx.cluster, 'client')
+            client_roles = [f'client.{id_}' for id_ in ids]
+            self.config = dict([r, dict()] for r in client_rols)
+        elif isinstance(self.config, list):
+            client_roles = self.config
+            self.config = dict([r, dict()] for r in client_roles)
+        elif isinstance(self.config, dict):
+            client_roles = filter(lambda x: 'client.' in x, self.config.keys())
+        else:
+            raise ValueError(f"Invalid config object: {self.config} ({self.config.__class__})")
+        log.info(f"config is {self.config}")
+
+        mounts = {}
+        overrides = self.ctx.config.get('overrides', {}).get('ganesha-client', {})
+        top_overrides = dict(filter(lambda x: 'client.' not in x[0], overrides.items()))
+
+        clients = list(misc.get_clients(ctx=self.ctx, roles=client_roles))
+        test_dir = misc.get_testdir(self.ctx)
+
+        for id_, remote in clients:
+            entity = f'client.{id_}'
+            client_config = self.config.get(entity)
+            if client_config is None:
+                client_config = {}
+            # top level overrides
+            deep_merge(client_config, top_overrides)
+            # mount specific overrides
+            client_config_overrides = overrides.get(entity)
+            deep_merge(client_config, client_config_overrides)
+            log.info(f"{entity} config is {client_config}")
+
+            cluster_id = client_config['cluster_id']
+            pseudo_path = client_config['pseudo_path']
+            nfs_version = client_config.get('version', 'latest')
+
+            try:
+                first_mon = misc.get_first_mon(self.ctx, None)
+                (mon0_remote,) = self.ctx.cluster.only(first_mon).remotes.keys()
+
+                proc = mon0_remote.run(args=['ceph', 'nfs', 'export', 'info', cluster_id, pseudo_path],
+                                       stdout=StringIO(), wait=True)
+                res = proc.stdout.getvalue()
+                export_json = json.loads(res)
+                log.debug(f'export_json: {export_json}')
+
+                proc = mon0_remote.run(args=['ceph', 'nfs', 'cluster', 'info', cluster_id],
+                                       stdout=StringIO(), wait=True)
+                res = proc.stdout.getvalue()
+                cluster_info = json.loads(res)
+                log.debug(f'cluster_info: {cluster_info}')
+
+                info_output = cluster_info[cluster_id]['sufsend'][0]
+                port = info_output['port']
+                ip = info_output['ip']
+
+                mntpt = os.path.join(test_dir, f'mnt.{id_}')
+                remote.run(args=['mkdir', '-p', mntpt], timeout=60)
+                if nfs_version == 'latest':
+                    remote.run(args=['sudo', 'mount', '-t', 'nfs', '-o',
+                                     f'port={port}', f'{ip}:{pseudo_path}', mntpt])
+                else:
+                    remote.run(args=['sudo', 'mount', '-t', 'nfs', '-o',
+                                     f'port={port},vers={nfs_version}', f'{ip}:{pseudo_path}', mntpt])
+                remote.run(args=['sudo', 'chmod', '1777', mntpt], timeout=60)
+                remote.run(args=['stat', mntpt])
+                mounts[id_] = (remote, mntpt)
+            except Exception as e:
+                log.error(f'failed: {e}')
+        self.mounts = mounts
+
+    def end(self):
+        super(GaneshaClient, self).end()
+        log.debug('unmounting ganesha client(s)')
+        for (remote, mntpt) in self.mounts.values():
+            log.debug(f'unmounting {mntpt}')
+            remote.run(args=['sudo', 'umount', mntpt])
+        self.mounts = {}
+
+task = GaneshaClient
diff --git a/qa/tasks/ganesha_reconf.py b/qa/tasks/ganesha_reconf.py
new file mode 100644 (file)
index 0000000..c8685c2
--- /dev/null
@@ -0,0 +1,72 @@
+"""
+reconfigure a ganesha server
+"""
+
+import json
+import logging
+from io import StringIO
+
+from teuthology.misc import deep_merge
+from teuthology.task import Task
+from teuthology import misc
+
+log = logging.getLogger(__name__)
+
+class GaneshaReconf(Task):
+    def __init__(self, ctx, config):
+        super(GaneshaReconf, self).__init__(ctx, config)
+        self.log = log
+
+    def setup(self):
+        super(GaneshaReconf, self).setup()
+
+    def begin(self):
+        super(GaneshaReconf, self).begin()
+        log.info('reconfiguring ganesha server')
+
+        ganesha_config = self.config
+        log.info(f'ganesha_config is {ganesha_config}')
+        overrides = self.ctx.config.get('overrides', {}).get('ganesha-reconf', {})
+        log.info(f'overrides is {overrides}')
+
+        deep_merge(ganesha_config, overrides)
+        log.info(f'ganesha_config is {ganesha_config}')
+
+        try:
+            first_mon = misc.get_first_mon(self.ctx, None)
+            (mon0_remote,) = self.ctx.cluster.only(first_mon).remotes.keys()
+
+            cluster_id = ganesha_config['cluster_id']
+            pseudo_path = ganesha_config['pseudo_path']
+
+            proc = mon0_remote.run(args=['ceph', 'nfs', 'export', 'info', cluster_id, pseudo_path],
+                                   stdout=StringIO(), wait=True)
+            res = proc.stdout.getvalue()
+            export_json = json.loads(res)
+            log.debug(f'export_json: {export_json}')
+
+            ceph_section = {'async': False, 'zerocopy': False}
+            is_async = ganesha_config.get('async', False)
+            if is_async:
+                ceph_section["async"] = True
+            is_zerocopy = ganesha_config.get('zerocopy', False)
+            if is_zerocopy:
+                ceph_section["zerocopy"] = True
+
+            new_export = {}
+            if "export" in export_json.keys():
+                new_export = export_json
+            else:
+                new_export["export"] = export_json
+            new_export["ceph"] = ceph_section
+
+            log.debug(f'new_export is {json.dumps(new_export)}')
+            mon0_remote.run(args=['ceph', 'nfs', 'export', 'apply', cluster_id, "-i", "-"],
+                            stdin=json.dumps(new_export))
+        except Exception as e:
+                log.error(f'failed: {e}')
+
+    def end(self):
+        super(GaneshaReconf, self).end()
+
+task = GaneshaReconf