qa/cephfs: introduce nfs-ganesha tests

Signed-off-by: Venky Shankar <vshankar@redhat.com>

diff --git a/qa/suites/fs/nfs-ganesha/% b/qa/suites/fs/nfs-ganesha/%
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/fs/nfs-ganesha/.qa b/qa/suites/fs/nfs-ganesha/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/begin/+ b/qa/suites/fs/nfs-ganesha/begin/+
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/fs/nfs-ganesha/begin/.qa b/qa/suites/fs/nfs-ganesha/begin/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/begin/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/begin/0-install.yaml b/qa/suites/fs/nfs-ganesha/begin/0-install.yaml
new file mode 120000 (symlink)
index 0000000..3b18529
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/begin/0-install.yaml
@@ -0,0 +1 @@
+.qa/cephfs/begin/0-install.yaml
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/begin/2-logrotate.yaml b/qa/suites/fs/nfs-ganesha/begin/2-logrotate.yaml
new file mode 120000 (symlink)
index 0000000..9d6e7ba
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/begin/2-logrotate.yaml
@@ -0,0 +1 @@
+.qa/cephfs/begin/2-logrotate.yaml
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/centos_9.stream.yaml b/qa/suites/fs/nfs-ganesha/centos_9.stream.yaml
new file mode 120000 (symlink)
index 0000000..dca92dd
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/centos_9.stream.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_9.stream.yaml
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/clusters/.qa b/qa/suites/fs/nfs-ganesha/clusters/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/clusters/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/clusters/1a3s-mds-1c.yaml b/qa/suites/fs/nfs-ganesha/clusters/1a3s-mds-1c.yaml
new file mode 100644 (file)
index 0000000..ff9e3c9
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/clusters/1a3s-mds-1c.yaml
@@ -0,0 +1,9 @@
+roles:
+- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3, client.0]
+- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7]
+openstack:
+- volumes: # attached to each instance
+    count: 4
+    size: 20 # GB
+- machine:
+    disk: 200 # GB

diff --git a/qa/suites/fs/nfs-ganesha/conf b/qa/suites/fs/nfs-ganesha/conf
new file mode 120000 (symlink)
index 0000000..16e8cc4
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/conf
@@ -0,0 +1 @@
+.qa/cephfs/conf
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/overrides/+ b/qa/suites/fs/nfs-ganesha/overrides/+
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/fs/nfs-ganesha/overrides/.qa b/qa/suites/fs/nfs-ganesha/overrides/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/overrides/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/overrides/ignorelist_health.yaml b/qa/suites/fs/nfs-ganesha/overrides/ignorelist_health.yaml
new file mode 100644 (file)
index 0000000..834400c
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/overrides/ignorelist_health.yaml
@@ -0,0 +1,29 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - FS_DEGRADED
+      - fs.*is degraded
+      - filesystem is degraded
+      - FS_INLINE_DATA_DEPRECATED
+      - FS_WITH_FAILED_MDS
+      - MDS_ALL_DOWN
+      - filesystem is offline
+      - is offline because no MDS
+      - MDS_DAMAGE
+      - MDS_DEGRADED
+      - MDS_FAILED
+      - MDS_INSUFFICIENT_STANDBY
+      - insufficient standby MDS daemons available
+      - MDS_UP_LESS_THAN_MAX
+      - online, but wants
+      - filesystem is online with fewer MDS than max_mds
+      - POOL_APP_NOT_ENABLED
+      - do not have an application enabled
+      - overall HEALTH_
+      - Replacing daemon
+      - deprecated feature inline_data
+      - BLUESTORE_SLOW_OP_ALERT
+      - slow operation indications in BlueStore
+      - experiencing slow operations in BlueStore
+      - MGR_MODULE_ERROR
+      - OSD_NEARFULL

diff --git a/qa/suites/fs/nfs-ganesha/overrides/pg_health.yaml b/qa/suites/fs/nfs-ganesha/overrides/pg_health.yaml
new file mode 120000 (symlink)
index 0000000..5b6be3a
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/tasks/% b/qa/suites/fs/nfs-ganesha/tasks/%
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/fs/nfs-ganesha/tasks/.qa b/qa/suites/fs/nfs-ganesha/tasks/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/tasks/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/tasks/0-create-export.yaml b/qa/suites/fs/nfs-ganesha/tasks/0-create-export.yaml
new file mode 100644 (file)
index 0000000..86a588b
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/tasks/0-create-export.yaml
@@ -0,0 +1,54 @@
+overrides:
+  ceph:
+    conf:
+      osd:
+        osd shutdown pgref assert: true
+tasks:
+- cephadm:
+    roleless: false
+- cephadm.shell:
+    mon.a:
+      - ceph orch status
+      - ceph orch ps
+      - ceph orch ls
+      - ceph orch host ls
+      - ceph orch device ls
+- cephadm.shell:
+    mon.a:
+      - cmd: ceph nfs cluster create nfs-ganesha-test
+      - cmd: ceph nfs export apply nfs-ganesha-test -i /dev/stdin
+        stdin: |
+          {
+             "export": {
+                "export_id": 1,
+                "path": "/",
+                "cluster_id": "nfs-ganesha-test",
+                "pseudo": "/nfsganesha",
+                "access_type": "RW",
+                "squash": "none",
+                "security_label": true,
+                "protocols": [
+                  3,
+                  4
+                ],
+                "transports": [
+                  "TCP"
+                ],
+                "fsal": {
+                 "name": "CEPH",
+                 "user_id": "nfs.nfs-ganesha-test.cephfs.a4cd9f65",
+                 "fs_name": "cephfs",
+                 "cmount_path": "/"
+                },
+                "clients": []
+             },
+             "ceph": {
+                "async": true,
+                "zerocopy": true
+             }
+          }
+
+      # for debug
+      - cmd: ceph nfs export info nfs-ganesha-test --pseudo_path=/nfsganesha
+      # for debug
+      - cmd: ceph orch ls --service-name nfs.nfs-ganesha-test --export

diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/% b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/%
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/.qa b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/.qa b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/no.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/no.yaml
new file mode 100644 (file)
index 0000000..d45f84a
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/no.yaml
@@ -0,0 +1,3 @@
+overrides:
+  ganesha-reconf:
+      async: no
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/yes.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/yes.yaml
new file mode 100644 (file)
index 0000000..12a9518
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/yes.yaml
@@ -0,0 +1,3 @@
+overrides:
+  ganesha-reconf:
+      async: true
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/ganesha.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/ganesha.yaml
new file mode 100644 (file)
index 0000000..7ee4bb5
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/ganesha.yaml
@@ -0,0 +1,4 @@
+tasks:
+- ganesha-reconf:
+    cluster_id: 'nfs-ganesha-test'
+    pseudo_path: '/nfsganesha'
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/.qa b/qa/suites/fs/nfs-ganesha/tasks/2-mount/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/tasks/2-mount/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/ganesha-client.yaml b/qa/suites/fs/nfs-ganesha/tasks/2-mount/ganesha-client.yaml
new file mode 100644 (file)
index 0000000..1cf9002
--- /dev/null
+++ b/qa/suites/fs/nfs-ganesha/tasks/2-mount/ganesha-client.yaml
@@ -0,0 +1,2 @@
+tasks:
+- ganesha-client:
\ No newline at end of file

diff --git a/qa/tasks/ganesha_client.py b/qa/tasks/ganesha_client.py
new file mode 100644 (file)
index 0000000..995b8e2
--- /dev/null
+++ b/qa/tasks/ganesha_client.py
@@ -0,0 +1,27 @@
+"""
+reconfigure a ganesha server
+"""
+import logging
+
+from teuthology.task import Task
+
+log = logging.getLogger(__name__)
+
+class GaneshaClient(Task):
+    def __init__(self, ctx, config):
+        super(GaneshaClient, self).__init__(ctx, config)
+        self.log = log
+
+    def setup(self):
+        super(GaneshaClient, self).setup()
+
+    def begin(self):
+        super(GaneshaClient, self).begin()
+
+        log.info('mounting ganesha client')
+        log.info(f'config is {self.config}')
+
+    def end(self):
+        super(GaneshaClient, self).end()
+
+task = GaneshaClient

diff --git a/qa/tasks/ganesha_reconf.py b/qa/tasks/ganesha_reconf.py
new file mode 100644 (file)
index 0000000..26f1a01
--- /dev/null
+++ b/qa/tasks/ganesha_reconf.py
@@ -0,0 +1,75 @@
+"""
+reconfigure a ganesha server
+"""
+
+import json
+import logging
+from io import StringIO
+
+from teuthology.misc import deep_merge
+from teuthology.task import Task
+from teuthology import misc
+
+log = logging.getLogger(__name__)
+
+class GaneshaReconf(Task):
+    def __init__(self, ctx, config):
+        super(GaneshaReconf, self).__init__(ctx, config)
+        self.log = log
+
+    def setup(self):
+        super(GaneshaReconf, self).setup()
+
+    def begin(self):
+        super(GaneshaReconf, self).begin()
+        log.info('reconfiguring ganesha server')
+
+        ganesha_config = self.config
+        log.info(f'ganesha_config is {ganesha_config}')
+        overrides = self.ctx.config.get('overrides', {}).get('ganesha-reconf', {})
+        log.info(f'overrides is {overrides}')
+
+        deep_merge(ganesha_config, overrides)
+        log.info(f'ganesha_config is {ganesha_config}')
+
+        try:
+            first_mon = misc.get_first_mon(self.ctx, None)
+            (mon0_remote,) = self.ctx.cluster.only(first_mon).remotes.keys()
+
+            cluster_id = ganesha_config['cluster_id']
+            pseudo_path = ganesha_config['pseudo_path']
+
+            proc = mon0_remote.run(args=['ceph', 'nfs', 'export', 'info', cluster_id, pseudo_path],
+                                   stdout=StringIO(), wait=True)
+            res = proc.stdout.getvalue()
+            log.debug(f'res: {res} type {type(res)}')
+            export_json = json.loads(res)
+            log.debug(f'export_json: {export_json} type {type(export_json)}')
+
+            is_async = ganesha_config.get('async', False)
+            if is_async:
+                export_json.setdefault("ceph", {})
+                export_json["ceph"]["async"] = True
+            is_zerocopy = ganesha_config.get('zerocopy', False)
+            if is_async:
+                export_json.setdefault("ceph", {})
+                export_json["ceph"]["zerocopy"] = True
+
+            if is_async or is_zerocopy:
+                log.debug(f'export_json is {json.dumps(export_json)}')
+                mon0_remote.run(args=['ceph', 'nfs', 'export', 'apply', cluster_id, "-i", "-"],
+                                      stdin=json.dumps(export_json))
+
+            proc = mon0_remote.run(args=['ceph', 'nfs', 'export', 'info', cluster_id, pseudo_path],
+                                   stdout=StringIO(), wait=True)
+            res = proc.stdout.getvalue()
+            log.debug(f'verify res: {res} type {type(res)}')
+            export_json = json.loads(res)
+            log.debug(f'verify export_json: {export_json} type {type(export_json)}')
+        except Exception as e:
+            log.error(f'failed: {e}')
+
+    def end(self):
+        super(GaneshaReconf, self).end()
+
+task = GaneshaReconf