From 2d3ea3ad29ee8661222df3184df33350b6a58dbd Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Fri, 1 Aug 2025 10:39:46 +0000 Subject: [PATCH] qa/cephfs: introduce nfs-ganesha tests Signed-off-by: Venky Shankar --- qa/suites/fs/nfs-ganesha/% | 0 qa/suites/fs/nfs-ganesha/.qa | 1 + qa/suites/fs/nfs-ganesha/begin/+ | 0 qa/suites/fs/nfs-ganesha/begin/.qa | 1 + qa/suites/fs/nfs-ganesha/begin/0-install.yaml | 1 + .../fs/nfs-ganesha/begin/2-logrotate.yaml | 1 + qa/suites/fs/nfs-ganesha/centos_9.stream.yaml | 1 + qa/suites/fs/nfs-ganesha/clusters/.qa | 1 + .../fs/nfs-ganesha/clusters/1a3s-mds-1c.yaml | 9 +++ qa/suites/fs/nfs-ganesha/conf | 1 + qa/suites/fs/nfs-ganesha/overrides/+ | 0 qa/suites/fs/nfs-ganesha/overrides/.qa | 1 + .../overrides/ignorelist_health.yaml | 29 +++++++ .../fs/nfs-ganesha/overrides/pg_health.yaml | 1 + qa/suites/fs/nfs-ganesha/tasks/% | 0 qa/suites/fs/nfs-ganesha/tasks/.qa | 1 + .../fs/nfs-ganesha/tasks/0-create-export.yaml | 54 +++++++++++++ .../fs/nfs-ganesha/tasks/1-apply-config/% | 0 .../fs/nfs-ganesha/tasks/1-apply-config/.qa | 1 + .../tasks/1-apply-config/async/.qa | 1 + .../tasks/1-apply-config/async/no.yaml | 3 + .../tasks/1-apply-config/async/yes.yaml | 3 + .../tasks/1-apply-config/ganesha.yaml | 4 + qa/suites/fs/nfs-ganesha/tasks/2-mount/.qa | 1 + .../tasks/2-mount/ganesha-client.yaml | 2 + qa/tasks/ganesha_client.py | 27 +++++++ qa/tasks/ganesha_reconf.py | 75 +++++++++++++++++++ 27 files changed, 219 insertions(+) create mode 100644 qa/suites/fs/nfs-ganesha/% create mode 120000 qa/suites/fs/nfs-ganesha/.qa create mode 100644 qa/suites/fs/nfs-ganesha/begin/+ create mode 120000 qa/suites/fs/nfs-ganesha/begin/.qa create mode 120000 qa/suites/fs/nfs-ganesha/begin/0-install.yaml create mode 120000 qa/suites/fs/nfs-ganesha/begin/2-logrotate.yaml create mode 120000 qa/suites/fs/nfs-ganesha/centos_9.stream.yaml create mode 120000 qa/suites/fs/nfs-ganesha/clusters/.qa create mode 100644 qa/suites/fs/nfs-ganesha/clusters/1a3s-mds-1c.yaml create mode 120000 qa/suites/fs/nfs-ganesha/conf create mode 100644 qa/suites/fs/nfs-ganesha/overrides/+ create mode 120000 qa/suites/fs/nfs-ganesha/overrides/.qa create mode 100644 qa/suites/fs/nfs-ganesha/overrides/ignorelist_health.yaml create mode 120000 qa/suites/fs/nfs-ganesha/overrides/pg_health.yaml create mode 100644 qa/suites/fs/nfs-ganesha/tasks/% create mode 120000 qa/suites/fs/nfs-ganesha/tasks/.qa create mode 100644 qa/suites/fs/nfs-ganesha/tasks/0-create-export.yaml create mode 100644 qa/suites/fs/nfs-ganesha/tasks/1-apply-config/% create mode 120000 qa/suites/fs/nfs-ganesha/tasks/1-apply-config/.qa create mode 120000 qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/.qa create mode 100644 qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/no.yaml create mode 100644 qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/yes.yaml create mode 100644 qa/suites/fs/nfs-ganesha/tasks/1-apply-config/ganesha.yaml create mode 120000 qa/suites/fs/nfs-ganesha/tasks/2-mount/.qa create mode 100644 qa/suites/fs/nfs-ganesha/tasks/2-mount/ganesha-client.yaml create mode 100644 qa/tasks/ganesha_client.py create mode 100644 qa/tasks/ganesha_reconf.py diff --git a/qa/suites/fs/nfs-ganesha/% b/qa/suites/fs/nfs-ganesha/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/fs/nfs-ganesha/.qa b/qa/suites/fs/nfs-ganesha/.qa new file mode 120000 index 00000000000..fea2489fdf6 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/begin/+ b/qa/suites/fs/nfs-ganesha/begin/+ new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/fs/nfs-ganesha/begin/.qa b/qa/suites/fs/nfs-ganesha/begin/.qa new file mode 120000 index 00000000000..fea2489fdf6 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/begin/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/begin/0-install.yaml b/qa/suites/fs/nfs-ganesha/begin/0-install.yaml new file mode 120000 index 00000000000..3b18529732d --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/begin/0-install.yaml @@ -0,0 +1 @@ +.qa/cephfs/begin/0-install.yaml \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/begin/2-logrotate.yaml b/qa/suites/fs/nfs-ganesha/begin/2-logrotate.yaml new file mode 120000 index 00000000000..9d6e7ba8335 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/begin/2-logrotate.yaml @@ -0,0 +1 @@ +.qa/cephfs/begin/2-logrotate.yaml \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/centos_9.stream.yaml b/qa/suites/fs/nfs-ganesha/centos_9.stream.yaml new file mode 120000 index 00000000000..dca92ddbf45 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/centos_9.stream.yaml @@ -0,0 +1 @@ +.qa/distros/podman/centos_9.stream.yaml \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/clusters/.qa b/qa/suites/fs/nfs-ganesha/clusters/.qa new file mode 120000 index 00000000000..fea2489fdf6 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/clusters/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/clusters/1a3s-mds-1c.yaml b/qa/suites/fs/nfs-ganesha/clusters/1a3s-mds-1c.yaml new file mode 100644 index 00000000000..ff9e3c9ac28 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/clusters/1a3s-mds-1c.yaml @@ -0,0 +1,9 @@ +roles: +- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3, client.0] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7] +openstack: +- volumes: # attached to each instance + count: 4 + size: 20 # GB +- machine: + disk: 200 # GB diff --git a/qa/suites/fs/nfs-ganesha/conf b/qa/suites/fs/nfs-ganesha/conf new file mode 120000 index 00000000000..16e8cc44b7d --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/conf @@ -0,0 +1 @@ +.qa/cephfs/conf \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/overrides/+ b/qa/suites/fs/nfs-ganesha/overrides/+ new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/fs/nfs-ganesha/overrides/.qa b/qa/suites/fs/nfs-ganesha/overrides/.qa new file mode 120000 index 00000000000..fea2489fdf6 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/overrides/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/overrides/ignorelist_health.yaml b/qa/suites/fs/nfs-ganesha/overrides/ignorelist_health.yaml new file mode 100644 index 00000000000..834400c7c9b --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/overrides/ignorelist_health.yaml @@ -0,0 +1,29 @@ +overrides: + ceph: + log-ignorelist: + - FS_DEGRADED + - fs.*is degraded + - filesystem is degraded + - FS_INLINE_DATA_DEPRECATED + - FS_WITH_FAILED_MDS + - MDS_ALL_DOWN + - filesystem is offline + - is offline because no MDS + - MDS_DAMAGE + - MDS_DEGRADED + - MDS_FAILED + - MDS_INSUFFICIENT_STANDBY + - insufficient standby MDS daemons available + - MDS_UP_LESS_THAN_MAX + - online, but wants + - filesystem is online with fewer MDS than max_mds + - POOL_APP_NOT_ENABLED + - do not have an application enabled + - overall HEALTH_ + - Replacing daemon + - deprecated feature inline_data + - BLUESTORE_SLOW_OP_ALERT + - slow operation indications in BlueStore + - experiencing slow operations in BlueStore + - MGR_MODULE_ERROR + - OSD_NEARFULL diff --git a/qa/suites/fs/nfs-ganesha/overrides/pg_health.yaml b/qa/suites/fs/nfs-ganesha/overrides/pg_health.yaml new file mode 120000 index 00000000000..5b6be3a65b6 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/overrides/pg_health.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/pg_health.yaml \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/% b/qa/suites/fs/nfs-ganesha/tasks/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/fs/nfs-ganesha/tasks/.qa b/qa/suites/fs/nfs-ganesha/tasks/.qa new file mode 120000 index 00000000000..fea2489fdf6 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/0-create-export.yaml b/qa/suites/fs/nfs-ganesha/tasks/0-create-export.yaml new file mode 100644 index 00000000000..86a588b653a --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/0-create-export.yaml @@ -0,0 +1,54 @@ +overrides: + ceph: + conf: + osd: + osd shutdown pgref assert: true +tasks: +- cephadm: + roleless: false +- cephadm.shell: + mon.a: + - ceph orch status + - ceph orch ps + - ceph orch ls + - ceph orch host ls + - ceph orch device ls +- cephadm.shell: + mon.a: + - cmd: ceph nfs cluster create nfs-ganesha-test + - cmd: ceph nfs export apply nfs-ganesha-test -i /dev/stdin + stdin: | + { + "export": { + "export_id": 1, + "path": "/", + "cluster_id": "nfs-ganesha-test", + "pseudo": "/nfsganesha", + "access_type": "RW", + "squash": "none", + "security_label": true, + "protocols": [ + 3, + 4 + ], + "transports": [ + "TCP" + ], + "fsal": { + "name": "CEPH", + "user_id": "nfs.nfs-ganesha-test.cephfs.a4cd9f65", + "fs_name": "cephfs", + "cmount_path": "/" + }, + "clients": [] + }, + "ceph": { + "async": true, + "zerocopy": true + } + } + + # for debug + - cmd: ceph nfs export info nfs-ganesha-test --pseudo_path=/nfsganesha + # for debug + - cmd: ceph orch ls --service-name nfs.nfs-ganesha-test --export diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/% b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/.qa b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/.qa b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/no.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/no.yaml new file mode 100644 index 00000000000..d45f84ab73a --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/no.yaml @@ -0,0 +1,3 @@ +overrides: + ganesha-reconf: + async: no \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/yes.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/yes.yaml new file mode 100644 index 00000000000..12a95180a2e --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/yes.yaml @@ -0,0 +1,3 @@ +overrides: + ganesha-reconf: + async: true \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/ganesha.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/ganesha.yaml new file mode 100644 index 00000000000..7ee4bb5363b --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/ganesha.yaml @@ -0,0 +1,4 @@ +tasks: +- ganesha-reconf: + cluster_id: 'nfs-ganesha-test' + pseudo_path: '/nfsganesha' \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/.qa b/qa/suites/fs/nfs-ganesha/tasks/2-mount/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/2-mount/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/ganesha-client.yaml b/qa/suites/fs/nfs-ganesha/tasks/2-mount/ganesha-client.yaml new file mode 100644 index 00000000000..1cf9002befd --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/2-mount/ganesha-client.yaml @@ -0,0 +1,2 @@ +tasks: +- ganesha-client: \ No newline at end of file diff --git a/qa/tasks/ganesha_client.py b/qa/tasks/ganesha_client.py new file mode 100644 index 00000000000..995b8e295db --- /dev/null +++ b/qa/tasks/ganesha_client.py @@ -0,0 +1,27 @@ +""" +reconfigure a ganesha server +""" +import logging + +from teuthology.task import Task + +log = logging.getLogger(__name__) + +class GaneshaClient(Task): + def __init__(self, ctx, config): + super(GaneshaClient, self).__init__(ctx, config) + self.log = log + + def setup(self): + super(GaneshaClient, self).setup() + + def begin(self): + super(GaneshaClient, self).begin() + + log.info('mounting ganesha client') + log.info(f'config is {self.config}') + + def end(self): + super(GaneshaClient, self).end() + +task = GaneshaClient diff --git a/qa/tasks/ganesha_reconf.py b/qa/tasks/ganesha_reconf.py new file mode 100644 index 00000000000..26f1a01f003 --- /dev/null +++ b/qa/tasks/ganesha_reconf.py @@ -0,0 +1,75 @@ +""" +reconfigure a ganesha server +""" + +import json +import logging +from io import StringIO + +from teuthology.misc import deep_merge +from teuthology.task import Task +from teuthology import misc + +log = logging.getLogger(__name__) + +class GaneshaReconf(Task): + def __init__(self, ctx, config): + super(GaneshaReconf, self).__init__(ctx, config) + self.log = log + + def setup(self): + super(GaneshaReconf, self).setup() + + def begin(self): + super(GaneshaReconf, self).begin() + log.info('reconfiguring ganesha server') + + ganesha_config = self.config + log.info(f'ganesha_config is {ganesha_config}') + overrides = self.ctx.config.get('overrides', {}).get('ganesha-reconf', {}) + log.info(f'overrides is {overrides}') + + deep_merge(ganesha_config, overrides) + log.info(f'ganesha_config is {ganesha_config}') + + try: + first_mon = misc.get_first_mon(self.ctx, None) + (mon0_remote,) = self.ctx.cluster.only(first_mon).remotes.keys() + + cluster_id = ganesha_config['cluster_id'] + pseudo_path = ganesha_config['pseudo_path'] + + proc = mon0_remote.run(args=['ceph', 'nfs', 'export', 'info', cluster_id, pseudo_path], + stdout=StringIO(), wait=True) + res = proc.stdout.getvalue() + log.debug(f'res: {res} type {type(res)}') + export_json = json.loads(res) + log.debug(f'export_json: {export_json} type {type(export_json)}') + + is_async = ganesha_config.get('async', False) + if is_async: + export_json.setdefault("ceph", {}) + export_json["ceph"]["async"] = True + is_zerocopy = ganesha_config.get('zerocopy', False) + if is_async: + export_json.setdefault("ceph", {}) + export_json["ceph"]["zerocopy"] = True + + if is_async or is_zerocopy: + log.debug(f'export_json is {json.dumps(export_json)}') + mon0_remote.run(args=['ceph', 'nfs', 'export', 'apply', cluster_id, "-i", "-"], + stdin=json.dumps(export_json)) + + proc = mon0_remote.run(args=['ceph', 'nfs', 'export', 'info', cluster_id, pseudo_path], + stdout=StringIO(), wait=True) + res = proc.stdout.getvalue() + log.debug(f'verify res: {res} type {type(res)}') + export_json = json.loads(res) + log.debug(f'verify export_json: {export_json} type {type(export_json)}') + except Exception as e: + log.error(f'failed: {e}') + + def end(self): + super(GaneshaReconf, self).end() + +task = GaneshaReconf -- 2.39.5