From: Venky Shankar Date: Fri, 1 Aug 2025 10:39:46 +0000 (+0000) Subject: qa/cephfs: introduce nfs-ganesha tests X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=b573e39cc8a7c42e2a824143d701f6b613940149;p=ceph-ci.git qa/cephfs: introduce nfs-ganesha tests Fixes: http://tracker.ceph.com/issues/73172 Signed-off-by: Venky Shankar --- diff --git a/qa/suites/fs/nfs-ganesha/% b/qa/suites/fs/nfs-ganesha/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/fs/nfs-ganesha/.qa b/qa/suites/fs/nfs-ganesha/.qa new file mode 120000 index 00000000000..fea2489fdf6 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/begin/+ b/qa/suites/fs/nfs-ganesha/begin/+ new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/fs/nfs-ganesha/begin/.qa b/qa/suites/fs/nfs-ganesha/begin/.qa new file mode 120000 index 00000000000..fea2489fdf6 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/begin/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/begin/0-install.yaml b/qa/suites/fs/nfs-ganesha/begin/0-install.yaml new file mode 120000 index 00000000000..3b18529732d --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/begin/0-install.yaml @@ -0,0 +1 @@ +.qa/cephfs/begin/0-install.yaml \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/begin/2-logrotate.yaml b/qa/suites/fs/nfs-ganesha/begin/2-logrotate.yaml new file mode 120000 index 00000000000..9d6e7ba8335 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/begin/2-logrotate.yaml @@ -0,0 +1 @@ +.qa/cephfs/begin/2-logrotate.yaml \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/centos_9.stream.yaml b/qa/suites/fs/nfs-ganesha/centos_9.stream.yaml new file mode 120000 index 00000000000..dca92ddbf45 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/centos_9.stream.yaml @@ -0,0 +1 @@ +.qa/distros/podman/centos_9.stream.yaml \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/clusters/.qa b/qa/suites/fs/nfs-ganesha/clusters/.qa new file mode 120000 index 00000000000..fea2489fdf6 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/clusters/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/clusters/1a3s-mds-1c.yaml b/qa/suites/fs/nfs-ganesha/clusters/1a3s-mds-1c.yaml new file mode 100644 index 00000000000..ff9e3c9ac28 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/clusters/1a3s-mds-1c.yaml @@ -0,0 +1,9 @@ +roles: +- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3, client.0] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7] +openstack: +- volumes: # attached to each instance + count: 4 + size: 20 # GB +- machine: + disk: 200 # GB diff --git a/qa/suites/fs/nfs-ganesha/conf b/qa/suites/fs/nfs-ganesha/conf new file mode 120000 index 00000000000..16e8cc44b7d --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/conf @@ -0,0 +1 @@ +.qa/cephfs/conf \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/overrides/+ b/qa/suites/fs/nfs-ganesha/overrides/+ new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/fs/nfs-ganesha/overrides/.qa b/qa/suites/fs/nfs-ganesha/overrides/.qa new file mode 120000 index 00000000000..fea2489fdf6 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/overrides/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/overrides/ignorelist_health.yaml b/qa/suites/fs/nfs-ganesha/overrides/ignorelist_health.yaml new file mode 120000 index 00000000000..5cb891a95c3 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/overrides/ignorelist_health.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/ignorelist_health.yaml \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/overrides/pg_health.yaml b/qa/suites/fs/nfs-ganesha/overrides/pg_health.yaml new file mode 120000 index 00000000000..5b6be3a65b6 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/overrides/pg_health.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/pg_health.yaml \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/% b/qa/suites/fs/nfs-ganesha/tasks/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/fs/nfs-ganesha/tasks/.qa b/qa/suites/fs/nfs-ganesha/tasks/.qa new file mode 120000 index 00000000000..fea2489fdf6 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/0-create-export.yaml b/qa/suites/fs/nfs-ganesha/tasks/0-create-export.yaml new file mode 100644 index 00000000000..43b9fbf32da --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/0-create-export.yaml @@ -0,0 +1,50 @@ +overrides: + ceph: + conf: + osd: + osd shutdown pgref assert: true +tasks: +- cephadm: + roleless: false +- cephadm.shell: + mon.a: + - ceph orch status + - ceph orch ps + - ceph orch ls + - ceph orch host ls + - ceph orch device ls +- cephadm.shell: + mon.a: + - cmd: ceph nfs cluster create nfs-ganesha-test + - cmd: ceph nfs export apply nfs-ganesha-test -i /dev/stdin + stdin: | + { + "export_id": 1, + "path": "/", + "cluster_id": "nfs-ganesha-test", + "pseudo": "/nfsganesha", + "access_type": "RW", + "squash": "none", + "security_label": true, + "protocols": [ + 4 + ], + "transports": [ + "TCP" + ], + "fsal": { + "name": "CEPH", + "user_id": "nfs.nfs-ganesha-test.cephfs.a4cd9f65", + "fs_name": "cephfs", + "cmount_path": "/" + }, + "clients": [] + } + # for debug + - cmd: ceph nfs export info nfs-ganesha-test --pseudo_path=/nfsganesha + # for debug + - cmd: ceph orch ls --service-name nfs.nfs-ganesha-test --export + # sleep a bit + - cmd: sleep 60 + # more debug + - cmd: ceph orch ps diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/% b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/.qa b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/.qa b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/no.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/no.yaml new file mode 100644 index 00000000000..d45f84ab73a --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/no.yaml @@ -0,0 +1,3 @@ +overrides: + ganesha-reconf: + async: no \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/yes.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/yes.yaml new file mode 100644 index 00000000000..12a95180a2e --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/async/yes.yaml @@ -0,0 +1,3 @@ +overrides: + ganesha-reconf: + async: true \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/ganesha.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/ganesha.yaml new file mode 100644 index 00000000000..5a1b69b8299 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/ganesha.yaml @@ -0,0 +1,11 @@ +tasks: +- ganesha-reconf: + cluster_id: 'nfs-ganesha-test' + pseudo_path: '/nfsganesha' + +- cephadm.shell: + mon.a: + # sleep a bit + - cmd: sleep 30 + # more debug + - cmd: ceph orch ps \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/.qa b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/no.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/no.yaml new file mode 100644 index 00000000000..9bf12e54c60 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/no.yaml @@ -0,0 +1,3 @@ +overrides: + ganesha-reconf: + zerocopy: no \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/yes.yaml b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/yes.yaml new file mode 100644 index 00000000000..4f6ede0d5b7 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/1-apply-config/zerocopy/yes.yaml @@ -0,0 +1,3 @@ +overrides: + ganesha-reconf: + zerocopy: true \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/% b/qa/suites/fs/nfs-ganesha/tasks/2-mount/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/.qa b/qa/suites/fs/nfs-ganesha/tasks/2-mount/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/2-mount/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/ganesha-client.yaml b/qa/suites/fs/nfs-ganesha/tasks/2-mount/ganesha-client.yaml new file mode 100644 index 00000000000..2839d8665c1 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/2-mount/ganesha-client.yaml @@ -0,0 +1,5 @@ +tasks: +- ganesha-client: + client.0: + cluster_id: nfs-ganesha-test + pseudo_path: /nfsganesha diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/.qa b/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/4.1.yaml b/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/4.1.yaml new file mode 100644 index 00000000000..5aae179249a --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/4.1.yaml @@ -0,0 +1,3 @@ +overrides: + ganesha-client: + version: 4.1 \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/4.2.yaml b/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/4.2.yaml new file mode 100644 index 00000000000..56e48f55170 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/4.2.yaml @@ -0,0 +1,3 @@ +overrides: + ganesha-client: + version: 4.2 \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/latest.yaml b/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/latest.yaml new file mode 100644 index 00000000000..de4a0028023 --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/2-mount/nfs-version/latest.yaml @@ -0,0 +1,3 @@ +overrides: + ganesha-client: + version: latest \ No newline at end of file diff --git a/qa/suites/fs/nfs-ganesha/tasks/3-workload/iogen.yaml b/qa/suites/fs/nfs-ganesha/tasks/3-workload/iogen.yaml new file mode 100644 index 00000000000..98d3b52db0a --- /dev/null +++ b/qa/suites/fs/nfs-ganesha/tasks/3-workload/iogen.yaml @@ -0,0 +1,5 @@ +tasks: +- workunit: + clients: + client.0: + - suites/iogen.sh \ No newline at end of file diff --git a/qa/tasks/ganesha_client.py b/qa/tasks/ganesha_client.py new file mode 100644 index 00000000000..03f550d4780 --- /dev/null +++ b/qa/tasks/ganesha_client.py @@ -0,0 +1,108 @@ +""" +mount a ganesha client +""" + +import os +import json +import logging +from io import StringIO + +from teuthology.misc import deep_merge +from teuthology.task import Task +from teuthology import misc + +log = logging.getLogger(__name__) + +class GaneshaClient(Task): + def __init__(self, ctx, config): + super(GaneshaClient, self).__init__(ctx, config) + self.log = log + self.mounts = {} + + def setup(self): + super(GaneshaClient, self).setup() + + def begin(self): + super(GaneshaClient, self).begin() + log.info('mounting ganesha client(s)') + + if self.config is None: + ids = misc.all_roles_of_type(self.ctx.cluster, 'client') + client_roles = [f'client.{id_}' for id_ in ids] + self.config = dict([r, dict()] for r in client_rols) + elif isinstance(self.config, list): + client_roles = self.config + self.config = dict([r, dict()] for r in client_roles) + elif isinstance(self.config, dict): + client_roles = filter(lambda x: 'client.' in x, self.config.keys()) + else: + raise ValueError(f"Invalid config object: {self.config} ({self.config.__class__})") + log.info(f"config is {self.config}") + + mounts = {} + overrides = self.ctx.config.get('overrides', {}).get('ganesha-client', {}) + top_overrides = dict(filter(lambda x: 'client.' not in x[0], overrides.items())) + + clients = list(misc.get_clients(ctx=self.ctx, roles=client_roles)) + test_dir = misc.get_testdir(self.ctx) + + for id_, remote in clients: + entity = f'client.{id_}' + client_config = self.config.get(entity) + if client_config is None: + client_config = {} + # top level overrides + deep_merge(client_config, top_overrides) + # mount specific overrides + client_config_overrides = overrides.get(entity) + deep_merge(client_config, client_config_overrides) + log.info(f"{entity} config is {client_config}") + + cluster_id = client_config['cluster_id'] + pseudo_path = client_config['pseudo_path'] + nfs_version = client_config.get('version', 'latest') + + try: + first_mon = misc.get_first_mon(self.ctx, None) + (mon0_remote,) = self.ctx.cluster.only(first_mon).remotes.keys() + + proc = mon0_remote.run(args=['ceph', 'nfs', 'export', 'info', cluster_id, pseudo_path], + stdout=StringIO(), wait=True) + res = proc.stdout.getvalue() + export_json = json.loads(res) + log.debug(f'export_json: {export_json}') + + proc = mon0_remote.run(args=['ceph', 'nfs', 'cluster', 'info', cluster_id], + stdout=StringIO(), wait=True) + res = proc.stdout.getvalue() + cluster_info = json.loads(res) + log.debug(f'cluster_info: {cluster_info}') + + info_output = cluster_info[cluster_id]['sufsend'][0] + port = info_output['port'] + ip = info_output['ip'] + + mntpt = os.path.join(test_dir, f'mnt.{id_}') + remote.run(args=['mkdir', '-p', mntpt], timeout=60) + if nfs_version == 'latest': + remote.run(args=['sudo', 'mount', '-t', 'nfs', '-o', + f'port={port}', f'{ip}:{pseudo_path}', mntpt]) + else: + remote.run(args=['sudo', 'mount', '-t', 'nfs', '-o', + f'port={port},vers={nfs_version}', f'{ip}:{pseudo_path}', mntpt]) + remote.run(args=['sudo', 'chmod', '1777', mntpt], timeout=60) + remote.run(args=['stat', mntpt]) + mounts[id_] = (remote, mntpt) + except Exception as e: + log.error(f'failed: {e}') + self.mounts = mounts + + def end(self): + super(GaneshaClient, self).end() + log.debug('unmounting ganesha client(s)') + for (remote, mntpt) in self.mounts.values(): + log.debug(f'unmounting {mntpt}') + remote.run(args=['sudo', 'umount', mntpt]) + self.mounts = {} + +task = GaneshaClient diff --git a/qa/tasks/ganesha_reconf.py b/qa/tasks/ganesha_reconf.py new file mode 100644 index 00000000000..c8685c25186 --- /dev/null +++ b/qa/tasks/ganesha_reconf.py @@ -0,0 +1,72 @@ +""" +reconfigure a ganesha server +""" + +import json +import logging +from io import StringIO + +from teuthology.misc import deep_merge +from teuthology.task import Task +from teuthology import misc + +log = logging.getLogger(__name__) + +class GaneshaReconf(Task): + def __init__(self, ctx, config): + super(GaneshaReconf, self).__init__(ctx, config) + self.log = log + + def setup(self): + super(GaneshaReconf, self).setup() + + def begin(self): + super(GaneshaReconf, self).begin() + log.info('reconfiguring ganesha server') + + ganesha_config = self.config + log.info(f'ganesha_config is {ganesha_config}') + overrides = self.ctx.config.get('overrides', {}).get('ganesha-reconf', {}) + log.info(f'overrides is {overrides}') + + deep_merge(ganesha_config, overrides) + log.info(f'ganesha_config is {ganesha_config}') + + try: + first_mon = misc.get_first_mon(self.ctx, None) + (mon0_remote,) = self.ctx.cluster.only(first_mon).remotes.keys() + + cluster_id = ganesha_config['cluster_id'] + pseudo_path = ganesha_config['pseudo_path'] + + proc = mon0_remote.run(args=['ceph', 'nfs', 'export', 'info', cluster_id, pseudo_path], + stdout=StringIO(), wait=True) + res = proc.stdout.getvalue() + export_json = json.loads(res) + log.debug(f'export_json: {export_json}') + + ceph_section = {'async': False, 'zerocopy': False} + is_async = ganesha_config.get('async', False) + if is_async: + ceph_section["async"] = True + is_zerocopy = ganesha_config.get('zerocopy', False) + if is_zerocopy: + ceph_section["zerocopy"] = True + + new_export = {} + if "export" in export_json.keys(): + new_export = export_json + else: + new_export["export"] = export_json + new_export["ceph"] = ceph_section + + log.debug(f'new_export is {json.dumps(new_export)}') + mon0_remote.run(args=['ceph', 'nfs', 'export', 'apply', cluster_id, "-i", "-"], + stdin=json.dumps(new_export)) + except Exception as e: + log.error(f'failed: {e}') + + def end(self): + super(GaneshaReconf, self).end() + +task = GaneshaReconf