From 0c8899c985763826fb77fed599eeea5624263a35 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Tue, 17 Dec 2019 15:10:34 -0800 Subject: [PATCH] qa: add upgrade test for volume upgrade from legacy This tests that volumes created using the ceph_volume_client.py library continue to be accessible/function via the Nautilus/Octopus ceph-mgr volumes plugin. Fixes: https://tracker.ceph.com/issues/42723 Signed-off-by: Patrick Donnelly --- qa/cephfs/clusters/1-mds-1-client-micro.yaml | 7 ++ qa/suites/fs/upgrade/volumes/.qa | 1 + qa/suites/fs/upgrade/volumes/import-legacy/% | 0 .../fs/upgrade/volumes/import-legacy/.qa | 1 + .../import-legacy/bluestore-bitmap.yaml | 1 + .../volumes/import-legacy/clusters/.qa | 1 + .../clusters/1-mds-2-client-micro.yaml | 7 ++ .../fs/upgrade/volumes/import-legacy/conf | 1 + .../upgrade/volumes/import-legacy/overrides/+ | 0 .../volumes/import-legacy/overrides/.qa | 1 + .../import-legacy/overrides/frag_enable.yaml | 1 + .../import-legacy/overrides/pg-warn.yaml | 5 + .../overrides/whitelist_health.yaml | 1 + .../whitelist_wrongly_marked_down.yaml | 1 + .../fs/upgrade/volumes/import-legacy/tasks/% | 0 .../upgrade/volumes/import-legacy/tasks/.qa | 1 + .../volumes/import-legacy/tasks/0-mimic.yaml | 42 +++++++ .../volumes/import-legacy/tasks/1-client.yaml | 33 ++++++ .../import-legacy/tasks/2-upgrade.yaml | 54 +++++++++ .../volumes/import-legacy/tasks/3-verify.yaml | 25 ++++ .../volumes/import-legacy/ubuntu_18.04.yaml | 1 + qa/tasks/ceph_fuse.py | 20 ++-- qa/tasks/cephfs/fuse_mount.py | 13 ++- qa/workunits/fs/upgrade/volume_client | 110 ++++++++++++++++++ 24 files changed, 317 insertions(+), 10 deletions(-) create mode 100644 qa/cephfs/clusters/1-mds-1-client-micro.yaml create mode 120000 qa/suites/fs/upgrade/volumes/.qa create mode 100644 qa/suites/fs/upgrade/volumes/import-legacy/% create mode 120000 qa/suites/fs/upgrade/volumes/import-legacy/.qa create mode 120000 qa/suites/fs/upgrade/volumes/import-legacy/bluestore-bitmap.yaml create mode 120000 qa/suites/fs/upgrade/volumes/import-legacy/clusters/.qa create mode 100644 qa/suites/fs/upgrade/volumes/import-legacy/clusters/1-mds-2-client-micro.yaml create mode 120000 qa/suites/fs/upgrade/volumes/import-legacy/conf create mode 100644 qa/suites/fs/upgrade/volumes/import-legacy/overrides/+ create mode 120000 qa/suites/fs/upgrade/volumes/import-legacy/overrides/.qa create mode 120000 qa/suites/fs/upgrade/volumes/import-legacy/overrides/frag_enable.yaml create mode 100644 qa/suites/fs/upgrade/volumes/import-legacy/overrides/pg-warn.yaml create mode 120000 qa/suites/fs/upgrade/volumes/import-legacy/overrides/whitelist_health.yaml create mode 120000 qa/suites/fs/upgrade/volumes/import-legacy/overrides/whitelist_wrongly_marked_down.yaml create mode 100644 qa/suites/fs/upgrade/volumes/import-legacy/tasks/% create mode 120000 qa/suites/fs/upgrade/volumes/import-legacy/tasks/.qa create mode 100644 qa/suites/fs/upgrade/volumes/import-legacy/tasks/0-mimic.yaml create mode 100644 qa/suites/fs/upgrade/volumes/import-legacy/tasks/1-client.yaml create mode 100644 qa/suites/fs/upgrade/volumes/import-legacy/tasks/2-upgrade.yaml create mode 100644 qa/suites/fs/upgrade/volumes/import-legacy/tasks/3-verify.yaml create mode 120000 qa/suites/fs/upgrade/volumes/import-legacy/ubuntu_18.04.yaml create mode 100755 qa/workunits/fs/upgrade/volume_client diff --git a/qa/cephfs/clusters/1-mds-1-client-micro.yaml b/qa/cephfs/clusters/1-mds-1-client-micro.yaml new file mode 100644 index 00000000000..8b66c39061c --- /dev/null +++ b/qa/cephfs/clusters/1-mds-1-client-micro.yaml @@ -0,0 +1,7 @@ +roles: +- [mon.a, mon.b, mon.c, mgr.x, mds.a, osd.0, osd.1, osd.2, osd.3] +- [client.0] +openstack: +- volumes: # attached to each instance + count: 4 + size: 10 # GB diff --git a/qa/suites/fs/upgrade/volumes/.qa b/qa/suites/fs/upgrade/volumes/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/% b/qa/suites/fs/upgrade/volumes/import-legacy/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/.qa b/qa/suites/fs/upgrade/volumes/import-legacy/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/bluestore-bitmap.yaml b/qa/suites/fs/upgrade/volumes/import-legacy/bluestore-bitmap.yaml new file mode 120000 index 00000000000..17ad98e799e --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/bluestore-bitmap.yaml @@ -0,0 +1 @@ +../../../../../cephfs/objectstore-ec/bluestore-bitmap.yaml \ No newline at end of file diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/clusters/.qa b/qa/suites/fs/upgrade/volumes/import-legacy/clusters/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/clusters/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/clusters/1-mds-2-client-micro.yaml b/qa/suites/fs/upgrade/volumes/import-legacy/clusters/1-mds-2-client-micro.yaml new file mode 100644 index 00000000000..9b443f7d2bf --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/clusters/1-mds-2-client-micro.yaml @@ -0,0 +1,7 @@ +roles: +- [mon.a, mon.b, mon.c, mgr.x, mgr.y, mds.a, mds.b, mds.c, osd.0, osd.1, osd.2, osd.3] +- [client.0, client.1] +openstack: +- volumes: # attached to each instance + count: 4 + size: 10 # GB diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/conf b/qa/suites/fs/upgrade/volumes/import-legacy/conf new file mode 120000 index 00000000000..6d47129847f --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/conf @@ -0,0 +1 @@ +.qa/cephfs/conf/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/overrides/+ b/qa/suites/fs/upgrade/volumes/import-legacy/overrides/+ new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/overrides/.qa b/qa/suites/fs/upgrade/volumes/import-legacy/overrides/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/overrides/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/overrides/frag_enable.yaml b/qa/suites/fs/upgrade/volumes/import-legacy/overrides/frag_enable.yaml new file mode 120000 index 00000000000..34a39a368cf --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/overrides/frag_enable.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/frag_enable.yaml \ No newline at end of file diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/overrides/pg-warn.yaml b/qa/suites/fs/upgrade/volumes/import-legacy/overrides/pg-warn.yaml new file mode 100644 index 00000000000..4ae54a40d31 --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/overrides/pg-warn.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + global: + mon pg warn min per osd: 0 diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/overrides/whitelist_health.yaml b/qa/suites/fs/upgrade/volumes/import-legacy/overrides/whitelist_health.yaml new file mode 120000 index 00000000000..74f39a49b27 --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/overrides/whitelist_health.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/whitelist_health.yaml \ No newline at end of file diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/overrides/whitelist_wrongly_marked_down.yaml b/qa/suites/fs/upgrade/volumes/import-legacy/overrides/whitelist_wrongly_marked_down.yaml new file mode 120000 index 00000000000..b4528c0f8c0 --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/overrides/whitelist_wrongly_marked_down.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/whitelist_wrongly_marked_down.yaml \ No newline at end of file diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/tasks/% b/qa/suites/fs/upgrade/volumes/import-legacy/tasks/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/tasks/.qa b/qa/suites/fs/upgrade/volumes/import-legacy/tasks/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/tasks/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/tasks/0-mimic.yaml b/qa/suites/fs/upgrade/volumes/import-legacy/tasks/0-mimic.yaml new file mode 100644 index 00000000000..1ca8973bbc2 --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/tasks/0-mimic.yaml @@ -0,0 +1,42 @@ +meta: +- desc: | + install ceph/mimic latest +tasks: +- install: + branch: mimic #tag: v13.2.8 + exclude_packages: + - librados3 + - ceph-mgr-dashboard + - ceph-mgr-diskprediction-local + - ceph-mgr-diskprediction-cloud + - ceph-mgr-rook + - ceph-mgr-cephadm + - cephadm + extra_packages: ['librados2'] +- print: "**** done installing mimic" +- ceph: + mon_bind_addrvec: false + mon_bind_msgr2: false + log-whitelist: + - overall HEALTH_ + - \(FS_ + - \(MDS_ + - \(OSD_ + - \(MON_DOWN\) + - \(CACHE_POOL_ + - \(POOL_ + - \(MGR_DOWN\) + - \(PG_ + - \(SMALLER_PGP_NUM\) + - Monitor daemon marked osd + - Behind on trimming + - Manager daemon + conf: + global: + mon warn on pool no app: false + ms bind msgr2: false +- exec: + osd.0: + - ceph osd require-osd-release mimic + - ceph osd set-require-min-compat-client mimic +- print: "**** done ceph" diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/tasks/1-client.yaml b/qa/suites/fs/upgrade/volumes/import-legacy/tasks/1-client.yaml new file mode 100644 index 00000000000..82731071f1d --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/tasks/1-client.yaml @@ -0,0 +1,33 @@ +tasks: +- workunit: + clients: + client.0: + - fs/upgrade/volume_client + env: + ACTION: create +- print: "**** fs/volume_client create" +- ceph-fuse: + client.0: + mount_path: /volumes/_nogroup/vol_isolated + mountpoint: mnt.0 + auth_id: vol_data_isolated + client.1: + mount_path: /volumes/_nogroup/vol_default + mountpoint: mnt.1 + auth_id: vol_default +- print: "**** ceph-fuse vol_isolated" +- workunit: + clients: + client.0: + - fs/upgrade/volume_client + env: + ACTION: populate + cleanup: false +- workunit: + clients: + client.1: + - fs/upgrade/volume_client + env: + ACTION: populate + cleanup: false +- print: "**** fs/volume_client populate" diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/tasks/2-upgrade.yaml b/qa/suites/fs/upgrade/volumes/import-legacy/tasks/2-upgrade.yaml new file mode 100644 index 00000000000..715ef4887bd --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/tasks/2-upgrade.yaml @@ -0,0 +1,54 @@ +overrides: + ceph: + mon_bind_msgr2: false + mon_bind_addrvec: false + log-whitelist: + - scrub mismatch + - ScrubResult + - wrongly marked + - \(POOL_APP_NOT_ENABLED\) + - \(SLOW_OPS\) + - overall HEALTH_ + - \(MON_MSGR2_NOT_ENABLED\) + - slow request + conf: + global: + bluestore warn on legacy statfs: false + bluestore warn on no per pool omap: false + mon: + mon warn on osd down out interval zero: false + +tasks: +- mds_pre_upgrade: +- print: "**** done mds pre-upgrade sequence" +- install.upgrade: + mon.a: +- print: "**** done install.upgrade both hosts" +- ceph.restart: + daemons: [mon.*, mgr.*] + mon-health-to-clog: false + wait-for-healthy: false +- exec: + mon.a: + - ceph config set global mon_warn_on_msgr2_not_enabled false +- ceph.healthy: +- ceph.restart: + daemons: [osd.*] + wait-for-healthy: false + wait-for-osds-up: true +- ceph.stop: [mds.*] +- ceph.restart: + daemons: [mds.*] + wait-for-healthy: false + wait-for-osds-up: true +- exec: + mon.a: + - ceph mon enable-msgr2 + - ceph versions + - ceph osd dump -f json-pretty + - ceph config rm global mon_warn_on_msgr2_not_enabled + - ceph osd require-osd-release octopus + - for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done + #- ceph osd set-require-min-compat-client nautilus +- ceph.healthy: +- print: "**** done ceph.restart" diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/tasks/3-verify.yaml b/qa/suites/fs/upgrade/volumes/import-legacy/tasks/3-verify.yaml new file mode 100644 index 00000000000..003409ca329 --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/tasks/3-verify.yaml @@ -0,0 +1,25 @@ +overrides: + ceph: + log-whitelist: + - missing required features +tasks: +- exec: + mon.a: + - ceph fs dump --format=json-pretty + - ceph fs volume ls + - ceph fs subvolume ls cephfs +- workunit: + clients: + client.0: + - fs/upgrade/volume_client + env: + ACTION: verify + cleanup: false +- workunit: + clients: + client.1: + - fs/upgrade/volume_client + env: + ACTION: verify + cleanup: false +- print: "**** fs/volume_client verify" diff --git a/qa/suites/fs/upgrade/volumes/import-legacy/ubuntu_18.04.yaml b/qa/suites/fs/upgrade/volumes/import-legacy/ubuntu_18.04.yaml new file mode 120000 index 00000000000..cfb85f10ef5 --- /dev/null +++ b/qa/suites/fs/upgrade/volumes/import-legacy/ubuntu_18.04.yaml @@ -0,0 +1 @@ +.qa/distros/all/ubuntu_18.04.yaml \ No newline at end of file diff --git a/qa/tasks/ceph_fuse.py b/qa/tasks/ceph_fuse.py index 03f5a56e4a7..4f06e2b7630 100644 --- a/qa/tasks/ceph_fuse.py +++ b/qa/tasks/ceph_fuse.py @@ -112,30 +112,35 @@ def task(ctx, config): if client_config is None: client_config = {} + auth_id = client_config.get("auth_id", id_) + skip = client_config.get("skip", False) if skip: skipped[id_] = skip continue if id_ not in all_mounts: - fuse_mount = FuseMount(ctx, client_config, testdir, id_, remote) + fuse_mount = FuseMount(ctx, client_config, testdir, auth_id, remote) all_mounts[id_] = fuse_mount else: # Catch bad configs where someone has e.g. tried to use ceph-fuse and kcephfs for the same client assert isinstance(all_mounts[id_], FuseMount) if not config.get("disabled", False) and client_config.get('mounted', True): - mounted_by_me[id_] = all_mounts[id_] + mounted_by_me[id_] = {"config": client_config, "mount": all_mounts[id_]} ctx.mounts = all_mounts # Mount any clients we have been asked to (default to mount all) log.info('Mounting ceph-fuse clients...') - for mount in mounted_by_me.values(): - mount.mount() + for info in mounted_by_me.values(): + config = info["config"] + mount_path = config.get("mount_path") + mountpoint = config.get("mountpoint") + info["mount"].mount(mountpoint=mountpoint, mount_path=mount_path) - for mount in mounted_by_me.values(): - mount.wait_until_mounted() + for info in mounted_by_me.values(): + info["mount"].wait_until_mounted() # Umount any pre-existing clients that we have not been asked to mount for client_id in set(all_mounts.keys()) - set(mounted_by_me.keys()) - set(skipped.keys()): @@ -148,7 +153,8 @@ def task(ctx, config): finally: log.info('Unmounting ceph-fuse clients...') - for mount in mounted_by_me.values(): + for info in mounted_by_me.values(): # Conditional because an inner context might have umounted it + mount = info["mount"] if mount.is_mounted(): mount.umount_wait() diff --git a/qa/tasks/cephfs/fuse_mount.py b/qa/tasks/cephfs/fuse_mount.py index 27d89437df7..d1242ef80f1 100644 --- a/qa/tasks/cephfs/fuse_mount.py +++ b/qa/tasks/cephfs/fuse_mount.py @@ -51,7 +51,7 @@ class FuseMount(CephFSMount): id=self.client_id, remote=self.client_remote, mnt=self.mountpoint)) self.client_remote.run(args=['mkdir', '-p', self.mountpoint], - timeout=(15*60)) + timeout=(15*60), cwd=self.test_dir) run_cmd = [ 'sudo', @@ -78,6 +78,7 @@ class FuseMount(CephFSMount): self.mountpoint, ] + cwd = self.test_dir if self.client_config.get('valgrind') is not None: run_cmd = misc.get_valgrind_args( self.test_dir, @@ -85,6 +86,7 @@ class FuseMount(CephFSMount): run_cmd, self.client_config.get('valgrind'), ) + cwd = None # misc.get_valgrind_args chdir for us run_cmd.extend(fuse_cmd) @@ -116,6 +118,7 @@ class FuseMount(CephFSMount): proc = self.client_remote.run( args=run_cmd, + cwd=cwd, logger=log.getChild('ceph-fuse.{id}'.format(id=self.client_id)), stdin=run.PIPE, wait=False, @@ -184,6 +187,7 @@ class FuseMount(CephFSMount): '--', self.mountpoint, ], + cwd=self.test_dir, stdout=StringIO(), stderr=StringIO(), wait=False, @@ -228,7 +232,7 @@ class FuseMount(CephFSMount): # unrestricted access to the filesystem mount. try: stderr = StringIO() - self.client_remote.run(args=['sudo', 'chmod', '1777', self.mountpoint], timeout=(15*60), stderr=stderr) + self.client_remote.run(args=['sudo', 'chmod', '1777', self.mountpoint], timeout=(15*60), cwd=self.test_dir, stderr=stderr) except run.CommandFailedError: stderr = stderr.getvalue() if "Read-only file system".lower() in stderr.lower(): @@ -237,7 +241,7 @@ class FuseMount(CephFSMount): raise def _mountpoint_exists(self): - return self.client_remote.run(args=["ls", "-d", self.mountpoint], check_status=False, timeout=(15*60)).exitstatus == 0 + return self.client_remote.run(args=["ls", "-d", self.mountpoint], check_status=False, cwd=self.test_dir, timeout=(15*60)).exitstatus == 0 def umount(self): try: @@ -249,6 +253,7 @@ class FuseMount(CephFSMount): '-u', self.mountpoint, ], + cwd=self.test_dir, timeout=(30*60), ) except run.CommandFailedError: @@ -348,6 +353,7 @@ class FuseMount(CephFSMount): '--', self.mountpoint, ], + cwd=self.test_dir, stderr=stderr, timeout=(60*5), check_status=False, @@ -399,6 +405,7 @@ class FuseMount(CephFSMount): '-rf', self.mountpoint, ], + cwd=self.test_dir, timeout=(60*5) ) diff --git a/qa/workunits/fs/upgrade/volume_client b/qa/workunits/fs/upgrade/volume_client new file mode 100755 index 00000000000..37ee954cbb2 --- /dev/null +++ b/qa/workunits/fs/upgrade/volume_client @@ -0,0 +1,110 @@ +#!/bin/bash + +set -ex + +PYTHON="python2" + +function run_payload { + local payload="$1" + sudo "$PYTHON" <&2 + sudo touch -- "$keyring" + sudo ceph-authtool "$keyring" --import-keyring "$T" + rm -f -- "$T" +} + +function conf_keys { + local client="$1" + ls /etc/ceph >&2 + ceph auth get-or-create "client.manila" mds 'allow *' osd 'allow rw' mon 'allow *' | import_key "$client" /etc/ceph/ceph.keyring +} + +function create_data_isolated { + local PAYLOAD=' +vp = VolumePath(None, "vol_isolated") +vc.create_volume(vp, (1<<33), data_isolated=True) +auth_result = vc.authorize(vp, "vol_data_isolated", tenant_id="test") +print("[client.vol_data_isolated]\n\tkey = ", auth_result["auth_key"]) +' + + run_payload "$PAYLOAD" | import_key "vol_data_isolated" +} + +function create_default { + local PAYLOAD=' +vp = VolumePath(None, "vol_default") +vc.create_volume(vp, (1<<33)) +auth_result = vc.authorize(vp, "vol_default", tenant_id="test") +print("[client.vol_default]\n\tkey = ", auth_result["auth_key"]) +' + run_payload "$PAYLOAD" | import_key "vol_default" +} + +function create { + create_data_isolated + create_default +} + +function populate { + pwd + df -h . + ls -l + cp -a /usr/bin . +} + +function verify_data_isolated { + ceph fs subvolume getpath cephfs vol_isolated + stat bin + ls bin | tail +} + +function verify_default { + ceph fs subvolume getpath cephfs vol_default + stat bin + ls bin | tail +} + +function verify { + diff <(ceph fs subvolume ls cephfs | jq -cS 'sort_by(.name)' | tee /dev/stderr) <(printf '[{"name":"vol_isolated"},{"name":"vol_default"}]' | jq -cS 'sort_by(.name)') + verify_data_isolated + verify_default +} + +function main { + if [ "$1" = create ]; then + conf_keys + create + elif [ "$1" = populate ]; then + populate + elif [ "$1" = verify ]; then + # verify (sub)volumes still exist and are configured correctly + verify + else + exit 1 + fi +} + +main "$ACTION" -- 2.47.3