From: Nitzan Mordechai Date: Wed, 21 Dec 2022 07:06:21 +0000 (+0000) Subject: rados: upgrade suite test filestore removal X-Git-Tag: v18.1.0~304^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d8202bc67c3979880f7692e3315fd48005b8f5f8;p=ceph.git rados: upgrade suite test filestore removal When upgrading osd with filestore to reef, restart should not be possible the osd won't boot and error message will be showed in the osd log Signed-off-by: Nitzan Mordechai --- diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/% b/qa/suites/upgrade/quincy-x/filestore-remove-check/% new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/.qa b/qa/suites/upgrade/quincy-x/filestore-remove-check/.qa new file mode 120000 index 000000000000..fea2489fdf6d --- /dev/null +++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/+ b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/+ new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/.qa b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/.qa new file mode 120000 index 000000000000..a602a0353e75 --- /dev/null +++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/openstack.yaml b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/openstack.yaml new file mode 100644 index 000000000000..5caffc353f96 --- /dev/null +++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/openstack.yaml @@ -0,0 +1,6 @@ +openstack: + - machine: + disk: 100 # GB + - volumes: # attached to each instance + count: 4 + size: 30 # GB diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/start.yaml b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/start.yaml new file mode 100644 index 000000000000..57e455ba78d6 --- /dev/null +++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/start.yaml @@ -0,0 +1,33 @@ +meta: +- desc: | + Run ceph on one nodes, + Use xfs beneath the osds. upgrade to reef + should fail to start the osds with filestore +overrides: + ceph: + mon_bind_msgr2: false + mon_bind_addrvec: false + mon-health-to-clog: false + wait-for-healthy: false + wait-for-osds-up: false + wait-for-scrub: false + skip_stop_pg_num_changes: true + fs: xfs + log-ignorelist: + - overall HEALTH_ + - \(MON_DOWN\) + - \(MGR_DOWN\) + - slow request + - \(MON_MSGR2_NOT_ENABLED\) + conf: + global: + enable experimental unrecoverable data corrupting features: "*" + mon warn on msgr2 not enabled: false + mon: + mon warn on osd down out interval zero: false +roles: +- - mon.a + - mgr.x + - osd.0 + - osd.1 + - osd.2 \ No newline at end of file diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/1-ceph-install/.qa b/qa/suites/upgrade/quincy-x/filestore-remove-check/1-ceph-install/.qa new file mode 120000 index 000000000000..a602a0353e75 --- /dev/null +++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/1-ceph-install/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/1-ceph-install/quincy.yaml b/qa/suites/upgrade/quincy-x/filestore-remove-check/1-ceph-install/quincy.yaml new file mode 100644 index 000000000000..471bd61df1b2 --- /dev/null +++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/1-ceph-install/quincy.yaml @@ -0,0 +1,32 @@ +meta: +- desc: install ceph/quincy latest +tasks: +- install: + exclude_packages: + - ceph-mgr-cephadm + - cephadm + - libcephfs-dev + branch: quincy +- print: "**** done install quincy" +- ceph: + create_rbd_pool: false + conf: + global: + bluestore_warn_on_legacy_statfs: false + bluestore warn on no per pool omap: false + mon pg warn min per osd: 0 + mon: + mon_warn_on_insecure_global_id_reclaim: false + mon_warn_on_insecure_global_id_reclaim_allowed: false + log-ignorelist: + - Not found or unloadable + - evicting unresponsive client +- exec: + osd.0: + - ceph osd require-osd-release quincy +- print: "**** done ceph" +overrides: + ceph: + conf: + mon: + mon warn on osd down out interval zero: false diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/2 - upgrade.yaml b/qa/suites/upgrade/quincy-x/filestore-remove-check/2 - upgrade.yaml new file mode 100644 index 000000000000..6aa429f18b52 --- /dev/null +++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/2 - upgrade.yaml @@ -0,0 +1,20 @@ +meta: +- desc: | + install upgrade ceph/-x on cluster + restart : mons, osd.* +tasks: +- install.upgrade: + mon.a: +- exec: + osd.0: + - ceph osd require-osd-release quincy +- print: "**** done install.upgrade of nodes" +- ceph.restart: + daemons: [mon.a,mgr.x,osd.0,osd.1,osd.2] + mon-health-to-clog: false + wait-for-healthy: false + wait-for-osds-up: false + wait-for-scrub: false + skip_stop_pg_num_changes: true + expected-failure: "FileStore has been deprecated and is no longer supported" +- print: "**** done ceph.restart of all mons and osds" diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/objectstore/.qa b/qa/suites/upgrade/quincy-x/filestore-remove-check/objectstore/.qa new file mode 120000 index 000000000000..a602a0353e75 --- /dev/null +++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/objectstore/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/objectstore/filestore-xfs.yaml b/qa/suites/upgrade/quincy-x/filestore-remove-check/objectstore/filestore-xfs.yaml new file mode 100644 index 000000000000..b6ef47b06f74 --- /dev/null +++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/objectstore/filestore-xfs.yaml @@ -0,0 +1,14 @@ +overrides: + ceph: + fs: xfs + conf: + osd: + osd objectstore: filestore + osd sloppy crc: true + ceph-deploy: + fs: xfs + filestore: True + conf: + osd: + osd objectstore: filestore + osd sloppy crc: true \ No newline at end of file diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/ubuntu_20.04.yaml b/qa/suites/upgrade/quincy-x/filestore-remove-check/ubuntu_20.04.yaml new file mode 100644 index 000000000000..e1374c4105d8 --- /dev/null +++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/ubuntu_20.04.yaml @@ -0,0 +1,9 @@ +os_type: ubuntu +os_version: "20.04" +# the normal ubuntu 20.04 kernel (5.4.0-88-generic currently) have a bug that prevents the nvme_loop +# from behaving. I think it is this: +# https://lkml.org/lkml/2020/9/21/1456 +# (at least, that is the symptom: nvme nvme1: Connect command failed, error wo/DNR bit: 880) +overrides: + kernel: + hwe: true \ No newline at end of file diff --git a/qa/tasks/ceph.py b/qa/tasks/ceph.py index b10bc6723108..08d66e60d4cf 100644 --- a/qa/tasks/ceph.py +++ b/qa/tasks/ceph.py @@ -1585,6 +1585,20 @@ def restart(ctx, config): if config.get('wait-for-osds-up', False): for cluster in clusters: ctx.managers[cluster].wait_for_all_osds_up() + if config.get('expected-failure') is not None: + log.info('Checking for expected-failure in osds logs after restart...') + expected_fail = config.get('expected-failure') + is_osd = teuthology.is_type('osd') + for role in daemons: + if not is_osd(role): + continue + (remote,) = ctx.cluster.only(role).remotes.keys() + cluster, type_, id_ = teuthology.split_role(role) + remote.run( + args = ['sudo', + 'egrep', expected_fail, + '/var/log/ceph/{cluster}-{type_}.{id_}.log'.format(cluster=cluster, type_=type_, id_=id_), + ]) yield @@ -1899,7 +1913,8 @@ def task(ctx, config): finally: # set pg_num_targets back to actual pg_num, so we don't have to # wait for pending merges (which can take a while!) - ctx.managers[config['cluster']].stop_pg_num_changes() + if not config.get('skip_stop_pg_num_changes', True): + ctx.managers[config['cluster']].stop_pg_num_changes() if config.get('wait-for-scrub', True): # wait for pgs to become active+clean in case any