qa: add tasks to check mds upgrade state

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 5a7382214fe4dbd4b79773c6e732512ade22793a)

diff --git a/qa/tasks/fs.py b/qa/tasks/fs.py
index 71d7b22e814620e168dcb0e04d6bd9beb0871d12..f7a9330e29b50541e97bac6ff3aea3601ba2c9f8 100644 (file)
--- a/qa/tasks/fs.py
+++ b/qa/tasks/fs.py
@@ -9,6 +9,74 @@ from tasks.cephfs.filesystem import Filesystem, MDSCluster
 
 log = logging.getLogger(__name__)
 
+# Everything up to CEPH_MDSMAP_ALLOW_STANDBY_REPLAY
+CEPH_MDSMAP_ALLOW_STANDBY_REPLAY = (1<<5)
+CEPH_MDSMAP_LAST = CEPH_MDSMAP_ALLOW_STANDBY_REPLAY
+UPGRADE_FLAGS_MASK = ((CEPH_MDSMAP_LAST<<1) - 1)
+def pre_upgrade_save(ctx, config):
+    """
+    That the upgrade procedure doesn't clobber state: save state.
+    """
+
+    mdsc = MDSCluster(ctx)
+    status = mdsc.status()
+
+    state = {}
+    ctx['mds-upgrade-state'] = state
+
+    for fs in list(status.get_filesystems()):
+        fscid = fs['id']
+        mdsmap = fs['mdsmap']
+        fs_state = {}
+        fs_state['epoch'] = mdsmap['epoch']
+        fs_state['max_mds'] = mdsmap['max_mds']
+        fs_state['flags'] = mdsmap['flags'] & UPGRADE_FLAGS_MASK
+        state[fscid] = fs_state
+        log.debug(f"fs fscid={fscid},name={mdsmap['fs_name']} state = {fs_state}")
+
+
+def post_upgrade_checks(ctx, config):
+    """
+    That the upgrade procedure doesn't clobber state.
+    """
+
+    state = ctx['mds-upgrade-state']
+
+    mdsc = MDSCluster(ctx)
+    status = mdsc.status()
+
+    for fs in list(status.get_filesystems()):
+        fscid = fs['id']
+        mdsmap = fs['mdsmap']
+        fs_state = state[fscid]
+        log.debug(f"checking fs fscid={fscid},name={mdsmap['fs_name']} state = {fs_state}")
+
+        # check state was restored to previous values
+        assert fs_state['max_mds'] == mdsmap['max_mds']
+        assert fs_state['flags'] == (mdsmap['flags'] & UPGRADE_FLAGS_MASK)
+
+        # now confirm that the upgrade procedure was followed
+        epoch = mdsmap['epoch']
+        pre_upgrade_epoch = fs_state['epoch']
+        assert pre_upgrade_epoch < epoch
+        should_decrease_max_mds = fs_state['max_mds'] > 1
+        did_decrease_max_mds = False
+        should_disable_allow_standby_replay = fs_state['flags'] & CEPH_MDSMAP_ALLOW_STANDBY_REPLAY
+        did_disable_allow_standby_replay = False
+        for i in range(pre_upgrade_epoch+1, mdsmap['epoch']):
+            old_status = mdsc.status(epoch=i)
+            old_fs = old_status.get_fsmap(fscid)
+            old_mdsmap = old_fs['mdsmap']
+            if should_decrease_max_mds and old_mdsmap['max_mds'] == 1:
+                log.debug(f"max_mds reduced in epoch {i}")
+                did_decrease_max_mds = True
+            if should_disable_allow_standby_replay and not (old_mdsmap['flags'] & CEPH_MDSMAP_ALLOW_STANDBY_REPLAY):
+                log.debug(f"allow_standby_replay disabled in epoch {i}")
+                did_disable_allow_standby_replay = True
+        assert not should_decrease_max_mds or did_decrease_max_mds
+        assert not should_disable_allow_standby_replay or did_disable_allow_standby_replay
+
+
 def ready(ctx, config):
     """
     That the file system is ready for clients.