From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 12 Jul 2018 13:25:53 +0000 (-0700)
Subject: qa: add tests for client features
X-Git-Tag: v13.2.2~8^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=9aab68cee81bbcec917dc4b4066436b6de5813ca;p=ceph.git

qa: add tests for client features

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 441a2730e5eed333589d6b536cec3745b3827b23)
---

diff --git a/qa/cephfs/clusters/1-mds-1-client.yaml b/qa/cephfs/clusters/1-mds-1-client.yaml
index e64b0b88d26a..18d18f5d052a 100644
--- a/qa/cephfs/clusters/1-mds-1-client.yaml
+++ b/qa/cephfs/clusters/1-mds-1-client.yaml
@@ -1,6 +1,6 @@
 roles:
-- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3]
-- [mon.b, mon.c, mgr.x, mds.a-s, osd.4, osd.5, osd.6, osd.7]
+- [mon.a, mgr.y, mds.a, mds.x-s, osd.0, osd.1, osd.2, osd.3]
+- [mon.b, mon.c, mgr.x, mds.y-s, osd.4, osd.5, osd.6, osd.7]
 - [client.0]
 openstack:
 - volumes: # attached to each instance
diff --git a/qa/cephfs/clusters/1-mds-2-client-micro.yaml b/qa/cephfs/clusters/1-mds-2-client-micro.yaml
new file mode 100644
index 000000000000..42ad4efcd23d
--- /dev/null
+++ b/qa/cephfs/clusters/1-mds-2-client-micro.yaml
@@ -0,0 +1,8 @@
+roles:
+- [mon.a, mon.b, mon.c, mgr.x, mgr.y, mds.a, mds.b, mds.c, osd.0, osd.1, osd.2, osd.3]
+- [client.0]
+- [client.1]
+openstack:
+- volumes: # attached to each instance
+    count: 4
+    size: 10 # GB
diff --git a/qa/cephfs/clusters/1-mds-2-client.yaml b/qa/cephfs/clusters/1-mds-2-client.yaml
index 006e15a7b23d..76942947d6fa 100644
--- a/qa/cephfs/clusters/1-mds-2-client.yaml
+++ b/qa/cephfs/clusters/1-mds-2-client.yaml
@@ -1,6 +1,6 @@
 roles:
-- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3]
-- [mon.b, mon.c, mgr.x, mds.a-s, osd.4, osd.5, osd.6, osd.7]
+- [mon.a, mgr.y, mds.a, mds.x-s, osd.0, osd.1, osd.2, osd.3]
+- [mon.b, mon.c, mgr.x, mds.y-s, osd.4, osd.5, osd.6, osd.7]
 - [client.0]
 - [client.1]
 openstack:
diff --git a/qa/cephfs/clusters/1-mds-4-client.yaml b/qa/cephfs/clusters/1-mds-4-client.yaml
index a6be36deaf8d..95b732b25c28 100644
--- a/qa/cephfs/clusters/1-mds-4-client.yaml
+++ b/qa/cephfs/clusters/1-mds-4-client.yaml
@@ -1,6 +1,6 @@
 roles:
-- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3]
-- [mon.b, mon.c, mgr.x, mds.a-s, osd.4, osd.5, osd.6, osd.7]
+- [mon.a, mgr.y, mds.a, mds.x-s, osd.0, osd.1, osd.2, osd.3]
+- [mon.b, mon.c, mgr.x, mds.y-s, osd.4, osd.5, osd.6, osd.7]
 - [client.0]
 - [client.1]
 - [client.2]
diff --git a/qa/suites/fs/upgrade/featureful_client/.qa b/qa/suites/fs/upgrade/featureful_client/.qa
new file mode 120000
index 000000000000..11a54ed36010
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/.qa
@@ -0,0 +1 @@
+../../../../
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/% b/qa/suites/fs/upgrade/featureful_client/old_client/%
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/.qa b/qa/suites/fs/upgrade/featureful_client/old_client/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/bluestore.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/bluestore.yaml
new file mode 120000
index 000000000000..1728accf9c80
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/bluestore.yaml
@@ -0,0 +1 @@
+../../../../../cephfs/objectstore-ec/bluestore.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/clusters/.qa b/qa/suites/fs/upgrade/featureful_client/old_client/clusters/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/clusters/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/clusters/1-mds-2-client-micro.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/clusters/1-mds-2-client-micro.yaml
new file mode 120000
index 000000000000..feb68f34306e
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/clusters/1-mds-2-client-micro.yaml
@@ -0,0 +1 @@
+.qa/cephfs/clusters/1-mds-2-client-micro.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/conf b/qa/suites/fs/upgrade/featureful_client/old_client/conf
new file mode 120000
index 000000000000..6d47129847fa
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/conf
@@ -0,0 +1 @@
+.qa/cephfs/conf/
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/overrides/% b/qa/suites/fs/upgrade/featureful_client/old_client/overrides/%
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/overrides/.qa b/qa/suites/fs/upgrade/featureful_client/old_client/overrides/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/overrides/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/overrides/frag_enable.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/overrides/frag_enable.yaml
new file mode 120000
index 000000000000..34a39a368cf1
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/overrides/frag_enable.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/frag_enable.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/overrides/multimds/no.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/overrides/multimds/no.yaml
new file mode 100644
index 000000000000..c740a450a6f2
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/overrides/multimds/no.yaml
@@ -0,0 +1,3 @@
+overrides:
+  ceph:
+    max_mds: 1
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/overrides/multimds/yes.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/overrides/multimds/yes.yaml
new file mode 100644
index 000000000000..ecf118d9f867
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/overrides/multimds/yes.yaml
@@ -0,0 +1,3 @@
+overrides:
+  ceph:
+    max_mds: 2
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/overrides/whitelist_health.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/overrides/whitelist_health.yaml
new file mode 120000
index 000000000000..74f39a49b27e
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/overrides/whitelist_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/whitelist_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/overrides/whitelist_wrongly_marked_down.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/overrides/whitelist_wrongly_marked_down.yaml
new file mode 120000
index 000000000000..b4528c0f8c09
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/overrides/whitelist_wrongly_marked_down.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/whitelist_wrongly_marked_down.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/tasks/% b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/%
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/tasks/.qa b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/tasks/0-luminous.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/0-luminous.yaml
new file mode 100644
index 000000000000..bf627dc394c9
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/0-luminous.yaml
@@ -0,0 +1,30 @@
+meta:
+- desc: |
+   install ceph/luminous latest
+tasks:
+- install:
+    branch: luminous
+- print: "**** done installing luminous"
+- ceph:
+    log-whitelist:
+      - overall HEALTH_
+      - \(FS_
+      - \(MDS_
+      - \(OSD_
+      - \(MON_DOWN\)
+      - \(CACHE_POOL_
+      - \(POOL_
+      - \(MGR_DOWN\)
+      - \(PG_
+      - \(SMALLER_PGP_NUM\)
+      - Monitor daemon marked osd
+      - Behind on trimming
+      - Manager daemon
+    conf:
+      global:
+        mon warn on pool no app: false
+- exec:
+    osd.0:
+      - ceph osd require-osd-release luminous
+      - ceph osd set-require-min-compat-client luminous
+- print: "**** done ceph"
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/tasks/1-client.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/1-client.yaml
new file mode 100644
index 000000000000..88d686fab12e
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/1-client.yaml
@@ -0,0 +1,8 @@
+tasks:
+- ceph-fuse:
+- print: "**** done luminous client"
+- workunit:
+    clients:
+      all:
+      - suites/fsstress.sh
+- print: "**** done fsstress"
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/tasks/2-upgrade.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/2-upgrade.yaml
new file mode 100644
index 000000000000..5c50d60c4606
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/2-upgrade.yaml
@@ -0,0 +1,12 @@
+tasks:
+- mds_pre_upgrade:
+- print: "**** done mds pre-upgrade sequence"
+- install.upgrade:
+    mon.a:
+    mon.b:
+- print: "**** done install.upgrade both hosts"
+- ceph.stop: [mds.*]
+- ceph.restart:
+    daemons: [mon.*, mgr.*, osd.*, mds.*]
+    mon-health-to-clog: false
+- print: "**** done ceph.restart"
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/tasks/3-compat_client/mimic.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/3-compat_client/mimic.yaml
new file mode 100644
index 000000000000..367206766d8b
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/3-compat_client/mimic.yaml
@@ -0,0 +1,10 @@
+overrides:
+  ceph:
+    log-whitelist:
+      - missing required features
+tasks:
+- exec:
+    mon.a:
+      - ceph fs dump --format=json-pretty
+      - ceph fs set cephfs min_compat_client mimic
+- fs.clients_evicted:
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/tasks/3-compat_client/no.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/3-compat_client/no.yaml
new file mode 100644
index 000000000000..b495eb41b54c
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/3-compat_client/no.yaml
@@ -0,0 +1,6 @@
+tasks:
+- workunit:
+    clients:
+      all:
+      - suites/fsstress.sh
+- print: "**** done fsstress"
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/% b/qa/suites/fs/upgrade/featureful_client/upgraded_client/%
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/.qa b/qa/suites/fs/upgrade/featureful_client/upgraded_client/.qa
new file mode 120000
index 000000000000..fea2489fdf6d
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/bluestore.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/bluestore.yaml
new file mode 120000
index 000000000000..1728accf9c80
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/bluestore.yaml
@@ -0,0 +1 @@
+../../../../../cephfs/objectstore-ec/bluestore.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/clusters/.qa b/qa/suites/fs/upgrade/featureful_client/upgraded_client/clusters/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/clusters/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/clusters/1-mds-2-client-micro.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/clusters/1-mds-2-client-micro.yaml
new file mode 120000
index 000000000000..feb68f34306e
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/clusters/1-mds-2-client-micro.yaml
@@ -0,0 +1 @@
+.qa/cephfs/clusters/1-mds-2-client-micro.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/conf b/qa/suites/fs/upgrade/featureful_client/upgraded_client/conf
new file mode 120000
index 000000000000..6d47129847fa
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/conf
@@ -0,0 +1 @@
+.qa/cephfs/conf/
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/% b/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/%
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/.qa b/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/frag_enable.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/frag_enable.yaml
new file mode 120000
index 000000000000..34a39a368cf1
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/frag_enable.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/frag_enable.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/multimds/no.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/multimds/no.yaml
new file mode 100644
index 000000000000..c740a450a6f2
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/multimds/no.yaml
@@ -0,0 +1,3 @@
+overrides:
+  ceph:
+    max_mds: 1
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/multimds/yes.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/multimds/yes.yaml
new file mode 100644
index 000000000000..ecf118d9f867
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/multimds/yes.yaml
@@ -0,0 +1,3 @@
+overrides:
+  ceph:
+    max_mds: 2
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/whitelist_health.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/whitelist_health.yaml
new file mode 120000
index 000000000000..74f39a49b27e
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/whitelist_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/whitelist_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/whitelist_wrongly_marked_down.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/whitelist_wrongly_marked_down.yaml
new file mode 120000
index 000000000000..b4528c0f8c09
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/overrides/whitelist_wrongly_marked_down.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/whitelist_wrongly_marked_down.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/% b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/%
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/.qa b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/0-luminous.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/0-luminous.yaml
new file mode 100644
index 000000000000..bf627dc394c9
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/0-luminous.yaml
@@ -0,0 +1,30 @@
+meta:
+- desc: |
+   install ceph/luminous latest
+tasks:
+- install:
+    branch: luminous
+- print: "**** done installing luminous"
+- ceph:
+    log-whitelist:
+      - overall HEALTH_
+      - \(FS_
+      - \(MDS_
+      - \(OSD_
+      - \(MON_DOWN\)
+      - \(CACHE_POOL_
+      - \(POOL_
+      - \(MGR_DOWN\)
+      - \(PG_
+      - \(SMALLER_PGP_NUM\)
+      - Monitor daemon marked osd
+      - Behind on trimming
+      - Manager daemon
+    conf:
+      global:
+        mon warn on pool no app: false
+- exec:
+    osd.0:
+      - ceph osd require-osd-release luminous
+      - ceph osd set-require-min-compat-client luminous
+- print: "**** done ceph"
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/1-client.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/1-client.yaml
new file mode 100644
index 000000000000..dc1c0d8d1efc
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/1-client.yaml
@@ -0,0 +1,11 @@
+nuke-on-error: false
+overrides:
+  nuke-on-error: false
+tasks:
+- ceph-fuse:
+- print: "**** done luminous client"
+#- workunit:
+#    clients:
+#      all:
+#      - suites/fsstress.sh
+- print: "**** done fsstress"
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/2-upgrade.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/2-upgrade.yaml
new file mode 100644
index 000000000000..5c50d60c4606
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/2-upgrade.yaml
@@ -0,0 +1,12 @@
+tasks:
+- mds_pre_upgrade:
+- print: "**** done mds pre-upgrade sequence"
+- install.upgrade:
+    mon.a:
+    mon.b:
+- print: "**** done install.upgrade both hosts"
+- ceph.stop: [mds.*]
+- ceph.restart:
+    daemons: [mon.*, mgr.*, osd.*, mds.*]
+    mon-health-to-clog: false
+- print: "**** done ceph.restart"
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/3-client-upgrade.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/3-client-upgrade.yaml
new file mode 100644
index 000000000000..58fdfdbd8d9e
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/3-client-upgrade.yaml
@@ -0,0 +1,14 @@
+tasks:
+- install.upgrade:
+    client.0:
+- print: "**** done install.upgrade on client.0"
+- ceph-fuse:
+    client.0:
+      mounted: false
+    client.1:
+      skip: true
+- ceph-fuse:
+    client.0:
+    client.1:
+      skip: true
+- print: "**** done remount client"
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/4-compat_client.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/4-compat_client.yaml
new file mode 100644
index 000000000000..bdf484da6499
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/4-compat_client.yaml
@@ -0,0 +1,13 @@
+overrides:
+  ceph:
+    log-whitelist:
+      - missing required features
+tasks:
+- exec:
+    mon.a:
+      - ceph fs dump --format=json-pretty
+      - ceph fs set cephfs min_compat_client mimic
+- fs.clients_evicted:
+    clients:
+      client.0: False
+      client.1: True
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/5-client-sanity.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/5-client-sanity.yaml
new file mode 100644
index 000000000000..e206457e6867
--- /dev/null
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/5-client-sanity.yaml
@@ -0,0 +1,6 @@
+tasks:
+- workunit:
+    clients:
+      client.0:
+      - suites/fsstress.sh
+- print: "**** done fsstress"
diff --git a/qa/tasks/ceph_fuse.py b/qa/tasks/ceph_fuse.py
index 14c6989720b5..08254fed3c3d 100644
--- a/qa/tasks/ceph_fuse.py
+++ b/qa/tasks/ceph_fuse.py
@@ -7,7 +7,6 @@ import logging
 
 from teuthology import misc as teuthology
 from cephfs.fuse_mount import FuseMount
-from tasks.cephfs.filesystem import MDSCluster
 from tasks.cephfs.filesystem import Filesystem
 
 log = logging.getLogger(__name__)
@@ -94,24 +93,19 @@ def task(ctx, config):
     :param ctx: Context
     :param config: Configuration
     """
-    log.info('Mounting ceph-fuse clients...')
+    log.info('Running ceph_fuse task...')
 
     testdir = teuthology.get_testdir(ctx)
+    log.info("config is {}".format(str(config)))
     config = get_client_configs(ctx, config)
+    log.info("new config is {}".format(str(config)))
 
     # List clients we will configure mounts for, default is all clients
     clients = list(teuthology.get_clients(ctx=ctx, roles=filter(lambda x: 'client.' in x, config.keys())))
 
     all_mounts = getattr(ctx, 'mounts', {})
     mounted_by_me = {}
-
-    log.info('Wait for MDS to reach steady state...')
-    mds_cluster = MDSCluster(ctx)
-    status = mds_cluster.status()
-    for filesystem in status.get_filesystems():
-        fs = Filesystem(ctx, fscid=filesystem['id']) 
-        fs.wait_for_daemons()
-    log.info('Ready to start ceph-fuse...')
+    skipped = {}
 
     # Construct any new FuseMount instances
     for id_, remote in clients:
@@ -119,8 +113,13 @@ def task(ctx, config):
         if client_config is None:
             client_config = {}
 
+        skip = client_config.get("skip", False)
+        if skip:
+            skipped[id_] = skip
+            continue
+
         if id_ not in all_mounts:
-            fuse_mount = FuseMount(client_config, testdir, id_, remote)
+            fuse_mount = FuseMount(ctx, client_config, testdir, id_, remote)
             all_mounts[id_] = fuse_mount
         else:
             # Catch bad configs where someone has e.g. tried to use ceph-fuse and kcephfs for the same client
@@ -132,6 +131,7 @@ def task(ctx, config):
     ctx.mounts = all_mounts
 
     # Mount any clients we have been asked to (default to mount all)
+    log.info('Mounting ceph-fuse clients...')
     for mount in mounted_by_me.values():
         mount.mount()
 
@@ -139,7 +139,7 @@ def task(ctx, config):
         mount.wait_until_mounted()
 
     # Umount any pre-existing clients that we have not been asked to mount
-    for client_id in set(all_mounts.keys()) - set(mounted_by_me.keys()):
+    for client_id in set(all_mounts.keys()) - set(mounted_by_me.keys()) - set(skipped.keys()):
         mount = all_mounts[client_id]
         if mount.is_mounted():
             mount.umount_wait()
diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py
index 4ff3cc01413f..4b3cffa9ad07 100644
--- a/qa/tasks/cephfs/filesystem.py
+++ b/qa/tasks/cephfs/filesystem.py
@@ -909,8 +909,8 @@ class Filesystem(MDSCluster):
 
         return self.json_asok(command, 'mds', mds_id)
 
-    def rank_asok(self, command, rank=0):
-        info = self.get_rank(rank=rank)
+    def rank_asok(self, command, rank=0, status=None):
+        info = self.get_rank(rank=rank, status=status)
         return self.json_asok(command, 'mds', info['name'])
 
     def read_cache(self, path, depth=None):
diff --git a/qa/tasks/cephfs/fuse_mount.py b/qa/tasks/cephfs/fuse_mount.py
index 30de6051c0b2..4b83355343eb 100644
--- a/qa/tasks/cephfs/fuse_mount.py
+++ b/qa/tasks/cephfs/fuse_mount.py
@@ -10,19 +10,25 @@ from teuthology.contextutil import MaxWhileTries
 from teuthology.orchestra import run
 from teuthology.orchestra.run import CommandFailedError
 from .mount import CephFSMount
+from tasks.cephfs.filesystem import Filesystem
 
 log = logging.getLogger(__name__)
 
 
 class FuseMount(CephFSMount):
-    def __init__(self, client_config, test_dir, client_id, client_remote):
-        super(FuseMount, self).__init__(test_dir, client_id, client_remote)
+    def __init__(self, ctx, client_config, test_dir, client_id, client_remote):
+        super(FuseMount, self).__init__(ctx, test_dir, client_id, client_remote)
 
         self.client_config = client_config if client_config else {}
         self.fuse_daemon = None
         self._fuse_conn = None
+        self.id = None
+        self.inst = None
+        self.addr = None
 
     def mount(self, mount_path=None, mount_fs_name=None):
+        self.setupfs(name=mount_fs_name)
+
         try:
             return self._mount(mount_path, mount_fs_name)
         except RuntimeError:
@@ -152,6 +158,20 @@ class FuseMount(CephFSMount):
         else:
             self._fuse_conn = new_conns[0]
 
+        status = self.admin_socket(['status'])
+        self.id = status['id']
+        try:
+            self.inst = status['inst_str']
+            self.addr = status['addr_str']
+        except KeyError as e:
+            sessions = self.fs.rank_asok(['session', 'ls'])
+            for s in sessions:
+                if s['id'] == self.id:
+                    self.inst = s['inst']
+                    self.addr = self.inst.split()[1]
+            if self.inst is None:
+                raise RuntimeError("cannot find client session")
+
     def is_mounted(self):
         proc = self.client_remote.run(
             args=[
@@ -260,6 +280,9 @@ class FuseMount(CephFSMount):
 
         assert not self.is_mounted()
         self._fuse_conn = None
+        self.id = None
+        self.inst = None
+        self.addr = None
 
     def umount_wait(self, force=False, require_clean=False, timeout=900):
         """
@@ -411,6 +434,18 @@ print find_socket("{client_name}")
         """
         return self.admin_socket(['mds_sessions'])['id']
 
+    def get_global_inst(self):
+        """
+        Look up the CephFS client instance for this mount
+        """
+        return self.inst
+
+    def get_global_addr(self):
+        """
+        Look up the CephFS client addr for this mount
+        """
+        return self.addr
+
     def get_client_pid(self):
         """
         return pid of ceph-fuse process
diff --git a/qa/tasks/cephfs/kernel_mount.py b/qa/tasks/cephfs/kernel_mount.py
index d237f65fb335..3f10657b8558 100644
--- a/qa/tasks/cephfs/kernel_mount.py
+++ b/qa/tasks/cephfs/kernel_mount.py
@@ -17,9 +17,9 @@ UMOUNT_TIMEOUT = 300
 
 
 class KernelMount(CephFSMount):
-    def __init__(self, mons, test_dir, client_id, client_remote,
+    def __init__(self, ctx, mons, test_dir, client_id, client_remote,
                  ipmi_user, ipmi_password, ipmi_domain):
-        super(KernelMount, self).__init__(test_dir, client_id, client_remote)
+        super(KernelMount, self).__init__(ctx, test_dir, client_id, client_remote)
         self.mons = mons
 
         self.mounted = False
@@ -46,6 +46,8 @@ class KernelMount(CephFSMount):
         )
 
     def mount(self, mount_path=None, mount_fs_name=None):
+        self.setupfs(name=mount_fs_name)
+
         log.info('Mounting kclient client.{id} at {remote} {mnt}...'.format(
             id=self.client_id, remote=self.client_remote, mnt=self.mountpoint))
 
diff --git a/qa/tasks/cephfs/mount.py b/qa/tasks/cephfs/mount.py
index 4f96e6cdc217..fbff376041ef 100644
--- a/qa/tasks/cephfs/mount.py
+++ b/qa/tasks/cephfs/mount.py
@@ -8,22 +8,25 @@ import os
 from StringIO import StringIO
 from teuthology.orchestra import run
 from teuthology.orchestra.run import CommandFailedError, ConnectionLostError
+from tasks.cephfs.filesystem import Filesystem
 
 log = logging.getLogger(__name__)
 
 
 class CephFSMount(object):
-    def __init__(self, test_dir, client_id, client_remote):
+    def __init__(self, ctx, test_dir, client_id, client_remote):
         """
         :param test_dir: Global teuthology test dir
         :param client_id: Client ID, the 'foo' in client.foo
         :param client_remote: Remote instance for the host where client will run
         """
 
+        self.ctx = ctx
         self.test_dir = test_dir
         self.client_id = client_id
         self.client_remote = client_remote
         self.mountpoint_dir_name = 'mnt.{id}'.format(id=self.client_id)
+        self.fs = None
 
         self.test_files = ['a', 'b', 'c']
 
@@ -37,6 +40,12 @@ class CephFSMount(object):
     def is_mounted(self):
         raise NotImplementedError()
 
+    def setupfs(self, name=None):
+        self.fs = Filesystem(self.ctx, name=name)
+        log.info('Wait for MDS to reach steady state...')
+        self.fs.wait_for_daemons()
+        log.info('Ready to start {}...'.format(type(self).__name__))
+
     def mount(self, mount_path=None, mount_fs_name=None):
         raise NotImplementedError()
 
@@ -91,6 +100,14 @@ class CephFSMount(object):
         finally:
             self.umount_wait()
 
+    def is_blacklisted(self):
+        addr = self.get_global_addr()
+        blacklist = json.loads(self.fs.mon_manager.raw_cluster_cmd("osd", "blacklist", "ls", "--format=json"))
+        for b in blacklist:
+            if addr == b["addr"]:
+                return True
+        return False
+
     def create_files(self):
         assert(self.is_mounted())
 
@@ -459,6 +476,12 @@ class CephFSMount(object):
     def get_global_id(self):
         raise NotImplementedError()
 
+    def get_global_inst(self):
+        raise NotImplementedError()
+
+    def get_global_addr(self):
+        raise NotImplementedError()
+
     def get_osd_epoch(self):
         raise NotImplementedError()
 
diff --git a/qa/tasks/fs.py b/qa/tasks/fs.py
new file mode 100644
index 000000000000..4286318527e7
--- /dev/null
+++ b/qa/tasks/fs.py
@@ -0,0 +1,67 @@
+"""
+CephFS sub-tasks.
+"""
+
+import contextlib
+import logging
+import re
+import time
+
+from tasks.cephfs.filesystem import Filesystem
+
+log = logging.getLogger(__name__)
+
+def clients_evicted(ctx, config):
+    """
+    Check clients are evicted, unmount (cleanup) if so.
+    """
+
+    if config is None:
+        config = {}
+    assert isinstance(config, dict), \
+        'task only accepts a dict for configuration'
+
+    clients = config.get('clients')
+
+    if clients is None:
+        clients = {("client."+client_id): True for client_id in ctx.mounts}
+
+    log.info("clients is {}".format(str(clients)))
+
+    fs = Filesystem(ctx)
+    status = fs.status()
+
+    has_session = set()
+    mounts = {}
+    for client in clients:
+        client_id = re.match("^client.([0-9]+)$", client).groups(1)[0]
+        mounts[client] = ctx.mounts.get(client_id)
+
+    for rank in fs.get_ranks(status=status):
+        ls = fs.rank_asok(['session', 'ls'], rank=rank['rank'], status=status)
+        for session in ls:
+            for client, evicted in clients.viewitems():
+                mount = mounts.get(client)
+                if mount is not None:
+                    global_id = mount.get_global_id()
+                    if session['id'] == global_id:
+                        if evicted:
+                            raise RuntimeError("client still has session: {}".format(str(session)))
+                        else:
+                            log.info("client {} has a session with MDS {}.{}".format(client, fs.id, rank['rank']))
+                            has_session.add(client)
+
+    no_session = set(clients) - has_session
+    should_assert = False
+    for client, evicted in clients.viewitems():
+        mount = mounts.get(client)
+        if mount is not None:
+            if evicted:
+                log.info("confirming client {} is blacklisted".format(client))
+                assert mount.is_blacklisted()
+            elif client in no_session:
+                log.info("client {} should not be evicted but has no session with an MDS".format(client))
+                mount.is_blacklisted() # for debugging
+                should_assert = True
+    if should_assert:
+        raise RuntimeError("some clients which should not be evicted have no session with an MDS?")
diff --git a/qa/tasks/kclient.py b/qa/tasks/kclient.py
index 7cc7ada35f4a..5e217b323585 100644
--- a/qa/tasks/kclient.py
+++ b/qa/tasks/kclient.py
@@ -89,6 +89,7 @@ def task(ctx, config):
             continue
 
         kernel_mount = KernelMount(
+            ctx,
             mons,
             test_dir,
             id_,