qa/suites/orch/cephadm/mgr-nfs-upgrade: add test for nfs migration

Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/% b/qa/suites/orch/cephadm/mgr-nfs-upgrade/%
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/.qa b/qa/suites/orch/cephadm/mgr-nfs-upgrade/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file

diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_8.2_kubic_stable.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_8.2_kubic_stable.yaml
new file mode 120000 (symlink)
index 0000000..3afeed7
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_8.2_kubic_stable.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_8.2_kubic_stable.yaml
\ No newline at end of file

diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.4.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.4.yaml
new file mode 100644 (file)
index 0000000..991562b
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.4.yaml
@@ -0,0 +1,8 @@
+tasks:
+- cephadm:
+    roleless: true
+    image: docker.io/ceph/ceph:v16.2.4
+    cephadm_branch: v16.2.4
+    cephadm_git_url: https://github.com/ceph/ceph
+    # needed for v16.2.4 due to --skip-admin-label
+    avoid_pacific_features: true

diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.5.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.5.yaml
new file mode 100644 (file)
index 0000000..2568f23
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.5.yaml
@@ -0,0 +1,6 @@
+tasks:
+- cephadm:
+    roleless: true
+    image: docker.io/ceph/ceph:v16.2.5
+    cephadm_branch: v16.2.5
+    cephadm_git_url: https://github.com/ceph/ceph

diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/octopus.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/octopus.yaml
new file mode 100644 (file)
index 0000000..16670e7
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/octopus.yaml
@@ -0,0 +1,8 @@
+tasks:
+- cephadm:
+    roleless: true
+    image: docker.io/ceph/ceph:v15
+    cephadm_branch: octopus
+    cephadm_git_url: https://github.com/ceph/ceph
+    avoid_pacific_features: true
+    add_mons_via_daemon_add: true

diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-start.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-start.yaml
new file mode 100644 (file)
index 0000000..2d9f09a
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-start.yaml
@@ -0,0 +1,29 @@
+tasks:
+- cephadm.shell:
+    host.a:
+      - ceph orch status
+      - ceph orch ps
+      - ceph orch ls
+      - ceph orch host ls
+      - ceph orch device ls
+roles:
+- - host.a
+  - osd.0
+  - osd.1
+  - osd.2
+  - osd.3
+  - client.0
+- - host.b
+  - osd.4
+  - osd.5
+  - osd.6
+  - osd.7
+openstack:
+- volumes: # attached to each instance
+    count: 4
+    size: 10 # GB
+overrides:
+  ceph:
+    conf:
+      osd:
+        osd shutdown pgref assert: true

diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml
new file mode 100644 (file)
index 0000000..eb8fa2a
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml
@@ -0,0 +1,29 @@
+tasks:
+
+# stop kernel nfs server, if running
+- vip.exec:
+    all-hosts:
+      - systemctl stop nfs-server
+
+- cephadm.shell:
+    host.a:
+      - ceph fs volume create foofs
+
+- cephadm.wait_for_service:
+    service: mds.foofs
+
+- cephadm.shell:
+    host.a:
+      - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2
+      - ceph nfs export create cephfs foofs foo --binding /fake || ceph nfs export create cephfs foofs foo --pseudo-path /fake 
+
+      # we can't do wait_for_service here because with octopus it's nfs.ganesha-foo not nfs.foo
+      - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done
+
+- vip.exec:
+    host.a:
+      - mkdir /mnt/foo
+      - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done
+      - echo test > /mnt/foo/testfile
+      - sync
+

diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml
new file mode 100644 (file)
index 0000000..6d10b35
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml
@@ -0,0 +1,41 @@
+tasks:
+- parallel:
+  - upgrade-tasks
+  - workload-tasks
+
+upgrade-tasks:
+  sequential:
+  - cephadm.shell:
+      env: [sha1]
+      host.a:
+        - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
+        - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
+        - ceph config set global log_to_journald false --force
+        - ceph mgr module enable nfs --force
+        - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
+  - cephadm.shell:
+      env: [sha1]
+      host.a:
+        - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+        - ceph orch ps
+        - ceph versions
+        - echo "wait for servicemap items w/ changing names to refresh"
+        - sleep 60
+        - ceph orch ps
+        - ceph versions
+        - ceph versions | jq -e '.overall | length == 1'
+        - ceph versions | jq -e '.overall | keys' | grep $sha1
+
+  # this should be a no-op, but confirms nfs.ganesha-foo was remapped to nfs.foo
+  - cephadm.wait_for_service:
+      service: nfs.foo
+
+workload-tasks:
+  sequential:
+  - exec:
+      host.a:
+      - cd /mnt/foo && dbench 5 -t 600 || true   # might fail with ESTALE
+      # make sure mount works
+      - umount /mnt/foo
+      - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done
+      - cd /mnt/foo && dbench 5 -t 5

diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml
new file mode 100644 (file)
index 0000000..11e8bb3
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml
@@ -0,0 +1,10 @@
+tasks:
+- vip.exec:
+    host.a:
+      - umount /mnt/foo
+- cephadm.shell:
+    host.a:
+      - ceph nfs cluster ls | grep foo
+      - ceph nfs export ls foo --detailed
+      - rados -p .nfs --all ls -
+      - ceph config get mgr mgr/cephadm/migration_current | grep 3