qa: add upgrade suite for cephx

To test upgrade paths for "aes" key type to "aes256k" including the expected
flows for service key updates and entity rotation.

Signed-off-by: Patrick Donnelly <pdonnell@ibm.com>

diff --git a/qa/suites/rados/upgrade/cephx/.qa b/qa/suites/rados/upgrade/cephx/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/X/% b/qa/suites/rados/upgrade/cephx/X/%
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/rados/upgrade/cephx/X/.qa b/qa/suites/rados/upgrade/cephx/X/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/X/centos_latest.yaml b/qa/suites/rados/upgrade/cephx/X/centos_latest.yaml
new file mode 120000 (symlink)
index 0000000..bd9854e
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/centos_latest.yaml
@@ -0,0 +1 @@
+.qa/distros/supported/centos_latest.yaml
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/X/conf.yaml b/qa/suites/rados/upgrade/cephx/X/conf.yaml
new file mode 100644 (file)
index 0000000..cdae164
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/conf.yaml
@@ -0,0 +1,13 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        debug ms: 1
+        debug auth: 30
+        debug monc: 30
+        log to file: true
+        auth mon ticket ttl: 300
+        auth service ticket ttl: 120
+      mon:
+        debug mon: 30
+        debug ms: 5

diff --git a/qa/suites/rados/upgrade/cephx/X/ignore_health.yaml b/qa/suites/rados/upgrade/cephx/X/ignore_health.yaml
new file mode 100644 (file)
index 0000000..e96acf5
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/ignore_health.yaml
@@ -0,0 +1,25 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - do not have an application enabled
+      - application not enabled
+      - or freeform for custom applications
+      - POOL_APP_NOT_ENABLED
+      - is down
+      - OSD_DOWN
+      - mons down
+      - mon down
+      - MON_DOWN
+      - out of quorum
+      - PG_AVAILABILITY
+      - PG_DEGRADED
+      - Reduced data availability
+      - Degraded data redundancy
+      - pg .* is stuck inactive
+      - pg .* is .*degraded
+      - FS_DEGRADED
+      - OSDMAP_FLAGS
+      - OSD_UPGRADE_FINISHED
+      - OSD_HOST_DOWN
+      - filesystem is degraded
+      - evicting unresponsive client

diff --git a/qa/suites/rados/upgrade/cephx/X/ms_bind/v1.yaml b/qa/suites/rados/upgrade/cephx/X/ms_bind/v1.yaml
new file mode 100644 (file)
index 0000000..439563f
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/ms_bind/v1.yaml
@@ -0,0 +1,17 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        mon warn on msgr2 not enabled: false
+      osd:
+        ms bind msgr1: true
+        ms bind msgr2: false
+      mds:
+        ms bind msgr1: true
+        ms bind msgr2: false
+      mon:
+        ms bind msgr1: true
+        ms bind msgr2: false
+      mgr:
+        ms bind msgr1: true
+        ms bind msgr2: false

diff --git a/qa/suites/rados/upgrade/cephx/X/ms_bind/v2.yaml b/qa/suites/rados/upgrade/cephx/X/ms_bind/v2.yaml
new file mode 100644 (file)
index 0000000..4e81054
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/ms_bind/v2.yaml
@@ -0,0 +1,17 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        mon warn on msgr2 not enabled: false
+      osd:
+        ms bind msgr1: false
+        ms bind msgr2: true
+      mds:
+        ms bind msgr1: false
+        ms bind msgr2: true
+      mon:
+        ms bind msgr1: false
+        ms bind msgr2: true
+      mgr:
+        ms bind msgr1: false
+        ms bind msgr2: true

diff --git a/qa/suites/rados/upgrade/cephx/X/roles.yaml b/qa/suites/rados/upgrade/cephx/X/roles.yaml
new file mode 100644 (file)
index 0000000..4e98cf0
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/roles.yaml
@@ -0,0 +1,4 @@
+roles:
+- [mon.a, mds.a, mgr.x, osd.0, osd.1]
+- [mon.b, mon.c, mds.b, mgr.y, osd.2, osd.3]
+- [client.0]

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/% b/qa/suites/rados/upgrade/cephx/X/tasks/%
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/.qa b/qa/suites/rados/upgrade/cephx/X/tasks/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/0-install.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/0-install.yaml
new file mode 100644 (file)
index 0000000..e6f4644
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/0-install.yaml
@@ -0,0 +1,2 @@
+tasks:
+- install:

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/1-ceph.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/1-ceph.yaml
new file mode 100644 (file)
index 0000000..eefb7a2
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/1-ceph.yaml
@@ -0,0 +1,10 @@
+tasks:
+- ceph:
+    cluster-conf:
+      global:
+        auth service cipher: aes
+        cephx preferred cipher: aes
+      mon:
+        mon auth allow insecure key: true
+    cephx:
+      key_type: aes

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/2-debug.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/2-debug.yaml
new file mode 100644 (file)
index 0000000..f0352e6
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/2-debug.yaml
@@ -0,0 +1,13 @@
+tasks:
+- full_sequential_finally:
+  - exec:
+      mon.a:
+        - sleep 1m
+- exec:
+    mon.a:
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 tell mon.a config diff
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 tell mon.b config diff
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 tell mon.c config diff
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 config dump
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 auth ls
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 auth --format=json dump-keys

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/3-workload/.qa b/qa/suites/rados/upgrade/cephx/X/tasks/3-workload/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/3-workload/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/3-workload/radosbench.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/3-workload/radosbench.yaml
new file mode 100644 (file)
index 0000000..cc18fab
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/3-workload/radosbench.yaml
@@ -0,0 +1,11 @@
+radosbench:
+  sequential_yield:
+    - radosbench:
+        extra_args: --log-to-stderr=false --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 --mon_client_target_rank=0
+        clients: [client.0]
+        time: 300
+        unique_pool: true
+
+tasks:
+  - sequential_yield:
+      - radosbench

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/4-cephx-service-upgrade.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/4-cephx-service-upgrade.yaml
new file mode 100644 (file)
index 0000000..c193a55
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/4-cephx-service-upgrade.yaml
@@ -0,0 +1,14 @@
+tasks:
+- ceph.key_rotate:
+    daemons: [mon.*]
+    key_type: aes256k
+- ceph.key_rotate:
+    daemons: [mgr.*, osd.*, mds.*]
+    key_type: aes256k
+- exec:
+    mon.a:
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 auth ls
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 auth --format=json-pretty dump-keys
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 auth --format=json dump-keys | jq 'any(.data.secrets[] | select(.key.type == 1 or .key.type == 2 or .key.type == 4 or .key.type == 16); .val.key.type != 2)'
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 auth --format=json dump-keys | jq '.data.rotating_secrets | all( .val.secrets | all(.val.key.type == 1) )'
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 config set global auth_service_cipher aes256k

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/.qa b/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/no.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/no.yaml
new file mode 100644 (file)
index 0000000..2290d55
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/no.yaml
@@ -0,0 +1,3 @@
+teuthology:
+  variables:
+    wipe_service_keys: false

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/yes.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/yes.yaml
new file mode 100644 (file)
index 0000000..b766074
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/yes.yaml
@@ -0,0 +1,6 @@
+tasks:
+- exec:
+    mon.a:
+      - ceph auth wipe-rotating-service-keys
+      - ceph auth --format=json-pretty dump-keys
+      - ceph auth --format=json dump-keys | jq '.data.rotating_secrets | all(  if .key == 32 then  (.val.secrets | all(.val.key.type == 1))  else  (.val.secrets | all(.val.key.type == 2))  end  )'

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/.qa b/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/no.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/no.yaml
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/sleep.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/sleep.yaml
new file mode 100644 (file)
index 0000000..398439f
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/sleep.yaml
@@ -0,0 +1,4 @@
+# Sleep for ticket refresh.
+tasks:
+- sleep:
+    duration: 500

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/yes.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/yes.yaml
new file mode 100644 (file)
index 0000000..56c7271
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/yes.yaml
@@ -0,0 +1,25 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - OSD_DOWN
+      - OSD_ROOT_DOWN
+tasks:
+- sleep:
+    duration: 10
+- ceph.restart:
+    daemons: [mon.*]
+    mon-health-to-clog: false
+    wait-for-healthy: true
+- ceph.restart:
+    daemons: [mgr.*]
+    mon-health-to-clog: false
+    wait-for-healthy: true
+- ceph.restart:
+    daemons: [osd.*]
+    mon-health-to-clog: false
+    wait-for-healthy: true
+    wait-for-osds-up: true
+- ceph.restart:
+    daemons: [mds.*]
+    mon-health-to-clog: false
+    wait-for-healthy: true

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/7-workload-after-rotate/.qa b/qa/suites/rados/upgrade/cephx/X/tasks/7-workload-after-rotate/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/7-workload-after-rotate/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/7-workload-after-rotate/radosbench.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/7-workload-after-rotate/radosbench.yaml
new file mode 100644 (file)
index 0000000..55bd378
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/X/tasks/7-workload-after-rotate/radosbench.yaml
@@ -0,0 +1,6 @@
+tasks:
+  - radosbench:
+      extra_args: --log-to-stderr=false --log-to-file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 --mon_client_target_rank=0
+      clients: [client.0]
+      time: 10
+      unique_pool: true

diff --git a/qa/suites/rados/upgrade/cephx/release-X/% b/qa/suites/rados/upgrade/cephx/release-X/%
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/rados/upgrade/cephx/release-X/.qa b/qa/suites/rados/upgrade/cephx/release-X/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/release-X/centos_latest.yaml b/qa/suites/rados/upgrade/cephx/release-X/centos_latest.yaml
new file mode 120000 (symlink)
index 0000000..bd9854e
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/centos_latest.yaml
@@ -0,0 +1 @@
+.qa/distros/supported/centos_latest.yaml
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/release-X/conf.yaml b/qa/suites/rados/upgrade/cephx/release-X/conf.yaml
new file mode 100644 (file)
index 0000000..cdae164
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/conf.yaml
@@ -0,0 +1,13 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        debug ms: 1
+        debug auth: 30
+        debug monc: 30
+        log to file: true
+        auth mon ticket ttl: 300
+        auth service ticket ttl: 120
+      mon:
+        debug mon: 30
+        debug ms: 5

diff --git a/qa/suites/rados/upgrade/cephx/release-X/ignore_health.yaml b/qa/suites/rados/upgrade/cephx/release-X/ignore_health.yaml
new file mode 100644 (file)
index 0000000..e96acf5
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/ignore_health.yaml
@@ -0,0 +1,25 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - do not have an application enabled
+      - application not enabled
+      - or freeform for custom applications
+      - POOL_APP_NOT_ENABLED
+      - is down
+      - OSD_DOWN
+      - mons down
+      - mon down
+      - MON_DOWN
+      - out of quorum
+      - PG_AVAILABILITY
+      - PG_DEGRADED
+      - Reduced data availability
+      - Degraded data redundancy
+      - pg .* is stuck inactive
+      - pg .* is .*degraded
+      - FS_DEGRADED
+      - OSDMAP_FLAGS
+      - OSD_UPGRADE_FINISHED
+      - OSD_HOST_DOWN
+      - filesystem is degraded
+      - evicting unresponsive client

diff --git a/qa/suites/rados/upgrade/cephx/release-X/ms_bind/v1.yaml b/qa/suites/rados/upgrade/cephx/release-X/ms_bind/v1.yaml
new file mode 100644 (file)
index 0000000..439563f
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/ms_bind/v1.yaml
@@ -0,0 +1,17 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        mon warn on msgr2 not enabled: false
+      osd:
+        ms bind msgr1: true
+        ms bind msgr2: false
+      mds:
+        ms bind msgr1: true
+        ms bind msgr2: false
+      mon:
+        ms bind msgr1: true
+        ms bind msgr2: false
+      mgr:
+        ms bind msgr1: true
+        ms bind msgr2: false

diff --git a/qa/suites/rados/upgrade/cephx/release-X/ms_bind/v2.yaml b/qa/suites/rados/upgrade/cephx/release-X/ms_bind/v2.yaml
new file mode 100644 (file)
index 0000000..4e81054
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/ms_bind/v2.yaml
@@ -0,0 +1,17 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        mon warn on msgr2 not enabled: false
+      osd:
+        ms bind msgr1: false
+        ms bind msgr2: true
+      mds:
+        ms bind msgr1: false
+        ms bind msgr2: true
+      mon:
+        ms bind msgr1: false
+        ms bind msgr2: true
+      mgr:
+        ms bind msgr1: false
+        ms bind msgr2: true

diff --git a/qa/suites/rados/upgrade/cephx/release-X/roles.yaml b/qa/suites/rados/upgrade/cephx/release-X/roles.yaml
new file mode 100644 (file)
index 0000000..5baa265
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/roles.yaml
@@ -0,0 +1,5 @@
+roles:
+- [mon.a, mds.a, mgr.x, osd.0, osd.1]
+- [mon.b, mon.c, mds.b, mgr.y, osd.2, osd.3]
+- [client.0]
+- [client.1]

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/% b/qa/suites/rados/upgrade/cephx/release-X/tasks/%
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/.qa b/qa/suites/rados/upgrade/cephx/release-X/tasks/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/.qa b/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/v18.2.7.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/v18.2.7.yaml
new file mode 100644 (file)
index 0000000..dac4fb1
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/v18.2.7.yaml
@@ -0,0 +1,38 @@
+meta:
+- desc: |
+   install ceph/reef v18.2.7
+tasks:
+- install:
+    tag: v18.2.7
+    exclude_packages:
+      - librados3
+      - ceph-mgr-dashboard
+      - ceph-mgr-diskprediction-local
+      - ceph-mgr-rook
+      - ceph-mgr-cephadm
+      - cephadm
+      - ceph-volume
+    extra_packages: ['librados2']
+- print: "**** done installing v18.2.7"
+- ceph:
+    log-ignorelist:
+      - overall HEALTH_
+      - \(FS_
+      - \(MDS_
+      - \(OSD_
+      - \(MON_DOWN\)
+      - \(CACHE_POOL_
+      - \(POOL_
+      - \(MGR_DOWN\)
+      - \(PG_
+      - \(SMALLER_PGP_NUM\)
+      - Monitor daemon marked osd
+      - Behind on trimming
+      - Manager daemon
+    conf:
+      global:
+        mon warn on pool no app: false
+- exec:
+    osd.0:
+      - ceph osd set-require-min-compat-client reef
+- print: "**** done ceph"

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/v19.2.2.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/v19.2.2.yaml
new file mode 100644 (file)
index 0000000..651f8d3
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/v19.2.2.yaml
@@ -0,0 +1,38 @@
+meta:
+- desc: |
+   install ceph/squid v19.2.2
+tasks:
+- install:
+    tag: v19.2.2
+    exclude_packages:
+      - librados3
+      - ceph-mgr-dashboard
+      - ceph-mgr-diskprediction-local
+      - ceph-mgr-rook
+      - ceph-mgr-cephadm
+      - cephadm
+      - ceph-volume
+    extra_packages: ['librados2']
+- print: "**** done installing squid v19.2.2"
+- ceph:
+    log-ignorelist:
+      - overall HEALTH_
+      - \(FS_
+      - \(MDS_
+      - \(OSD_
+      - \(MON_DOWN\)
+      - \(CACHE_POOL_
+      - \(POOL_
+      - \(MGR_DOWN\)
+      - \(PG_
+      - \(SMALLER_PGP_NUM\)
+      - Monitor daemon marked osd
+      - Behind on trimming
+      - Manager daemon
+    conf:
+      global:
+        mon warn on pool no app: false
+- exec:
+    osd.0:
+      - ceph osd set-require-min-compat-client squid
+- print: "**** done ceph"

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/1-debug.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/1-debug.yaml
new file mode 100644 (file)
index 0000000..233edca
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/1-debug.yaml
@@ -0,0 +1,12 @@
+tasks:
+- full_sequential_finally:
+  - exec:
+      mon.a:
+        - sleep 1m
+- exec:
+    mon.a:
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 tell mon.a config diff
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 tell mon.b config diff
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 tell mon.c config diff
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 config dump
+      - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 auth ls

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/2-workload/.qa b/qa/suites/rados/upgrade/cephx/release-X/tasks/2-workload/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/2-workload/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/2-workload/radosbench.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/2-workload/radosbench.yaml
new file mode 100644 (file)
index 0000000..4320ec6
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/2-workload/radosbench.yaml
@@ -0,0 +1,23 @@
+# We can't set auth_exit_on_failure here because squid/reef do not understand that switch.
+#teuthology:
+#  postmerge:
+#  - |
+#    if false and yaml.teuthology.variables.wipe_service_keys then
+#        yaml.radosbench.sequential_yield[0].radosbench.auth_exit_on_failure = 99
+#        yaml.radosbench.sequential_yield[0].radosbench.expected_rc = 99
+#    end
+
+# N.B. because `rados bench` has sessions open with the OSDs, we do not expect
+# it to fail any auth after upgrade / rotation / session key wipe. It will only
+# fail new connections with OSDs.
+radosbench:
+  sequential_yield:
+    - radosbench:
+        extra_args: --log-to-stderr=false --log-to-file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 --mon_client_target_rank=0
+        clients: [client.0]
+        time: 300
+        unique_pool: true
+
+tasks:
+  - sequential_yield:
+      - radosbench

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/3-upgrade.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/3-upgrade.yaml
new file mode 100644 (file)
index 0000000..a327fd1
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/3-upgrade.yaml
@@ -0,0 +1,32 @@
+tasks:
+- install.upgrade:
+    mon.a:
+    mon.b:
+    client.0:
+- ceph.restart:
+    daemons: [mgr.*]
+    mon-health-to-clog: false
+    wait-for-healthy: true
+- ceph.restart:
+    daemons: [mon.*]
+    mon-health-to-clog: false
+    wait-for-healthy: true
+- ceph.restart:
+    daemons: [osd.*]
+    mon-health-to-clog: false
+    wait-for-osds-up: true
+    wait-for-healthy: false
+- exec:
+    mon.a:
+      - ceph versions
+      - ceph osd dump -f json-pretty
+      - ceph osd require-osd-release tentacle
+      - for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done
+- ceph.restart:
+    daemons: [mds.*]
+    mon-health-to-clog: false
+    wait-for-healthy: true
+- exec:
+    mon.a:
+      - ceph versions
+      - ceph fs dump

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/4-cephx-service-upgrade.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/4-cephx-service-upgrade.yaml
new file mode 100644 (file)
index 0000000..44f1ab2
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/4-cephx-service-upgrade.yaml
@@ -0,0 +1,14 @@
+tasks:
+- ceph.key_rotate:
+    daemons: [mon.*]
+    key_type: aes256k
+- ceph.key_rotate:
+    daemons: [mgr.*, osd.*, mds.*]
+    key_type: aes256k
+- exec:
+    mon.a:
+      - ceph --debug_ms=5 --debug_auth=30 --debug_monc=30 auth ls
+      - ceph --debug_ms=5 --debug_auth=30 --debug_monc=30 auth --format=json-pretty dump-keys
+      - ceph --debug_ms=5 --debug_auth=30 --debug_monc=30 auth --format=json dump-keys | jq 'any(.data.secrets[] | select(.key.type == 1 or .key.type == 2 or .key.type == 4 or .key.type == 16); .val.key.type != 2)'
+      - ceph --debug_ms=5 --debug_auth=30 --debug_monc=30 auth --format=json dump-keys | jq '.data.rotating_secrets | all( .val.secrets | all(.val.key.type == 1) )'
+      - ceph --debug_ms=5 --debug_auth=30 --debug_monc=30 config set global auth_service_cipher aes256k

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/.qa b/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/no.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/no.yaml
new file mode 100644 (file)
index 0000000..2290d55
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/no.yaml
@@ -0,0 +1,3 @@
+teuthology:
+  variables:
+    wipe_service_keys: false

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/yes.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/yes.yaml
new file mode 100644 (file)
index 0000000..e4ee1a4
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/yes.yaml
@@ -0,0 +1,12 @@
+# N.B. in order to effect a service key wipe, the service daemons must be
+# restarted next. During this time, service daemons will be inaccessible to new
+# clients.
+teuthology:
+  variables:
+    wipe_service_keys: true
+tasks:
+- exec:
+    mon.a:
+      - ceph auth wipe-rotating-service-keys
+      - ceph auth --format=json-pretty dump-keys
+      - ceph auth --format=json dump-keys | jq '.data.rotating_secrets | all(  if .key == 32 then  (.val.secrets | all(.val.key.type == 1))  else  (.val.secrets | all(.val.key.type == 2))  end  )'

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/.qa b/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/no.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/no.yaml
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/yes.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/yes.yaml
new file mode 100644 (file)
index 0000000..56c7271
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/yes.yaml
@@ -0,0 +1,25 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - OSD_DOWN
+      - OSD_ROOT_DOWN
+tasks:
+- sleep:
+    duration: 10
+- ceph.restart:
+    daemons: [mon.*]
+    mon-health-to-clog: false
+    wait-for-healthy: true
+- ceph.restart:
+    daemons: [mgr.*]
+    mon-health-to-clog: false
+    wait-for-healthy: true
+- ceph.restart:
+    daemons: [osd.*]
+    mon-health-to-clog: false
+    wait-for-healthy: true
+    wait-for-osds-up: true
+- ceph.restart:
+    daemons: [mds.*]
+    mon-health-to-clog: false
+    wait-for-healthy: true

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/7-workload-after-rotate/.qa b/qa/suites/rados/upgrade/cephx/release-X/tasks/7-workload-after-rotate/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/7-workload-after-rotate/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/7-workload-after-rotate/radosbench.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/7-workload-after-rotate/radosbench.yaml
new file mode 100644 (file)
index 0000000..1678f63
--- /dev/null
+++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/7-workload-after-rotate/radosbench.yaml
@@ -0,0 +1,6 @@
+tasks:
+  - radosbench:
+      extra_args: --debug_ms=5 --debug_auth=30 --debug_monc=30 --mon_client_target_rank=0
+      clients: [client.0]
+      time: 10
+      unique_pool: true