From 25f1f62cba5a7eab86abda957f9b4c2d767da553 Mon Sep 17 00:00:00 2001 From: Yin Congmin Date: Sat, 9 May 2020 01:36:12 +0800 Subject: [PATCH] qa: add tests for immutable object cache based on qemu task, use immutable_object_cache task to test parent cache based on rbd_fio task, use immutable_object_cache task to test parent cache Signed-off-by: Yin Congmin Signed-off-by: Feng Hualong --- qa/suites/rbd/immutable-object-cache/% | 0 qa/suites/rbd/immutable-object-cache/.qa | 1 + .../rbd/immutable-object-cache/clusters/+ | 0 .../rbd/immutable-object-cache/clusters/.qa | 1 + .../clusters/fix-2.yaml | 3 + .../clusters/openstack.yaml | 4 + .../pool/ceph_and_immutable_object_cache.yaml | 11 +++ .../supported-random-distro$ | 1 + .../rbd/immutable-object-cache/workloads/.qa | 1 + .../fio_on_immutable_object_cache.yaml | 11 +++ ..._on_immutable_object_cache_and_thrash.yaml | 11 +++ qa/tasks/immutable_object_cache.py | 73 ++++++++++++++++ qa/tasks/immutable_object_cache_thrash.py | 86 +++++++++++++++++++ qa/tasks/rbd_fio.py | 2 + 14 files changed, 205 insertions(+) create mode 100644 qa/suites/rbd/immutable-object-cache/% create mode 120000 qa/suites/rbd/immutable-object-cache/.qa create mode 100644 qa/suites/rbd/immutable-object-cache/clusters/+ create mode 120000 qa/suites/rbd/immutable-object-cache/clusters/.qa create mode 100644 qa/suites/rbd/immutable-object-cache/clusters/fix-2.yaml create mode 100644 qa/suites/rbd/immutable-object-cache/clusters/openstack.yaml create mode 100644 qa/suites/rbd/immutable-object-cache/pool/ceph_and_immutable_object_cache.yaml create mode 120000 qa/suites/rbd/immutable-object-cache/supported-random-distro$ create mode 120000 qa/suites/rbd/immutable-object-cache/workloads/.qa create mode 100644 qa/suites/rbd/immutable-object-cache/workloads/fio_on_immutable_object_cache.yaml create mode 100644 qa/suites/rbd/immutable-object-cache/workloads/qemu_on_immutable_object_cache_and_thrash.yaml create mode 100644 qa/tasks/immutable_object_cache.py create mode 100644 qa/tasks/immutable_object_cache_thrash.py diff --git a/qa/suites/rbd/immutable-object-cache/% b/qa/suites/rbd/immutable-object-cache/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/rbd/immutable-object-cache/.qa b/qa/suites/rbd/immutable-object-cache/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rbd/immutable-object-cache/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rbd/immutable-object-cache/clusters/+ b/qa/suites/rbd/immutable-object-cache/clusters/+ new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/rbd/immutable-object-cache/clusters/.qa b/qa/suites/rbd/immutable-object-cache/clusters/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rbd/immutable-object-cache/clusters/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rbd/immutable-object-cache/clusters/fix-2.yaml b/qa/suites/rbd/immutable-object-cache/clusters/fix-2.yaml new file mode 100644 index 00000000000..dbccecbceaa --- /dev/null +++ b/qa/suites/rbd/immutable-object-cache/clusters/fix-2.yaml @@ -0,0 +1,3 @@ +roles: +- [mon.a, mgr.x, osd.0, osd.1] +- [mon.b, mgr.y, osd.2, osd.3, client.0] diff --git a/qa/suites/rbd/immutable-object-cache/clusters/openstack.yaml b/qa/suites/rbd/immutable-object-cache/clusters/openstack.yaml new file mode 100644 index 00000000000..b113e4f2ef7 --- /dev/null +++ b/qa/suites/rbd/immutable-object-cache/clusters/openstack.yaml @@ -0,0 +1,4 @@ +openstack: + - volumes: # attached to each instance + count: 4 + size: 10 # GB diff --git a/qa/suites/rbd/immutable-object-cache/pool/ceph_and_immutable_object_cache.yaml b/qa/suites/rbd/immutable-object-cache/pool/ceph_and_immutable_object_cache.yaml new file mode 100644 index 00000000000..540e33d0647 --- /dev/null +++ b/qa/suites/rbd/immutable-object-cache/pool/ceph_and_immutable_object_cache.yaml @@ -0,0 +1,11 @@ +tasks: +- install: + extra_packages: ['ceph-immutable-object-cache'] +- ceph: + conf: + client: + rbd_parent_cache_enabled: true + immutable object cache path: /tmp/ceph-immutable-object-cache + immutable object cache max size: 10G +- immutable_object_cache: + client.0: diff --git a/qa/suites/rbd/immutable-object-cache/supported-random-distro$ b/qa/suites/rbd/immutable-object-cache/supported-random-distro$ new file mode 120000 index 00000000000..0862b4457b3 --- /dev/null +++ b/qa/suites/rbd/immutable-object-cache/supported-random-distro$ @@ -0,0 +1 @@ +.qa/distros/supported-random-distro$ \ No newline at end of file diff --git a/qa/suites/rbd/immutable-object-cache/workloads/.qa b/qa/suites/rbd/immutable-object-cache/workloads/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rbd/immutable-object-cache/workloads/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rbd/immutable-object-cache/workloads/fio_on_immutable_object_cache.yaml b/qa/suites/rbd/immutable-object-cache/workloads/fio_on_immutable_object_cache.yaml new file mode 100644 index 00000000000..08d76ee159c --- /dev/null +++ b/qa/suites/rbd/immutable-object-cache/workloads/fio_on_immutable_object_cache.yaml @@ -0,0 +1,11 @@ +tasks: +- rbd_fio: + client.0: + thick-provision: true + fio-io-size: 100% + formats: [2] + features: [[layering]] + io-engine: rbd + test-clone-io: 1 + rw: randread + runtime: 600 diff --git a/qa/suites/rbd/immutable-object-cache/workloads/qemu_on_immutable_object_cache_and_thrash.yaml b/qa/suites/rbd/immutable-object-cache/workloads/qemu_on_immutable_object_cache_and_thrash.yaml new file mode 100644 index 00000000000..30cb9f0a16a --- /dev/null +++ b/qa/suites/rbd/immutable-object-cache/workloads/qemu_on_immutable_object_cache_and_thrash.yaml @@ -0,0 +1,11 @@ +- qemu: + client.0: + clone: true + test: qa/run_xfstests_qemu.sh + image_url: http://cloud-images.ubuntu.com/releases/bionic/release/ubuntu-18.04-server-cloudimg-amd64.img + cpus: 4 + memory: 4096 + disks: 3 +tasks: +- immutable_object_cache_thrash: + client.0: diff --git a/qa/tasks/immutable_object_cache.py b/qa/tasks/immutable_object_cache.py new file mode 100644 index 00000000000..9bbb5659d44 --- /dev/null +++ b/qa/tasks/immutable_object_cache.py @@ -0,0 +1,73 @@ +""" +immutable object cache task +""" +from io import StringIO + +import contextlib +import logging +import os +import yaml +import time + +from teuthology import misc as teuthology +from teuthology import contextutil +from tasks import rbd +from teuthology.orchestra import run +from teuthology.config import config as teuth_config + +log = logging.getLogger(__name__) + +@contextlib.contextmanager +def immutable_object_cache(ctx, config): + """ + setup and cleanup immutable object cache + """ + log.info("start immutable object cache daemon") + for client, client_config in config.items(): + (remote,) = ctx.cluster.only(client).remotes.keys() + # make sure that there is one immutable object cache daemon on the same node. + remote.run( + args=[ + 'sudo', 'killall', '-s', '9', 'ceph-immutable-object-cache', run.Raw('||'), 'true', + ] + ) + remote.run( + args=[ + 'ceph-immutable-object-cache', '-b', + ] + ) + try: + yield + finally: + log.info("cleanup immutable object cache") + for client, client_config in config.items(): + client_config = client_config if client_config is not None else dict() + (remote,) = ctx.cluster.only(client).remotes.keys() + remote.run( + args=[ + 'sudo', 'killall', '-s', '9', 'ceph-immutable-object-cache', run.Raw('||'), 'true', + ] + ) + cache_path = client_config.get('immutable object cache path', '/tmp/ceph-immutable-object-cache') + remote.run( + args=[ + 'sudo', 'rm', '-rf', cache_path, run.Raw('||'), 'true', + ] + ) + +@contextlib.contextmanager +def task(ctx, config): + """ + This is task for start immutable_object_cache. + """ + assert isinstance(config, dict), \ + "task immutable_object_cache only supports a dictionary for configuration" + + managers = [] + config = teuthology.replace_all_with_clients(ctx.cluster, config) + managers.append( + lambda: immutable_object_cache(ctx=ctx, config=config) + ) + + with contextutil.nested(*managers): + yield diff --git a/qa/tasks/immutable_object_cache_thrash.py b/qa/tasks/immutable_object_cache_thrash.py new file mode 100644 index 00000000000..46d2ca6ea09 --- /dev/null +++ b/qa/tasks/immutable_object_cache_thrash.py @@ -0,0 +1,86 @@ +""" +immutable object cache thrash task +""" +from io import StringIO + +import contextlib +import logging +import os +import yaml +import time + +from teuthology import misc as teuthology +from teuthology import contextutil +from tasks import rbd +from teuthology.orchestra import run +from teuthology.config import config as teuth_config + +DEFAULT_KILL_DAEMON_TIME = 2 +DEFAULT_DEAD_TIME = 30 +DEFAULT_LIVE_TIME = 120 + +log = logging.getLogger(__name__) + +@contextlib.contextmanager +def thrashes_immutable_object_cache_daemon(ctx, config): + """ + thrashes immutable object cache daemon. + It can test reconnection feature of RO cache when RO daemon crash + TODO : replace sleep with better method. + """ + log.info("thrashes immutable object cache daemon") + + # just thrash one rbd client. + client, client_config = list(config.items())[0] + (remote,) = ctx.cluster.only(client).remotes.keys() + client_config = client_config if client_config is not None else dict() + kill_daemon_time = client_config.get('kill_daemon_time', DEFAULT_KILL_DAEMON_TIME) + dead_time = client_config.get('dead_time', DEFAULT_DEAD_TIME) + live_time = client_config.get('live_time', DEFAULT_LIVE_TIME) + + for i in range(kill_daemon_time): + log.info("ceph-immutable-object-cache crash....") + remote.run( + args=[ + 'sudo', 'killall', '-s', '9', 'ceph-immutable-object-cache', run.Raw('||'), 'true', + ] + ) + # librbd shoud normally run when ceph-immutable-object-cache + remote.run( + args=[ + 'sleep', '{dead_time}'.format(dead_time=dead_time), + ] + ) + # librbd should reconnect daemon + log.info("startup ceph-immutable-object-cache") + remote.run( + args=[ + 'ceph-immutable-object-cache', '-b', + ] + ) + remote.run( + args=[ + 'sleep', '{live_time}'.format(live_time=live_time), + ] + ) + try: + yield + finally: + log.info("cleanup") + +@contextlib.contextmanager +def task(ctx, config): + """ + This is task for testing immutable_object_cache thrash. + """ + assert isinstance(config, dict), \ + "task immutable_object_cache_thrash only supports a dictionary for configuration" + + managers = [] + config = teuthology.replace_all_with_clients(ctx.cluster, config) + managers.append( + lambda: thrashes_immutable_object_cache_daemon(ctx=ctx, config=config) + ) + + with contextutil.nested(*managers): + yield diff --git a/qa/tasks/rbd_fio.py b/qa/tasks/rbd_fio.py index decc60c268b..393ac8c4b97 100644 --- a/qa/tasks/rbd_fio.py +++ b/qa/tasks/rbd_fio.py @@ -157,6 +157,8 @@ def run_fio(remote, config, rbd_test_dir): '--image', rbd_name, '--image-format', '{f}'.format(f=frmt)] map(lambda x: create_args.extend(['--image-feature', x]), feature) + if config.get('thick-provision'): + create_args.append('--thick-provision') remote.run(args=create_args) remote.run(args=['rbd', 'info', rbd_name]) if ioengine != 'rbd': -- 2.39.5