]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
qa: add workunit for testing rgw datacache
authorAli Maredia <amaredia@redhat.com>
Tue, 1 Dec 2020 04:25:01 +0000 (04:25 +0000)
committerMark Kogan <mkogan@redhat.com>
Sun, 4 Jul 2021 11:18:06 +0000 (14:18 +0300)
Add a workunit for testing the rgw object cache
by using s3cmd to write objects and then
verify the objects in the cache.

Also move the 0-install.yaml file out of tasks and
into the main dir for the rgw/verify subsuite.

Signed-off-by: Ali Maredia <amaredia@redhat.com>
qa: d3n: add debug logs

Signed-off-by: Ali Maredia <amaredia@redhat.com>
rgw: s3n: qa: fix netstat search for rgw process

Signed-off-by: Mark Kogan <mkogan@redhat.com>
qa/suites/rgw/verify/0-install.yaml [new file with mode: 0644]
qa/suites/rgw/verify/datacache/no_datacache.yaml [new file with mode: 0644]
qa/suites/rgw/verify/datacache/rgw-datacache.yaml [new file with mode: 0644]
qa/suites/rgw/verify/tasks/0-install.yaml [deleted file]
qa/tasks/rgw.py
qa/workunits/rgw/run-datacache.sh [new file with mode: 0755]
qa/workunits/rgw/test_rgw_datacache.py [new file with mode: 0755]

diff --git a/qa/suites/rgw/verify/0-install.yaml b/qa/suites/rgw/verify/0-install.yaml
new file mode 100644 (file)
index 0000000..b93212d
--- /dev/null
@@ -0,0 +1,19 @@
+tasks:
+- install:
+      # extra packages added for the rgw-datacache task
+      extra_packages:
+        deb: ['s3cmd']
+        rpm: ['s3cmd']
+- ceph:
+- openssl_keys:
+- rgw:
+    client.0:
+
+overrides:
+  ceph:
+    conf:
+      global:
+        osd_min_pg_log_entries: 10
+        osd_max_pg_log_entries: 10
+      client:
+        rgw lc debug interval: 10
diff --git a/qa/suites/rgw/verify/datacache/no_datacache.yaml b/qa/suites/rgw/verify/datacache/no_datacache.yaml
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/qa/suites/rgw/verify/datacache/rgw-datacache.yaml b/qa/suites/rgw/verify/datacache/rgw-datacache.yaml
new file mode 100644 (file)
index 0000000..f5f8c94
--- /dev/null
@@ -0,0 +1,18 @@
+overrides:
+  ceph:
+    conf:
+      client:
+        rgw d3n l1 local datacache enabled: true
+        rgw enable ops log: true
+        rgw d3n l1 datacache persistent path: /tmp/rgw_datacache/
+        rgw d3n l1 datacache size: 10737418240
+  rgw:
+    datacache: true
+    datacache_path: /tmp/rgw_datacache
+tasks:
+- workunit:
+    clients:
+      client.0:
+      - rgw/run-datacache.sh
+    env:
+      RGW_DATACACHE_PATH: /tmp/rgw_datacache
diff --git a/qa/suites/rgw/verify/tasks/0-install.yaml b/qa/suites/rgw/verify/tasks/0-install.yaml
deleted file mode 100644 (file)
index 013ccbd..0000000
+++ /dev/null
@@ -1,15 +0,0 @@
-tasks:
-- install:
-- ceph:
-- openssl_keys:
-- rgw:
-    client.0:
-
-overrides:
-  ceph:
-    conf:
-      global:
-        osd_min_pg_log_entries: 10
-        osd_max_pg_log_entries: 10
-      client:
-        rgw lc debug interval: 10
index 6961d060c42b07c06e3a4ae60a48ee5f83ddbb34..218cd20a0202adaec110b63ffc8a4e1ad17872d1 100644 (file)
@@ -282,6 +282,18 @@ def configure_compression(ctx, clients, compression):
                 check_status=True)
     yield
 
+@contextlib.contextmanager
+def configure_datacache(ctx, clients, datacache_path):
+    """ create directory for rgw datacache """
+    log.info('Creating directory for rgw datacache at %s', datacache_path)
+    for client in clients:
+        if(datacache_path != None):
+            ctx.cluster.only(client).run(args=['mkdir', '-p', datacache_path])
+            ctx.cluster.only(client).run(args=['sudo', 'chmod', 'a+rwx', datacache_path])
+        else:
+            log.info('path for datacache was not provided')
+    yield
+
 @contextlib.contextmanager
 def configure_storage_classes(ctx, clients, storage_classes):
     """ set a compression type in the default zone placement """
@@ -373,6 +385,8 @@ def task(ctx, config):
     default_cert = config.pop('ssl certificate', None)
     ctx.rgw.data_pool_pg_size = config.pop('data_pool_pg_size', 64)
     ctx.rgw.index_pool_pg_size = config.pop('index_pool_pg_size', 64)
+    ctx.rgw.datacache = bool(config.pop('datacache', False))
+    ctx.rgw.datacache_path = config.pop('datacache_path', None)
     ctx.rgw.config = config
 
     log.debug("config is {}".format(config))
@@ -388,6 +402,11 @@ def task(ctx, config):
             lambda: configure_compression(ctx=ctx, clients=clients,
                                           compression=ctx.rgw.compression_type),
         ])
+    if ctx.rgw.datacache:
+        subtasks.extend([
+            lambda: configure_datacache(ctx=ctx, clients=clients,
+                                              datacache_path=ctx.rgw.datacache_path),
+        ])
     if ctx.rgw.storage_classes:
         subtasks.extend([
             lambda: configure_storage_classes(ctx=ctx, clients=clients,
diff --git a/qa/workunits/rgw/run-datacache.sh b/qa/workunits/rgw/run-datacache.sh
new file mode 100755 (executable)
index 0000000..5c00da1
--- /dev/null
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -ex
+
+#assume working ceph environment (radosgw-admin in path) and rgw on localhost:80
+# localhost::443 for ssl
+
+mydir=`dirname $0`
+
+python3 -m venv $mydir
+source $mydir/bin/activate
+pip install pip --upgrade
+pip install configobj
+
+## run test
+$mydir/bin/python3 $mydir/test_rgw_datacache.py
+
+deactivate
+echo OK.
+
diff --git a/qa/workunits/rgw/test_rgw_datacache.py b/qa/workunits/rgw/test_rgw_datacache.py
new file mode 100755 (executable)
index 0000000..801f349
--- /dev/null
@@ -0,0 +1,211 @@
+#!/usr/bin/python3
+
+import logging as log
+from configobj import ConfigObj
+import subprocess
+import json
+import os
+
+"""
+Runs a test against a rgw with the data cache enabled. A client must be
+set in the config for this task. This client must be the same client
+that is in the config for the `rgw` task.
+
+In the `overrides` section `datacache` and `datacache` must be configured for
+the `rgw` task and the ceph conf overrides must contain the below config
+variables in the client section.
+
+`s3cmd` must be added as an extra_package to the install task.
+
+In the `workunit` task, `- rgw/run-datacache.sh` must be set for the client that
+is in the config for the `rgw` task. The `RGW_DATACACHE_PATH` variable must be
+set in the workunit's `env` and it must match the `datacache_path` given to the
+`rgw` task in `overrides`.
+Ex:
+- install:
+    extra_packages:
+      deb: ['s3cmd']
+      rpm: ['s3cmd']
+- overrides:
+    rgw:
+      datacache: true
+      datacache_path: /tmp/rgw_datacache
+    install:
+      extra_packages:
+        deb: ['s3cmd']
+        rpm: ['s3cmd']
+    ceph:
+      conf:
+        client:
+          rgw d3n l1 datacache persistent path: /tmp/rgw_datacache/
+          rgw d3n l1 datacache size: 10737417240
+          rgw d3n l1 local datacache enabled: true
+          rgw enable ops log: true
+- rgw:
+    client.0:
+- workunit:
+    clients:
+      client.0:
+      - rgw/run-datacache.sh
+    env:
+      RGW_DATACACHE_PATH: /tmp/rgw_datacache
+    cleanup: true
+"""
+
+log.basicConfig(level=log.DEBUG)
+
+""" Constants """
+USER = 'rgw_datacache_user'
+DISPLAY_NAME = 'DatacacheUser'
+ACCESS_KEY = 'NX5QOQKC6BH2IDN8HC7A'
+SECRET_KEY = 'LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn'
+BUCKET_NAME = 'datacachebucket'
+FILE_NAME = '7M.dat'
+GET_FILE_NAME = '7M-get.dat'
+
+def exec_cmd(cmd):
+    log.debug("exec_cmd(%s)", cmd)
+    try:
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+        out, err = proc.communicate()
+        if proc.returncode == 0:
+            log.info('command succeeded')
+            if out is not None: log.info(out)
+            return out
+        else:
+            raise Exception("error: %s \nreturncode: %s" % (err, proc.returncode))
+    except Exception as e:
+        log.error('command failed')
+        log.error(e)
+        return False
+
+def get_radosgw_endpoint():
+    out = exec_cmd('sudo netstat -nltp | egrep "rados|valgr"')  # short for radosgw/valgrind
+    x = out.decode('utf8').split(" ")
+    port = [i for i in x if ':' in i][0].split(':')[1]
+    log.info('radosgw port: %s' % port)
+    proto = "http"
+    hostname = 'localhost'
+
+    if port == '443':
+        proto = "https"
+        out = exec_cmd('hostname')
+        hostname = get_cmd_output(out)
+        hostname = hostname + ".front.sepia.ceph.com"
+
+    endpoint = proto + "://" + hostname + ":" + port
+    log.info("radosgw endpoint is: %s", endpoint)
+    return endpoint, proto
+
+def create_s3cmd_config(path, proto):
+    """
+    Creates a minimal config file for s3cmd
+    """
+    log.info("Creating s3cmd config...")
+
+    use_https_config = "False"
+    log.info("proto for s3cmd config is %s", proto)
+    if proto == "https":
+        use_https_config = "True"
+
+    s3cmd_config = ConfigObj(
+        indent_type='',
+        infile={
+            'default':
+                {
+                'host_bucket': 'no.way.in.hell',
+                'use_https': use_https_config,
+                },
+            }
+    )
+
+    f = open(path, 'wb')
+    s3cmd_config.write(f)
+    f.close()
+    log.info("s3cmd config written")
+
+def get_cmd_output(cmd_out):
+    out = cmd_out.decode('utf8')
+    out = out.strip('\n')
+    return out
+
+def main():
+    """
+    execute the datacache test
+    """
+    # setup for test
+    cache_dir = os.environ['RGW_DATACACHE_PATH']
+    log.debug("datacache dir from config is: %s", cache_dir)
+
+    out = exec_cmd('pwd')
+    pwd = get_cmd_output(out)
+    log.debug("pwd is: %s", pwd)
+
+    endpoint, proto = get_radosgw_endpoint()
+
+    # create 7M file to put
+    outfile = pwd + '/' + FILE_NAME
+    exec_cmd('dd if=/dev/urandom of=%s bs=1M count=7' % (outfile))
+
+    # create user
+    exec_cmd('radosgw-admin user create --uid %s --display-name %s --access-key %s --secret %s'
+            % (USER, DISPLAY_NAME, ACCESS_KEY, SECRET_KEY))
+
+    # create s3cmd config
+    s3cmd_config_path = pwd + '/s3cfg'
+    create_s3cmd_config(s3cmd_config_path, proto)
+
+    # create a bucket
+    exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --host=%s mb s3://%s'
+            % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, BUCKET_NAME))
+
+    # put an object in the bucket
+    exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --host=%s put %s s3://%s'
+            % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, outfile, BUCKET_NAME))
+
+    # get object from bucket
+    get_file_path = pwd + '/' + GET_FILE_NAME
+    exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --host=%s get s3://%s/%s %s --force'
+            % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, BUCKET_NAME, FILE_NAME, get_file_path))
+
+    # get info of object
+    out = exec_cmd('radosgw-admin object stat --bucket=%s --object=%s' % (BUCKET_NAME, FILE_NAME))
+
+    json_op = json.loads(out)
+    cached_object_name = json_op['manifest']['prefix']
+    log.debug("Cached object name is: %s", cached_object_name)
+
+    # list the files in the cache dir for troubleshooting
+    out = exec_cmd('ls -l %s' % (cache_dir))
+    # check that the cache is enabled (files exist in the cache directory)
+    out = exec_cmd('find %s -type f | wc -l' % (cache_dir))
+    cached_files_count = int(get_cmd_output(out))
+    log.debug("Number of cached files is: %s", cached_files_count)
+    if cached_files_count == 0:
+        log.info("ERROR: No files in the datacache directory, cache is disabled ?")
+        assert(cached_files_count > 0)
+
+    # get name of cached object and check if it exists in the cache
+    out = exec_cmd('find %s -name "*%s*"' % (cache_dir, cached_object_name))
+    cached_object_path = get_cmd_output(out)
+    log.debug("Path of file in datacache is: %s", cached_object_path)
+    out = exec_cmd('basename %s' % (cached_object_path))
+    basename_cmd_out = get_cmd_output(out)
+    log.debug("Name of file in datacache is: %s", basename_cmd_out)
+
+    # check to see if the cached object is in Ceph
+    out = exec_cmd('rados ls -p default.rgw.buckets.data')
+    rados_ls_out = get_cmd_output(out)
+    log.debug("rados ls output is: %s", rados_ls_out)
+
+    assert(basename_cmd_out in rados_ls_out)
+    log.debug("RGW Datacache test SUCCESS")
+
+    # remove datacache dir
+    #cmd = exec_cmd('rm -rf %s' % (cache_dir))
+    #log.debug("RGW Datacache dir deleted")
+    #^ commenting for future refrence - the work unit will continue running tests and if the cache_dir is removed
+    #  all the writes to cache will fail with errno 2 ENOENT No such file or directory.
+
+main()
+log.info("Completed Datacache tests")