From: Ali Maredia Date: Tue, 1 Dec 2020 04:25:01 +0000 (+0000) Subject: qa: add workunit for testing rgw datacache X-Git-Tag: v17.1.0~1461^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=1a6255530f63016a3877804594a64bc433333771;p=ceph.git qa: add workunit for testing rgw datacache Add a workunit for testing the rgw object cache by using s3cmd to write objects and then verify the objects in the cache. Also move the 0-install.yaml file out of tasks and into the main dir for the rgw/verify subsuite. Signed-off-by: Ali Maredia qa: d3n: add debug logs Signed-off-by: Ali Maredia rgw: s3n: qa: fix netstat search for rgw process Signed-off-by: Mark Kogan --- diff --git a/qa/suites/rgw/verify/0-install.yaml b/qa/suites/rgw/verify/0-install.yaml new file mode 100644 index 00000000000..b93212d30cc --- /dev/null +++ b/qa/suites/rgw/verify/0-install.yaml @@ -0,0 +1,19 @@ +tasks: +- install: + # extra packages added for the rgw-datacache task + extra_packages: + deb: ['s3cmd'] + rpm: ['s3cmd'] +- ceph: +- openssl_keys: +- rgw: + client.0: + +overrides: + ceph: + conf: + global: + osd_min_pg_log_entries: 10 + osd_max_pg_log_entries: 10 + client: + rgw lc debug interval: 10 diff --git a/qa/suites/rgw/verify/datacache/no_datacache.yaml b/qa/suites/rgw/verify/datacache/no_datacache.yaml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/rgw/verify/datacache/rgw-datacache.yaml b/qa/suites/rgw/verify/datacache/rgw-datacache.yaml new file mode 100644 index 00000000000..f5f8c94d54d --- /dev/null +++ b/qa/suites/rgw/verify/datacache/rgw-datacache.yaml @@ -0,0 +1,18 @@ +overrides: + ceph: + conf: + client: + rgw d3n l1 local datacache enabled: true + rgw enable ops log: true + rgw d3n l1 datacache persistent path: /tmp/rgw_datacache/ + rgw d3n l1 datacache size: 10737418240 + rgw: + datacache: true + datacache_path: /tmp/rgw_datacache +tasks: +- workunit: + clients: + client.0: + - rgw/run-datacache.sh + env: + RGW_DATACACHE_PATH: /tmp/rgw_datacache diff --git a/qa/suites/rgw/verify/tasks/0-install.yaml b/qa/suites/rgw/verify/tasks/0-install.yaml deleted file mode 100644 index 013ccbd26fe..00000000000 --- a/qa/suites/rgw/verify/tasks/0-install.yaml +++ /dev/null @@ -1,15 +0,0 @@ -tasks: -- install: -- ceph: -- openssl_keys: -- rgw: - client.0: - -overrides: - ceph: - conf: - global: - osd_min_pg_log_entries: 10 - osd_max_pg_log_entries: 10 - client: - rgw lc debug interval: 10 diff --git a/qa/tasks/rgw.py b/qa/tasks/rgw.py index 6961d060c42..218cd20a020 100644 --- a/qa/tasks/rgw.py +++ b/qa/tasks/rgw.py @@ -282,6 +282,18 @@ def configure_compression(ctx, clients, compression): check_status=True) yield +@contextlib.contextmanager +def configure_datacache(ctx, clients, datacache_path): + """ create directory for rgw datacache """ + log.info('Creating directory for rgw datacache at %s', datacache_path) + for client in clients: + if(datacache_path != None): + ctx.cluster.only(client).run(args=['mkdir', '-p', datacache_path]) + ctx.cluster.only(client).run(args=['sudo', 'chmod', 'a+rwx', datacache_path]) + else: + log.info('path for datacache was not provided') + yield + @contextlib.contextmanager def configure_storage_classes(ctx, clients, storage_classes): """ set a compression type in the default zone placement """ @@ -373,6 +385,8 @@ def task(ctx, config): default_cert = config.pop('ssl certificate', None) ctx.rgw.data_pool_pg_size = config.pop('data_pool_pg_size', 64) ctx.rgw.index_pool_pg_size = config.pop('index_pool_pg_size', 64) + ctx.rgw.datacache = bool(config.pop('datacache', False)) + ctx.rgw.datacache_path = config.pop('datacache_path', None) ctx.rgw.config = config log.debug("config is {}".format(config)) @@ -388,6 +402,11 @@ def task(ctx, config): lambda: configure_compression(ctx=ctx, clients=clients, compression=ctx.rgw.compression_type), ]) + if ctx.rgw.datacache: + subtasks.extend([ + lambda: configure_datacache(ctx=ctx, clients=clients, + datacache_path=ctx.rgw.datacache_path), + ]) if ctx.rgw.storage_classes: subtasks.extend([ lambda: configure_storage_classes(ctx=ctx, clients=clients, diff --git a/qa/workunits/rgw/run-datacache.sh b/qa/workunits/rgw/run-datacache.sh new file mode 100755 index 00000000000..5c00da1dae9 --- /dev/null +++ b/qa/workunits/rgw/run-datacache.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -ex + +#assume working ceph environment (radosgw-admin in path) and rgw on localhost:80 +# localhost::443 for ssl + +mydir=`dirname $0` + +python3 -m venv $mydir +source $mydir/bin/activate +pip install pip --upgrade +pip install configobj + +## run test +$mydir/bin/python3 $mydir/test_rgw_datacache.py + +deactivate +echo OK. + diff --git a/qa/workunits/rgw/test_rgw_datacache.py b/qa/workunits/rgw/test_rgw_datacache.py new file mode 100755 index 00000000000..801f34975f5 --- /dev/null +++ b/qa/workunits/rgw/test_rgw_datacache.py @@ -0,0 +1,211 @@ +#!/usr/bin/python3 + +import logging as log +from configobj import ConfigObj +import subprocess +import json +import os + +""" +Runs a test against a rgw with the data cache enabled. A client must be +set in the config for this task. This client must be the same client +that is in the config for the `rgw` task. + +In the `overrides` section `datacache` and `datacache` must be configured for +the `rgw` task and the ceph conf overrides must contain the below config +variables in the client section. + +`s3cmd` must be added as an extra_package to the install task. + +In the `workunit` task, `- rgw/run-datacache.sh` must be set for the client that +is in the config for the `rgw` task. The `RGW_DATACACHE_PATH` variable must be +set in the workunit's `env` and it must match the `datacache_path` given to the +`rgw` task in `overrides`. +Ex: +- install: + extra_packages: + deb: ['s3cmd'] + rpm: ['s3cmd'] +- overrides: + rgw: + datacache: true + datacache_path: /tmp/rgw_datacache + install: + extra_packages: + deb: ['s3cmd'] + rpm: ['s3cmd'] + ceph: + conf: + client: + rgw d3n l1 datacache persistent path: /tmp/rgw_datacache/ + rgw d3n l1 datacache size: 10737417240 + rgw d3n l1 local datacache enabled: true + rgw enable ops log: true +- rgw: + client.0: +- workunit: + clients: + client.0: + - rgw/run-datacache.sh + env: + RGW_DATACACHE_PATH: /tmp/rgw_datacache + cleanup: true +""" + +log.basicConfig(level=log.DEBUG) + +""" Constants """ +USER = 'rgw_datacache_user' +DISPLAY_NAME = 'DatacacheUser' +ACCESS_KEY = 'NX5QOQKC6BH2IDN8HC7A' +SECRET_KEY = 'LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn' +BUCKET_NAME = 'datacachebucket' +FILE_NAME = '7M.dat' +GET_FILE_NAME = '7M-get.dat' + +def exec_cmd(cmd): + log.debug("exec_cmd(%s)", cmd) + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + out, err = proc.communicate() + if proc.returncode == 0: + log.info('command succeeded') + if out is not None: log.info(out) + return out + else: + raise Exception("error: %s \nreturncode: %s" % (err, proc.returncode)) + except Exception as e: + log.error('command failed') + log.error(e) + return False + +def get_radosgw_endpoint(): + out = exec_cmd('sudo netstat -nltp | egrep "rados|valgr"') # short for radosgw/valgrind + x = out.decode('utf8').split(" ") + port = [i for i in x if ':' in i][0].split(':')[1] + log.info('radosgw port: %s' % port) + proto = "http" + hostname = 'localhost' + + if port == '443': + proto = "https" + out = exec_cmd('hostname') + hostname = get_cmd_output(out) + hostname = hostname + ".front.sepia.ceph.com" + + endpoint = proto + "://" + hostname + ":" + port + log.info("radosgw endpoint is: %s", endpoint) + return endpoint, proto + +def create_s3cmd_config(path, proto): + """ + Creates a minimal config file for s3cmd + """ + log.info("Creating s3cmd config...") + + use_https_config = "False" + log.info("proto for s3cmd config is %s", proto) + if proto == "https": + use_https_config = "True" + + s3cmd_config = ConfigObj( + indent_type='', + infile={ + 'default': + { + 'host_bucket': 'no.way.in.hell', + 'use_https': use_https_config, + }, + } + ) + + f = open(path, 'wb') + s3cmd_config.write(f) + f.close() + log.info("s3cmd config written") + +def get_cmd_output(cmd_out): + out = cmd_out.decode('utf8') + out = out.strip('\n') + return out + +def main(): + """ + execute the datacache test + """ + # setup for test + cache_dir = os.environ['RGW_DATACACHE_PATH'] + log.debug("datacache dir from config is: %s", cache_dir) + + out = exec_cmd('pwd') + pwd = get_cmd_output(out) + log.debug("pwd is: %s", pwd) + + endpoint, proto = get_radosgw_endpoint() + + # create 7M file to put + outfile = pwd + '/' + FILE_NAME + exec_cmd('dd if=/dev/urandom of=%s bs=1M count=7' % (outfile)) + + # create user + exec_cmd('radosgw-admin user create --uid %s --display-name %s --access-key %s --secret %s' + % (USER, DISPLAY_NAME, ACCESS_KEY, SECRET_KEY)) + + # create s3cmd config + s3cmd_config_path = pwd + '/s3cfg' + create_s3cmd_config(s3cmd_config_path, proto) + + # create a bucket + exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --host=%s mb s3://%s' + % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, BUCKET_NAME)) + + # put an object in the bucket + exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --host=%s put %s s3://%s' + % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, outfile, BUCKET_NAME)) + + # get object from bucket + get_file_path = pwd + '/' + GET_FILE_NAME + exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --host=%s get s3://%s/%s %s --force' + % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, BUCKET_NAME, FILE_NAME, get_file_path)) + + # get info of object + out = exec_cmd('radosgw-admin object stat --bucket=%s --object=%s' % (BUCKET_NAME, FILE_NAME)) + + json_op = json.loads(out) + cached_object_name = json_op['manifest']['prefix'] + log.debug("Cached object name is: %s", cached_object_name) + + # list the files in the cache dir for troubleshooting + out = exec_cmd('ls -l %s' % (cache_dir)) + # check that the cache is enabled (files exist in the cache directory) + out = exec_cmd('find %s -type f | wc -l' % (cache_dir)) + cached_files_count = int(get_cmd_output(out)) + log.debug("Number of cached files is: %s", cached_files_count) + if cached_files_count == 0: + log.info("ERROR: No files in the datacache directory, cache is disabled ?") + assert(cached_files_count > 0) + + # get name of cached object and check if it exists in the cache + out = exec_cmd('find %s -name "*%s*"' % (cache_dir, cached_object_name)) + cached_object_path = get_cmd_output(out) + log.debug("Path of file in datacache is: %s", cached_object_path) + out = exec_cmd('basename %s' % (cached_object_path)) + basename_cmd_out = get_cmd_output(out) + log.debug("Name of file in datacache is: %s", basename_cmd_out) + + # check to see if the cached object is in Ceph + out = exec_cmd('rados ls -p default.rgw.buckets.data') + rados_ls_out = get_cmd_output(out) + log.debug("rados ls output is: %s", rados_ls_out) + + assert(basename_cmd_out in rados_ls_out) + log.debug("RGW Datacache test SUCCESS") + + # remove datacache dir + #cmd = exec_cmd('rm -rf %s' % (cache_dir)) + #log.debug("RGW Datacache dir deleted") + #^ commenting for future refrence - the work unit will continue running tests and if the cache_dir is removed + # all the writes to cache will fail with errno 2 ENOENT No such file or directory. + +main() +log.info("Completed Datacache tests")