qa/rgw: notifications suite runs single job

pin to the beast frontend, default bluestore, replicated pools, and run
against a random distro

Signed-off-by: Casey Bodley <cbodley@redhat.com>

diff --git a/qa/suites/rgw/notifications/.qa b/qa/suites/rgw/notifications/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
+++ b/qa/suites/rgw/notifications/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file

diff --git a/qa/suites/rgw/notifications/beast.yaml b/qa/suites/rgw/notifications/beast.yaml
new file mode 120000 (symlink)
index 0000000..09ced62
--- /dev/null
+++ b/qa/suites/rgw/notifications/beast.yaml
@@ -0,0 +1 @@
+.qa/rgw_frontend/beast.yaml
\ No newline at end of file

diff --git a/qa/suites/rgw/notifications/bluestore-bitmap.yaml b/qa/suites/rgw/notifications/bluestore-bitmap.yaml
new file mode 120000 (symlink)
index 0000000..a59cf51
--- /dev/null
+++ b/qa/suites/rgw/notifications/bluestore-bitmap.yaml
@@ -0,0 +1 @@
+.qa/objectstore/bluestore-bitmap.yaml
\ No newline at end of file

diff --git a/qa/suites/rgw/notifications/centos_latest.yaml b/qa/suites/rgw/notifications/centos_latest.yaml
deleted file mode 100644 (file)
index 24ae800..0000000
--- a/qa/suites/rgw/notifications/centos_latest.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-os_type: centos
-os_version: "8.1"
-overrides:
-  selinux:
-    whitelist:
-      - scontext=system_u:system_r:logrotate_t:s0

diff --git a/qa/suites/rgw/notifications/clusters/fixed-2.yaml b/qa/suites/rgw/notifications/clusters/fixed-2.yaml
deleted file mode 100644 (file)
index e4448bb..0000000
--- a/qa/suites/rgw/notifications/clusters/fixed-2.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-roles:
-- [mon.a, mon.c, mgr.y, osd.0, osd.1, osd.2, osd.3, client.0, node-exporter.a]
-- [mon.b, mgr.x, osd.4, osd.5, osd.6, osd.7, client.1, prometheus.a, node-exporter.b]
-openstack:
-- volumes: # attached to each instance
-    count: 4
-    size: 10 # GB
-overrides:
-  ceph:
-    conf:
-      osd:
-        osd shutdown pgref assert: true

diff --git a/qa/suites/rgw/notifications/fixed-2.yaml b/qa/suites/rgw/notifications/fixed-2.yaml
new file mode 120000 (symlink)
index 0000000..230ff0f
--- /dev/null
+++ b/qa/suites/rgw/notifications/fixed-2.yaml
@@ -0,0 +1 @@
+.qa/clusters/fixed-2.yaml
\ No newline at end of file

diff --git a/qa/suites/rgw/notifications/frontend/civetweb.yaml b/qa/suites/rgw/notifications/frontend/civetweb.yaml
deleted file mode 100644 (file)
index 57e8cc4..0000000
--- a/qa/suites/rgw/notifications/frontend/civetweb.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-overrides:
-  rgw:
-    frontend: civetweb

diff --git a/qa/suites/rgw/notifications/objectstore/bluestore-bitmap.yaml b/qa/suites/rgw/notifications/objectstore/bluestore-bitmap.yaml
deleted file mode 100644 (file)
index 69c37ac..0000000
--- a/qa/suites/rgw/notifications/objectstore/bluestore-bitmap.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-overrides:
-  thrashosds:
-    bdev_inject_crash: 2
-    bdev_inject_crash_probability: .5
-  ceph:
-    fs: xfs
-    conf:
-      osd:
-        osd objectstore: bluestore
-        bluestore block size: 96636764160
-        debug bluestore: 20
-        debug bluefs: 20
-        debug rocksdb: 10
-        bluestore fsck on mount: true
-        bluestore allocator: bitmap
-        # lower the full ratios since we can fill up a 100gb osd so quickly
-        mon osd full ratio: .9
-        mon osd backfillfull_ratio: .85
-        mon osd nearfull ratio: .8
-        osd failsafe full ratio: .95
-# this doesn't work with failures bc the log writes are not atomic across the two backends
-#        bluestore bluefs env mirror: true
-        bdev enable discard: true
-        bdev async discard: true
-  ceph-deploy:
-    fs: xfs
-    bluestore: yes
-    conf:
-      osd:
-        osd objectstore: bluestore
-        bluestore block size: 96636764160
-        debug bluestore: 20
-        debug bluefs: 20
-        debug rocksdb: 10
-        bluestore fsck on mount: true
-        # lower the full ratios since we can fill up a 100gb osd so quickly
-        mon osd full ratio: .9
-        mon osd backfillfull_ratio: .85
-        mon osd nearfull ratio: .8
-        osd failsafe full ratio: .95
-        bdev enable discard: true
-        bdev async discard: true

diff --git a/qa/suites/rgw/notifications/objectstore/filestore-xfs.yaml b/qa/suites/rgw/notifications/objectstore/filestore-xfs.yaml
deleted file mode 100644 (file)
index bd18aca..0000000
--- a/qa/suites/rgw/notifications/objectstore/filestore-xfs.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-overrides:
-  ceph:
-    fs: xfs
-    conf:
-      osd:
-        osd objectstore: filestore
-        osd sloppy crc: true
-  ceph-deploy:
-    fs: xfs
-    filestore: True
-    conf:
-      osd:
-        osd objectstore: filestore
-        osd sloppy crc: true

diff --git a/qa/suites/rgw/notifications/rgw_pool_type/ec-profile.yaml b/qa/suites/rgw/notifications/rgw_pool_type/ec-profile.yaml
deleted file mode 100644 (file)
index f6fbf35..0000000
--- a/qa/suites/rgw/notifications/rgw_pool_type/ec-profile.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-overrides:
-  rgw:
-    ec-data-pool: true
-    erasure_code_profile:
-      name: testprofile
-      k: 3
-      m: 1
-      crush-failure-domain: osd

diff --git a/qa/suites/rgw/notifications/rgw_pool_type/ec.yaml b/qa/suites/rgw/notifications/rgw_pool_type/ec.yaml
deleted file mode 100644 (file)
index 7c0c5e6..0000000
--- a/qa/suites/rgw/notifications/rgw_pool_type/ec.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-overrides:
-  rgw:
-    ec-data-pool: true

diff --git a/qa/suites/rgw/notifications/rgw_pool_type/replicated.yaml b/qa/suites/rgw/notifications/rgw_pool_type/replicated.yaml
deleted file mode 100644 (file)
index c91709e..0000000
--- a/qa/suites/rgw/notifications/rgw_pool_type/replicated.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-overrides:
-  rgw:
-    ec-data-pool: false

diff --git a/qa/suites/rgw/notifications/supported-random-distro$ b/qa/suites/rgw/notifications/supported-random-distro$
new file mode 120000 (symlink)
index 0000000..0862b44
--- /dev/null
+++ b/qa/suites/rgw/notifications/supported-random-distro$
@@ -0,0 +1 @@
+.qa/distros/supported-random-distro$
\ No newline at end of file

diff --git a/qa/suites/rgw/notifications/tasks/test_kafka.yaml b/qa/suites/rgw/notifications/tasks/test_kafka.yaml
index 1fb7c37e31a6e60c2e3bed5f46e9e88647576900..36b30c4975f1effe47bcba30383e82649fba1554 100644 (file)
--- a/qa/suites/rgw/notifications/tasks/test_kafka.yaml
+++ b/qa/suites/rgw/notifications/tasks/test_kafka.yaml
@@ -2,6 +2,7 @@ tasks:
 - tox: [ client.0 ]
 - kafka:
     client.0:
+      kafka_version: 2.6.0
 - notification-tests:
     client.0:
       force-branch: wip-rgw-bucket-tests-separation-new

diff --git a/qa/tasks/kafka.py b/qa/tasks/kafka.py
index d549f342d6c6ce5aade9afea13409a7de7b2b918..93d9db3d3487d336455ef717605ce36fcc0f2dee 100644 (file)
--- a/qa/tasks/kafka.py
+++ b/qa/tasks/kafka.py
@@ -1,43 +1,26 @@
 """
 Deploy and configure Kafka for Teuthology
 """
-from io import BytesIO
-from io import StringIO
-from configobj import ConfigObj
-import base64
-import argparse
 import contextlib
 import logging
-import os
-import random
-import six
-import string
-import subprocess
-import json
-import sys
-from pathlib import Path
-
-from collections import OrderedDict
-from itertools import chain
 
 from teuthology import misc as teuthology
 from teuthology import contextutil
-from teuthology.config import config as teuth_config
 from teuthology.orchestra import run
-from teuthology.packaging import install_package
-from teuthology.packaging import remove_package
 from teuthology.exceptions import ConfigError
 
 log = logging.getLogger(__name__)
 
-def get_kafka_dir(ctx):
-    return '{tdir}/kafka-2.6.0-src'.format(tdir=teuthology.get_testdir(ctx))
+def get_kafka_version(config):
+    for client, client_config in config.items():
+        if 'kafka_version' in client_config:
+            kafka_version = client_config.get('kafka_version')
+    return kafka_version
 
-def run_in_kafka_dir(ctx, client, args, **kwargs):
-    return ctx.cluster.only(client).run(
-        args=[ 'cd', get_kafka_dir(ctx), run.Raw('&&'), ] + args,
-        **kwargs
-    )
+def get_kafka_dir(ctx, config):
+    kafka_version = get_kafka_version(config)
+    current_version = 'kafka-' + kafka_version + '-src'
+    return '{tdir}/{ver}'.format(tdir=teuthology.get_testdir(ctx),ver=current_version)
 
 def get_toxvenv_dir(ctx):
     return ctx.tox.venv_path
@@ -57,19 +40,30 @@ def install_kafka(ctx, config):
     for (client, _) in config.items():
         (remote,) = ctx.cluster.only(client).remotes.keys()
         test_dir=teuthology.get_testdir(ctx)
-        toxvenv_sh(ctx, remote, ['cd', '{tdir}'.format(tdir=test_dir), run.Raw('&&'), 'wget', 'https://archive.apache.org/dist/kafka/2.6.0/kafka-2.6.0-src.tgz'])
-        toxvenv_sh(ctx, remote, ['cd', '{tdir}'.format(tdir=test_dir), run.Raw('&&'), 'tar', '-xvzf', 'kafka-2.6.0-src.tgz'])
+        current_version = get_kafka_version(config)
+
+        link1 = 'https://archive.apache.org/dist/kafka/' + current_version + '/kafka-' + current_version + '-src.tgz'
+        toxvenv_sh(ctx, remote, ['cd', '{tdir}'.format(tdir=test_dir), run.Raw('&&'), 'wget', link1])
+
+        file1 = 'kafka-' + current_version + '-src.tgz'
+        toxvenv_sh(ctx, remote, ['cd', '{tdir}'.format(tdir=test_dir), run.Raw('&&'), 'tar', '-xvzf', file1])
 
     try:
         yield
     finally:
         log.info('Removing packaged dependencies of Kafka...')
-        test_dir=get_kafka_dir(ctx)
+        test_dir=get_kafka_dir(ctx, config)
+        current_version = get_kafka_version(config)
         for client in config:
             ctx.cluster.only(client).run(
                 args=['rm', '-rf', test_dir],
             )
 
+            rmfile1 = 'kafka-' + current_version + '-src.tgz'
+            ctx.cluster.only(client).run(
+                args=['rm', '-rf', '{tdir}/{doc}'.format(tdir=teuthology.get_testdir(ctx),doc=rmfile1)],
+            )
+
 
 @contextlib.contextmanager
 def run_kafka(ctx,config):
@@ -84,24 +78,24 @@ def run_kafka(ctx,config):
         (remote,) = ctx.cluster.only(client).remotes.keys()
 
         toxvenv_sh(ctx, remote,
-            ['cd', '{tdir}'.format(tdir=get_kafka_dir(ctx)), run.Raw('&&'),
+            ['cd', '{tdir}'.format(tdir=get_kafka_dir(ctx, config)), run.Raw('&&'),
              './gradlew', 'jar', 
              '-PscalaVersion=2.13.2'
             ],
         )
 
         toxvenv_sh(ctx, remote,
-            ['cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx)), run.Raw('&&'),
+            ['cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx, config)), run.Raw('&&'),
              './zookeeper-server-start.sh',
-             '{tir}/config/zookeeper.properties'.format(tir=get_kafka_dir(ctx)),
+             '{tir}/config/zookeeper.properties'.format(tir=get_kafka_dir(ctx, config)),
              run.Raw('&'), 'exit'
             ],
         )
 
         toxvenv_sh(ctx, remote,
-            ['cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx)), run.Raw('&&'),
+            ['cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx, config)), run.Raw('&&'),
              './kafka-server-start.sh',
-             '{tir}/config/server.properties'.format(tir=get_kafka_dir(ctx)),
+             '{tir}/config/server.properties'.format(tir=get_kafka_dir(ctx, config)),
              run.Raw('&'), 'exit'
             ],
         )
@@ -115,16 +109,16 @@ def run_kafka(ctx,config):
             (remote,) = ctx.cluster.only(client).remotes.keys()
 
             toxvenv_sh(ctx, remote, 
-                ['cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx)), run.Raw('&&'),
+                ['cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx, config)), run.Raw('&&'),
                  './kafka-server-stop.sh',  
-                 '{tir}/config/kafka.properties'.format(tir=get_kafka_dir(ctx)),
+                 '{tir}/config/kafka.properties'.format(tir=get_kafka_dir(ctx, config)),
                 ]
             )
 
             toxvenv_sh(ctx, remote, 
-                ['cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx)), run.Raw('&&'), 
+                ['cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx, config)), run.Raw('&&'), 
                  './zookeeper-server-stop.sh',
-                 '{tir}/config/zookeeper.properties'.format(tir=get_kafka_dir(ctx)),
+                 '{tir}/config/zookeeper.properties'.format(tir=get_kafka_dir(ctx, config)),
                 ]
             )
 
@@ -140,14 +134,14 @@ def run_admin_cmds(ctx,config):
 
         toxvenv_sh(ctx, remote,
             [
-                'cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx)), run.Raw('&&'), 
+                'cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx, config)), run.Raw('&&'), 
                 './kafka-topics.sh', '--create', '--topic', 'quickstart-events',
                 '--bootstrap-server', 'localhost:9092'
             ])
 
         toxvenv_sh(ctx, remote,
             [
-                'cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx)), run.Raw('&&'),
+                'cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx, config)), run.Raw('&&'),
                 'echo', "First", run.Raw('|'),
                 './kafka-console-producer.sh', '--topic', 'quickstart-events',
                 '--bootstrap-server', 'localhost:9092'
@@ -155,7 +149,7 @@ def run_admin_cmds(ctx,config):
 
         toxvenv_sh(ctx, remote,
             [
-                'cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx)), run.Raw('&&'),
+                'cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx, config)), run.Raw('&&'),
                 './kafka-console-consumer.sh', '--topic', 'quickstart-events',
                 '--from-beginning',
                 '--bootstrap-server', 'localhost:9092',
@@ -191,7 +185,6 @@ def task(ctx,config):
         config = all_clients
     if isinstance(config, list):
         config = dict.fromkeys(config)
-    clients=config.keys()
 
     log.debug('Kafka config is %s', config)
 

diff --git a/qa/tasks/notification_tests.py b/qa/tasks/notification_tests.py
index 774e42caa5b2ad600ce7547c1b25110d6ba30acc..1d856dc1a58291e6092d5f21068bf1f4d887efa1 100644 (file)
--- a/qa/tasks/notification_tests.py
+++ b/qa/tasks/notification_tests.py
@@ -14,7 +14,6 @@ from teuthology import misc as teuthology
 from teuthology import contextutil
 from teuthology.config import config as teuth_config
 from teuthology.orchestra import run
-from teuthology.exceptions import ConfigError
 
 log = logging.getLogger(__name__)