kernel: support overrides

This is needed for the new unmap subsuite of krbd suite.

normalize_config() config should actually copy config snippets into
new_config, otherwise popping keys from one role's snippet would also
affect other roles, potentially resulting in overrides: overriding more
than it should have.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>

diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py
index 09a64b48844c85907aca82c34ce9a186a99f8e21..1abe705bc9daf1bcc04249b3cd262014a1aeea5a 100644 (file)
--- a/teuthology/task/kernel.py
+++ b/teuthology/task/kernel.py
@@ -71,14 +71,14 @@ def normalize_config(ctx, config):
     :param ctx: Context
     :param config: Configuration
     """
-    if config is None or \
+    if not config or \
             len(filter(lambda x: x in VERSION_KEYS + ['kdb', 'flavor'],
                        config.keys())) == len(config.keys()):
         new_config = {}
-        if config is None:
+        if not config:
             config = CONFIG_DEFAULT
         for role in teuthology.all_roles(ctx.cluster):
-            new_config[role] = config
+            new_config[role] = config.copy()
         return new_config
 
     new_config = {}
@@ -86,15 +86,53 @@ def normalize_config(ctx, config):
         if role_config is None:
             role_config = CONFIG_DEFAULT
         if '.' in role:
-            new_config[role] = role_config
+            new_config[role] = role_config.copy()
         else:
             for id_ in teuthology.all_roles_of_type(ctx.cluster, role):
                 name = '{type}.{id}'.format(type=role, id=id_)
                 # specific overrides generic
                 if name not in config:
-                    new_config[name] = role_config
+                    new_config[name] = role_config.copy()
     return new_config
 
+def normalize_and_apply_overrides(ctx, config, overrides):
+    """
+    kernel task config is hierarchical and needs to be transformed into
+    a normal form, see normalize_config() for details.  Applying overrides is
+    also more involved compared to other tasks because of the number of ways
+    a version of the kernel to install can be specified.
+
+    Returns a (normalized config, timeout) tuple.
+
+    :param ctx: Context
+    :param config: Configuration
+    """
+    timeout = TIMEOUT_DEFAULT
+    if 'timeout' in config:
+        timeout = config.pop('timeout')
+    config = normalize_config(ctx, config)
+    log.debug('normalized config %s' % config)
+
+    if 'timeout' in overrides:
+        timeout = overrides.pop('timeout')
+    if overrides:
+        overrides = normalize_config(ctx, overrides)
+        log.debug('normalized overrides %s' % overrides)
+
+        # Handle a case when a version specified with one type of version key
+        # is overridden by a version specified with another type of version key
+        # (e.g. 'branch: foo' is overridden with 'tag: bar').  To be able to
+        # use deep_merge(), drop all version keys from the original config if
+        # the corresponding override has a version key.
+        for role, role_config in config.iteritems():
+            if (role in overrides and
+                    any(k in overrides[role] for k in VERSION_KEYS)):
+                for k in VERSION_KEYS:
+                    role_config.pop(k, None)
+        teuthology.deep_merge(config, overrides)
+
+    return (config, timeout)
+
 def validate_config(ctx, config):
     """
     Make sure that all kernels in the list of remove kernels
@@ -1101,16 +1139,15 @@ def task(ctx, config):
     :param ctx: Context
     :param config: Configuration
     """
-    assert config is None or isinstance(config, dict), \
+    if config is None:
+        config = {}
+    assert isinstance(config, dict), \
         "task kernel only supports a dictionary for configuration"
 
-    timeout = TIMEOUT_DEFAULT
-    if config is not None and 'timeout' in config:
-        timeout = config.pop('timeout')
-
-    config = normalize_config(ctx, config)
+    overrides = ctx.config.get('overrides', {}).get('kernel', {})
+    config, timeout = normalize_and_apply_overrides(ctx, config, overrides)
     validate_config(ctx, config)
-    log.info('config %s' % config)
+    log.info('config %s, timeout %d' % (config, timeout))
 
     need_install = {}  # sha1 to dl, or path to rpm or deb
     need_version = {}  # utsrelease or sha1

diff --git a/teuthology/test/task/test_kernel.py b/teuthology/test/task/test_kernel.py
new file mode 100644 (file)
index 0000000..7be8658
--- /dev/null
+++ b/teuthology/test/task/test_kernel.py
@@ -0,0 +1,243 @@
+from teuthology.config import FakeNamespace
+from teuthology.orchestra.cluster import Cluster
+from teuthology.orchestra.remote import Remote
+from teuthology.task.kernel import (
+    normalize_and_apply_overrides,
+    CONFIG_DEFAULT,
+    TIMEOUT_DEFAULT,
+)
+
+class TestKernelNormalizeAndApplyOverrides(object):
+
+    def setup(self):
+        self.ctx = FakeNamespace()
+        self.ctx.cluster = Cluster()
+        self.ctx.cluster.add(Remote('remote1'), ['mon.a', 'client.0'])
+        self.ctx.cluster.add(Remote('remote2'), ['osd.0', 'osd.1', 'osd.2'])
+        self.ctx.cluster.add(Remote('remote3'), ['client.1'])
+
+    def test_default(self):
+        config = {}
+        overrides = {}
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'mon.a': CONFIG_DEFAULT,
+            'osd.0': CONFIG_DEFAULT,
+            'osd.1': CONFIG_DEFAULT,
+            'osd.2': CONFIG_DEFAULT,
+            'client.0': CONFIG_DEFAULT,
+            'client.1': CONFIG_DEFAULT,
+        }
+        assert t == TIMEOUT_DEFAULT
+
+    def test_timeout_default(self):
+        config = {
+            'client.0': {'branch': 'testing'},
+        }
+        overrides = {}
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'client.0': {'branch': 'testing'},
+        }
+        assert t == TIMEOUT_DEFAULT
+
+    def test_timeout(self):
+        config = {
+            'client.0': {'branch': 'testing'},
+            'timeout': 100,
+        }
+        overrides = {}
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'client.0': {'branch': 'testing'},
+        }
+        assert t == 100
+
+    def test_override_timeout(self):
+        config = {
+            'client.0': {'branch': 'testing'},
+            'timeout': 100,
+        }
+        overrides = {
+            'timeout': 200,
+        }
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'client.0': {'branch': 'testing'},
+        }
+        assert t == 200
+
+    def test_override_same_version_key(self):
+        config = {
+            'client.0': {'branch': 'testing'},
+        }
+        overrides = {
+            'client.0': {'branch': 'wip-foobar'},
+        }
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'client.0': {'branch': 'wip-foobar'},
+        }
+        assert t == TIMEOUT_DEFAULT
+
+    def test_override_different_version_key(self):
+        config = {
+            'client.0': {'branch': 'testing'},
+        }
+        overrides = {
+            'client.0': {'tag': 'v4.1'},
+        }
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'client.0': {'tag': 'v4.1'},
+        }
+        assert t == TIMEOUT_DEFAULT
+
+    def test_override_actual(self):
+        config = {
+            'osd.1': {'tag': 'v4.1'},
+            'client.0': {'branch': 'testing'},
+        }
+        overrides = {
+            'osd.1': {'koji': 1234, 'kdb': True},
+        }
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'osd.1': {'koji': 1234, 'kdb': True},
+            'client.0': {'branch': 'testing'},
+        }
+        assert t == TIMEOUT_DEFAULT
+
+    def test_override_actual_with_generic(self):
+        config = {
+            'osd.1': {'tag': 'v4.1', 'kdb': False},
+            'client.0': {'branch': 'testing'},
+        }
+        overrides = {
+            'osd': {'koji': 1234},
+        }
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'osd.0': {'koji': 1234},
+            'osd.1': {'koji': 1234, 'kdb': False},
+            'osd.2': {'koji': 1234},
+            'client.0': {'branch': 'testing'},
+        }
+        assert t == TIMEOUT_DEFAULT
+
+    def test_override_actual_with_top_level(self):
+        config = {
+            'osd.1': {'tag': 'v4.1'},
+            'client.0': {'branch': 'testing', 'kdb': False},
+        }
+        overrides = {'koji': 1234, 'kdb': True}
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'mon.a': {'koji': 1234, 'kdb': True},
+            'osd.0': {'koji': 1234, 'kdb': True},
+            'osd.1': {'koji': 1234, 'kdb': True},
+            'osd.2': {'koji': 1234, 'kdb': True},
+            'client.0': {'koji': 1234, 'kdb': True},
+            'client.1': {'koji': 1234, 'kdb': True},
+        }
+        assert t == TIMEOUT_DEFAULT
+
+    def test_override_generic(self):
+        config = {
+            'osd': {'tag': 'v4.1'},
+            'client': {'branch': 'testing'},
+        }
+        overrides = {
+            'client': {'koji': 1234, 'kdb': True},
+        }
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'osd.0': {'tag': 'v4.1'},
+            'osd.1': {'tag': 'v4.1'},
+            'osd.2': {'tag': 'v4.1'},
+            'client.0': {'koji': 1234, 'kdb': True},
+            'client.1': {'koji': 1234, 'kdb': True},
+        }
+        assert t == TIMEOUT_DEFAULT
+
+    def test_override_generic_with_top_level(self):
+        config = {
+            'osd': {'tag': 'v4.1'},
+            'client': {'branch': 'testing', 'kdb': False},
+        }
+        overrides = {
+            'client': {'koji': 1234},
+        }
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'osd.0': {'tag': 'v4.1'},
+            'osd.1': {'tag': 'v4.1'},
+            'osd.2': {'tag': 'v4.1'},
+            'client.0': {'koji': 1234, 'kdb': False},
+            'client.1': {'koji': 1234, 'kdb': False},
+        }
+        assert t == TIMEOUT_DEFAULT
+
+    def test_override_generic_with_actual(self):
+        config = {
+            'osd': {'tag': 'v4.1', 'kdb': False},
+            'client': {'branch': 'testing'},
+        }
+        overrides = {
+            'osd.2': {'koji': 1234, 'kdb': True},
+        }
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'osd.0': {'tag': 'v4.1', 'kdb': False},
+            'osd.1': {'tag': 'v4.1', 'kdb': False},
+            'osd.2': {'koji': 1234, 'kdb': True},
+            'client.0': {'branch': 'testing'},
+            'client.1': {'branch': 'testing'},
+        }
+        assert t == TIMEOUT_DEFAULT
+
+    def test_override_top_level(self):
+        config = {'branch': 'testing'}
+        overrides = {'koji': 1234, 'kdb': True}
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'mon.a': {'koji': 1234, 'kdb': True},
+            'osd.0': {'koji': 1234, 'kdb': True},
+            'osd.1': {'koji': 1234, 'kdb': True},
+            'osd.2': {'koji': 1234, 'kdb': True},
+            'client.0': {'koji': 1234, 'kdb': True},
+            'client.1': {'koji': 1234, 'kdb': True},
+        }
+        assert t == TIMEOUT_DEFAULT
+
+    def test_override_top_level_with_actual(self):
+        config = {'branch': 'testing', 'kdb': False}
+        overrides = {
+            'mon.a': {'koji': 1234},
+        }
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'mon.a': {'koji': 1234, 'kdb': False},
+            'osd.0': {'branch': 'testing', 'kdb': False},
+            'osd.1': {'branch': 'testing', 'kdb': False},
+            'osd.2': {'branch': 'testing', 'kdb': False},
+            'client.0': {'branch': 'testing', 'kdb': False},
+            'client.1': {'branch': 'testing', 'kdb': False},
+        }
+        assert t == TIMEOUT_DEFAULT
+
+    def test_override_top_level_with_generic(self):
+        config = {'branch': 'testing', 'kdb': False}
+        overrides = {
+            'client': {'koji': 1234, 'kdb': True},
+        }
+        config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+        assert config == {
+            'mon.a': {'branch': 'testing', 'kdb': False},
+            'osd.0': {'branch': 'testing', 'kdb': False},
+            'osd.1': {'branch': 'testing', 'kdb': False},
+            'osd.2': {'branch': 'testing', 'kdb': False},
+            'client.0': {'koji': 1234, 'kdb': True},
+            'client.1': {'koji': 1234, 'kdb': True},
+        }
+        assert t == TIMEOUT_DEFAULT