From 0395124d6c3744eb5df1df0a2f1289773ec0da6e Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Thu, 7 May 2015 13:55:14 +0300 Subject: [PATCH] kernel: support overrides This is needed for the new unmap subsuite of krbd suite. normalize_config() config should actually copy config snippets into new_config, otherwise popping keys from one role's snippet would also affect other roles, potentially resulting in overrides: overriding more than it should have. Signed-off-by: Ilya Dryomov --- teuthology/task/kernel.py | 61 +++++-- teuthology/test/task/test_kernel.py | 243 ++++++++++++++++++++++++++++ 2 files changed, 292 insertions(+), 12 deletions(-) create mode 100644 teuthology/test/task/test_kernel.py diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py index 09a64b4884..1abe705bc9 100644 --- a/teuthology/task/kernel.py +++ b/teuthology/task/kernel.py @@ -71,14 +71,14 @@ def normalize_config(ctx, config): :param ctx: Context :param config: Configuration """ - if config is None or \ + if not config or \ len(filter(lambda x: x in VERSION_KEYS + ['kdb', 'flavor'], config.keys())) == len(config.keys()): new_config = {} - if config is None: + if not config: config = CONFIG_DEFAULT for role in teuthology.all_roles(ctx.cluster): - new_config[role] = config + new_config[role] = config.copy() return new_config new_config = {} @@ -86,15 +86,53 @@ def normalize_config(ctx, config): if role_config is None: role_config = CONFIG_DEFAULT if '.' in role: - new_config[role] = role_config + new_config[role] = role_config.copy() else: for id_ in teuthology.all_roles_of_type(ctx.cluster, role): name = '{type}.{id}'.format(type=role, id=id_) # specific overrides generic if name not in config: - new_config[name] = role_config + new_config[name] = role_config.copy() return new_config +def normalize_and_apply_overrides(ctx, config, overrides): + """ + kernel task config is hierarchical and needs to be transformed into + a normal form, see normalize_config() for details. Applying overrides is + also more involved compared to other tasks because of the number of ways + a version of the kernel to install can be specified. + + Returns a (normalized config, timeout) tuple. + + :param ctx: Context + :param config: Configuration + """ + timeout = TIMEOUT_DEFAULT + if 'timeout' in config: + timeout = config.pop('timeout') + config = normalize_config(ctx, config) + log.debug('normalized config %s' % config) + + if 'timeout' in overrides: + timeout = overrides.pop('timeout') + if overrides: + overrides = normalize_config(ctx, overrides) + log.debug('normalized overrides %s' % overrides) + + # Handle a case when a version specified with one type of version key + # is overridden by a version specified with another type of version key + # (e.g. 'branch: foo' is overridden with 'tag: bar'). To be able to + # use deep_merge(), drop all version keys from the original config if + # the corresponding override has a version key. + for role, role_config in config.iteritems(): + if (role in overrides and + any(k in overrides[role] for k in VERSION_KEYS)): + for k in VERSION_KEYS: + role_config.pop(k, None) + teuthology.deep_merge(config, overrides) + + return (config, timeout) + def validate_config(ctx, config): """ Make sure that all kernels in the list of remove kernels @@ -1101,16 +1139,15 @@ def task(ctx, config): :param ctx: Context :param config: Configuration """ - assert config is None or isinstance(config, dict), \ + if config is None: + config = {} + assert isinstance(config, dict), \ "task kernel only supports a dictionary for configuration" - timeout = TIMEOUT_DEFAULT - if config is not None and 'timeout' in config: - timeout = config.pop('timeout') - - config = normalize_config(ctx, config) + overrides = ctx.config.get('overrides', {}).get('kernel', {}) + config, timeout = normalize_and_apply_overrides(ctx, config, overrides) validate_config(ctx, config) - log.info('config %s' % config) + log.info('config %s, timeout %d' % (config, timeout)) need_install = {} # sha1 to dl, or path to rpm or deb need_version = {} # utsrelease or sha1 diff --git a/teuthology/test/task/test_kernel.py b/teuthology/test/task/test_kernel.py new file mode 100644 index 0000000000..7be86587dc --- /dev/null +++ b/teuthology/test/task/test_kernel.py @@ -0,0 +1,243 @@ +from teuthology.config import FakeNamespace +from teuthology.orchestra.cluster import Cluster +from teuthology.orchestra.remote import Remote +from teuthology.task.kernel import ( + normalize_and_apply_overrides, + CONFIG_DEFAULT, + TIMEOUT_DEFAULT, +) + +class TestKernelNormalizeAndApplyOverrides(object): + + def setup(self): + self.ctx = FakeNamespace() + self.ctx.cluster = Cluster() + self.ctx.cluster.add(Remote('remote1'), ['mon.a', 'client.0']) + self.ctx.cluster.add(Remote('remote2'), ['osd.0', 'osd.1', 'osd.2']) + self.ctx.cluster.add(Remote('remote3'), ['client.1']) + + def test_default(self): + config = {} + overrides = {} + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'mon.a': CONFIG_DEFAULT, + 'osd.0': CONFIG_DEFAULT, + 'osd.1': CONFIG_DEFAULT, + 'osd.2': CONFIG_DEFAULT, + 'client.0': CONFIG_DEFAULT, + 'client.1': CONFIG_DEFAULT, + } + assert t == TIMEOUT_DEFAULT + + def test_timeout_default(self): + config = { + 'client.0': {'branch': 'testing'}, + } + overrides = {} + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'client.0': {'branch': 'testing'}, + } + assert t == TIMEOUT_DEFAULT + + def test_timeout(self): + config = { + 'client.0': {'branch': 'testing'}, + 'timeout': 100, + } + overrides = {} + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'client.0': {'branch': 'testing'}, + } + assert t == 100 + + def test_override_timeout(self): + config = { + 'client.0': {'branch': 'testing'}, + 'timeout': 100, + } + overrides = { + 'timeout': 200, + } + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'client.0': {'branch': 'testing'}, + } + assert t == 200 + + def test_override_same_version_key(self): + config = { + 'client.0': {'branch': 'testing'}, + } + overrides = { + 'client.0': {'branch': 'wip-foobar'}, + } + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'client.0': {'branch': 'wip-foobar'}, + } + assert t == TIMEOUT_DEFAULT + + def test_override_different_version_key(self): + config = { + 'client.0': {'branch': 'testing'}, + } + overrides = { + 'client.0': {'tag': 'v4.1'}, + } + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'client.0': {'tag': 'v4.1'}, + } + assert t == TIMEOUT_DEFAULT + + def test_override_actual(self): + config = { + 'osd.1': {'tag': 'v4.1'}, + 'client.0': {'branch': 'testing'}, + } + overrides = { + 'osd.1': {'koji': 1234, 'kdb': True}, + } + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'osd.1': {'koji': 1234, 'kdb': True}, + 'client.0': {'branch': 'testing'}, + } + assert t == TIMEOUT_DEFAULT + + def test_override_actual_with_generic(self): + config = { + 'osd.1': {'tag': 'v4.1', 'kdb': False}, + 'client.0': {'branch': 'testing'}, + } + overrides = { + 'osd': {'koji': 1234}, + } + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'osd.0': {'koji': 1234}, + 'osd.1': {'koji': 1234, 'kdb': False}, + 'osd.2': {'koji': 1234}, + 'client.0': {'branch': 'testing'}, + } + assert t == TIMEOUT_DEFAULT + + def test_override_actual_with_top_level(self): + config = { + 'osd.1': {'tag': 'v4.1'}, + 'client.0': {'branch': 'testing', 'kdb': False}, + } + overrides = {'koji': 1234, 'kdb': True} + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'mon.a': {'koji': 1234, 'kdb': True}, + 'osd.0': {'koji': 1234, 'kdb': True}, + 'osd.1': {'koji': 1234, 'kdb': True}, + 'osd.2': {'koji': 1234, 'kdb': True}, + 'client.0': {'koji': 1234, 'kdb': True}, + 'client.1': {'koji': 1234, 'kdb': True}, + } + assert t == TIMEOUT_DEFAULT + + def test_override_generic(self): + config = { + 'osd': {'tag': 'v4.1'}, + 'client': {'branch': 'testing'}, + } + overrides = { + 'client': {'koji': 1234, 'kdb': True}, + } + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'osd.0': {'tag': 'v4.1'}, + 'osd.1': {'tag': 'v4.1'}, + 'osd.2': {'tag': 'v4.1'}, + 'client.0': {'koji': 1234, 'kdb': True}, + 'client.1': {'koji': 1234, 'kdb': True}, + } + assert t == TIMEOUT_DEFAULT + + def test_override_generic_with_top_level(self): + config = { + 'osd': {'tag': 'v4.1'}, + 'client': {'branch': 'testing', 'kdb': False}, + } + overrides = { + 'client': {'koji': 1234}, + } + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'osd.0': {'tag': 'v4.1'}, + 'osd.1': {'tag': 'v4.1'}, + 'osd.2': {'tag': 'v4.1'}, + 'client.0': {'koji': 1234, 'kdb': False}, + 'client.1': {'koji': 1234, 'kdb': False}, + } + assert t == TIMEOUT_DEFAULT + + def test_override_generic_with_actual(self): + config = { + 'osd': {'tag': 'v4.1', 'kdb': False}, + 'client': {'branch': 'testing'}, + } + overrides = { + 'osd.2': {'koji': 1234, 'kdb': True}, + } + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'osd.0': {'tag': 'v4.1', 'kdb': False}, + 'osd.1': {'tag': 'v4.1', 'kdb': False}, + 'osd.2': {'koji': 1234, 'kdb': True}, + 'client.0': {'branch': 'testing'}, + 'client.1': {'branch': 'testing'}, + } + assert t == TIMEOUT_DEFAULT + + def test_override_top_level(self): + config = {'branch': 'testing'} + overrides = {'koji': 1234, 'kdb': True} + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'mon.a': {'koji': 1234, 'kdb': True}, + 'osd.0': {'koji': 1234, 'kdb': True}, + 'osd.1': {'koji': 1234, 'kdb': True}, + 'osd.2': {'koji': 1234, 'kdb': True}, + 'client.0': {'koji': 1234, 'kdb': True}, + 'client.1': {'koji': 1234, 'kdb': True}, + } + assert t == TIMEOUT_DEFAULT + + def test_override_top_level_with_actual(self): + config = {'branch': 'testing', 'kdb': False} + overrides = { + 'mon.a': {'koji': 1234}, + } + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'mon.a': {'koji': 1234, 'kdb': False}, + 'osd.0': {'branch': 'testing', 'kdb': False}, + 'osd.1': {'branch': 'testing', 'kdb': False}, + 'osd.2': {'branch': 'testing', 'kdb': False}, + 'client.0': {'branch': 'testing', 'kdb': False}, + 'client.1': {'branch': 'testing', 'kdb': False}, + } + assert t == TIMEOUT_DEFAULT + + def test_override_top_level_with_generic(self): + config = {'branch': 'testing', 'kdb': False} + overrides = { + 'client': {'koji': 1234, 'kdb': True}, + } + config, t = normalize_and_apply_overrides(self.ctx, config, overrides) + assert config == { + 'mon.a': {'branch': 'testing', 'kdb': False}, + 'osd.0': {'branch': 'testing', 'kdb': False}, + 'osd.1': {'branch': 'testing', 'kdb': False}, + 'osd.2': {'branch': 'testing', 'kdb': False}, + 'client.0': {'koji': 1234, 'kdb': True}, + 'client.1': {'koji': 1234, 'kdb': True}, + } + assert t == TIMEOUT_DEFAULT -- 2.39.5