From c886a936882c99978d756d3616bfcfc068fd0a6d Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 9 Jul 2014 14:28:24 -0600 Subject: [PATCH] Make teuthology.config *far* more robust In addition to parsing its configuration from a yaml file, it can now be created from a dict. It can also be dumped as a dict, or as a yaml stream. It is also now split into multiple classes so that the same implementation can be used for job configs, and not just as a proxy for ~/.teuthology.yaml. Signed-off-by: Zack Cerza --- teuthology/config.py | 110 ++++++++++++++++--- teuthology/kill.py | 2 +- teuthology/orchestra/test/test_connection.py | 2 +- teuthology/report.py | 2 +- teuthology/test/test_config.py | 81 ++++++++++++-- 5 files changed, 169 insertions(+), 28 deletions(-) diff --git a/teuthology/config.py b/teuthology/config.py index 2fd6ff565a29a..3e6c14d996fe7 100644 --- a/teuthology/config.py +++ b/teuthology/config.py @@ -2,16 +2,101 @@ import os import yaml import logging -log = logging.getLogger(__name__) +def init_logging(): + log = logging.getLogger(__name__) + return log -class Config(object): +log = init_logging() + + +class YamlConfig(object): + defaults = dict() + + def __init__(self, yaml_path=None): + self.yaml_path = yaml_path + if self.yaml_path: + self.load() + else: + self.__conf = dict() + + def load(self): + if os.path.exists(self.yaml_path): + self.__conf = yaml.safe_load(file(self.yaml_path)) + else: + log.debug("%s not found", self.yaml_path) + self.__conf = dict() + + def update(self, in_dict): + """ + Update an existing configuration using dict.update() + + :param in_dict: The dict to use to update + """ + self.__conf.update(in_dict) + + @classmethod + def from_dict(cls, in_dict): + """ + Build a config object from a dict. + + :param in_dict: The dict to use + :returns: The config object + """ + conf_obj = cls() + conf_obj.__conf = in_dict + return conf_obj + + def to_dict(self): + """ + :returns: A shallow copy of the configuration as a dict + """ + return dict(self.__conf) + + @classmethod + def from_str(cls, in_str): + """ + Build a config object from a string or yaml stream. + + :param in_str: The stream or string + :returns: The config object + """ + conf_obj = cls() + conf_obj.__conf = yaml.safe_load(in_str) + return conf_obj + + def to_str(self): + """ + :returns: str(self) + """ + return str(self) + + def __str__(self): + return yaml.safe_dump(self.__conf, default_flow_style=False).strip() + + def __getitem__(self, name): + return self.__conf.__getitem__(name) + + def __getattr__(self, name): + return self.__conf.get(name, self.defaults.get(name)) + + def __setattr__(self, name, value): + if name.endswith('__conf') or name in ('yaml_path'): + object.__setattr__(self, name, value) + else: + self.__conf[name] = value + + def __delattr__(self, name): + del self.__conf[name] + + +class TeuthologyConfig(YamlConfig): """ This class is intended to unify teuthology's many configuration files and objects. Currently it serves as a convenient interface to ~/.teuthology.yaml and nothing else. """ - teuthology_yaml = os.path.join(os.environ['HOME'], '.teuthology.yaml') + yaml_path = os.path.join(os.environ['HOME'], '.teuthology.yaml') defaults = { 'archive_base': '/var/lib/teuthworker/archive', 'automated_scheduling': False, @@ -24,22 +109,11 @@ class Config(object): } def __init__(self): - self.load_files() + super(TeuthologyConfig, self).__init__(self.yaml_path) - def load_files(self): - if os.path.exists(self.teuthology_yaml): - self.__conf = yaml.safe_load(file(self.teuthology_yaml)) - else: - log.debug("%s not found", self.teuthology_yaml) - self.__conf = {} - def __getattr__(self, name): - return self.__conf.get(name, self.defaults.get(name)) +class JobConfig(YamlConfig): + pass - def __setattribute__(self, name, value): - if name.endswith('__conf'): - setattr(self, name, value) - else: - self.__conf[name] = value -config = Config() +config = TeuthologyConfig() diff --git a/teuthology/kill.py b/teuthology/kill.py index 3eef680ab1cea..bbdfc5bc4e7b2 100755 --- a/teuthology/kill.py +++ b/teuthology/kill.py @@ -107,7 +107,7 @@ def remove_beanstalk_jobs(run_name, tube_name): if qhost is None or qport is None: raise RuntimeError( 'Beanstalk queue information not found in {conf_path}'.format( - conf_path=config.teuthology_yaml)) + conf_path=config.yaml_path)) log.info("Checking Beanstalk Queue...") beanstalk_conn = beanstalk.connect() real_tube_name = beanstalk.watch_tube(beanstalk_conn, tube_name) diff --git a/teuthology/orchestra/test/test_connection.py b/teuthology/orchestra/test/test_connection.py index 5dd3861860f94..ef13c1edff3f3 100644 --- a/teuthology/orchestra/test/test_connection.py +++ b/teuthology/orchestra/test/test_connection.py @@ -12,7 +12,7 @@ class TestConnection(object): def clear_config(self): config.config.teuthology_yaml = '' - config.config.load_files() + config.config.load() def test_split_user_just_host(self): got = connection.split_user('somehost.example.com') diff --git a/teuthology/report.py b/teuthology/report.py index cb0489e09db5b..95ed0b34d25ea 100644 --- a/teuthology/report.py +++ b/teuthology/report.py @@ -190,7 +190,7 @@ class ResultsReporter(object): if not self.base_uri: msg = "No results_server set in {yaml}; cannot report results" - self.log.warn(msg.format(yaml=config.teuthology_yaml)) + self.log.warn(msg.format(yaml=config.yaml_path)) def _make_session(self, max_retries=10): session = requests.Session() diff --git a/teuthology/test/test_config.py b/teuthology/test/test_config.py index f94ff35fdaeba..5b443a2a30186 100644 --- a/teuthology/test/test_config.py +++ b/teuthology/test/test_config.py @@ -1,19 +1,86 @@ from .. import config -class TestConfig(object): +class TestYamlConfig(object): + def test_set_multiple(self): + conf_obj = config.YamlConfig() + conf_obj.foo = 'foo' + conf_obj.bar = 'bar' + assert conf_obj.foo == 'foo' + assert conf_obj.bar == 'bar' + assert conf_obj.to_dict()['foo'] == 'foo' + + def test_from_dict(self): + in_dict = dict(foo='bar') + conf_obj = config.YamlConfig.from_dict(in_dict) + assert conf_obj.foo == 'bar' + + def test_to_dict(self): + in_dict = dict(foo='bar') + conf_obj = config.YamlConfig.from_dict(in_dict) + assert conf_obj.to_dict() == in_dict + + def test_from_str(self): + in_str = "foo: bar" + conf_obj = config.YamlConfig.from_str(in_str) + assert conf_obj.foo == 'bar' + + def test_to_str(self): + in_str = "foo: bar" + conf_obj = config.YamlConfig.from_str(in_str) + assert conf_obj.to_str() == in_str + + def test_update(self): + conf_obj = config.YamlConfig() + conf_obj.foo = 'foo' + conf_obj.bar = 'bar' + conf_obj.update(dict(bar='baz')) + assert conf_obj.to_dict() == dict(foo='foo', bar='baz') + + def test_defaults(self): + conf_obj = config.YamlConfig() + # Save a copy of the original defaults so we can restore them later. + # Not doing so would break other tests. + old_defaults = dict(conf_obj.defaults) + conf_obj.defaults['foo'] = 'bar' + assert conf_obj.foo == 'bar' + # restore defaults + conf_obj.__class__.defaults = old_defaults + + def test_delattr(self): + conf_obj = config.YamlConfig() + conf_obj.foo = 'bar' + assert conf_obj.foo == 'bar' + del conf_obj.foo + assert conf_obj.foo is None + + +class TestTeuthologyConfig(object): + def test_defaults(self): + conf_obj = config.TeuthologyConfig() + conf_obj.defaults['archive_base'] = 'bar' + assert conf_obj.archive_base == 'bar' + def test_get_ceph_git_base_default(self): - conf_obj = config.Config() - conf_obj.teuthology_yaml = '' - conf_obj.load_files() + conf_obj = config.TeuthologyConfig() + conf_obj.yaml_path = '' + conf_obj.load() assert conf_obj.ceph_git_base_url == "https://github.com/ceph/" def test_set_ceph_git_base_via_private(self): - conf_obj = config.Config() - conf_obj._Config__conf['ceph_git_base_url'] = "git://ceph.com/" + conf_obj = config.TeuthologyConfig() + conf_obj._YamlConfig__conf['ceph_git_base_url'] = \ + "git://ceph.com/" assert conf_obj.ceph_git_base_url == "git://ceph.com/" def test_set_nonstandard(self): - conf_obj = config.Config() + conf_obj = config.TeuthologyConfig() conf_obj.something = 'something else' assert conf_obj.something == 'something else' + + +class TestJobConfig(object): + def test_to_str(self): + in_str = "foo: bar" + conf_obj = config.JobConfig.from_str(in_str) + assert conf_obj.to_str() == in_str -- 2.39.5